const fs = require('fs'); const path = require('path'); const readline = require('readline'); const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880'; const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-douyin-worker'; const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token'; const DEFAULT_ACCOUNT_ID = process.env.DOUYIN_ACCOUNT_ID || 'douyin_company'; const DEFAULT_CREATOR_URL = process.env.DOUYIN_CREATOR_URL || 'https://creator.douyin.com'; const DEFAULT_CONTENT_URL = process.env.DOUYIN_CONTENT_URL || 'https://creator.douyin.com/creator-micro/content/manage'; const DEFAULT_PROFILE_ROOT = process.env.DOUYIN_PROFILE_ROOT || path.resolve('data/platform-profiles'); const LOGIN_TEXT = new RegExp([ '\\u626b\\u7801\\u767b\\u5f55', '\\u9a8c\\u8bc1\\u7801', '\\u624b\\u673a\\u53f7', '\\u767b\\u5f55' ].join('|')); const CREATOR_TEXT = new RegExp([ '\\u521b\\u4f5c\\u670d\\u52a1', '\\u53d1\\u5e03\\u4f5c\\u54c1', '\\u5185\\u5bb9\\u7ba1\\u7406', '\\u6570\\u636e\\u4e2d\\u5fc3', '\\u4f5c\\u54c1\\u7ba1\\u7406', '\\u53d1\\u5e03\\u89c6\\u9891' ].join('|')); function requirePlaywright() { try { return require('playwright'); } catch (error) { throw new Error('Playwright is not installed. Run npm.cmd install in the project root first.'); } } function profileDir(accountId = DEFAULT_ACCOUNT_ID) { return path.resolve(DEFAULT_PROFILE_ROOT, accountId); } function ensureDir(dir) { fs.mkdirSync(dir, { recursive: true }); } function browserChannel() { if (process.env.DOUYIN_BROWSER_CHANNEL) { return process.env.DOUYIN_BROWSER_CHANNEL; } return process.platform === 'win32' ? 'msedge' : undefined; } async function launchDouyinContext({ accountId = DEFAULT_ACCOUNT_ID, headless = false } = {}) { const { chromium } = requirePlaywright(); const dir = profileDir(accountId); ensureDir(dir); const channel = browserChannel(); const options = { headless, viewport: { width: 1440, height: 1000 }, acceptDownloads: true, args: [ '--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage' ] }; if (channel) { options.channel = channel; } return chromium.launchPersistentContext(dir, options); } async function waitForEnter(message) { const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); return new Promise(resolve => { rl.question(message, () => { rl.close(); resolve(); }); }); } async function login() { const accountId = process.argv[3] || DEFAULT_ACCOUNT_ID; const context = await launchDouyinContext({ accountId, headless: false }); const page = context.pages()[0] || await context.newPage(); await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); console.log(`Douyin login profile: ${profileDir(accountId)}`); console.log('Please scan/login in the opened browser. Keep this window open until login succeeds.'); await waitForEnter('Press Enter here after Douyin creator center is logged in...'); const status = await detectLogin(page, accountId); await context.close(); console.log(JSON.stringify(status, null, 2)); } async function checkLogin(accountId = DEFAULT_ACCOUNT_ID) { const context = await launchDouyinContext({ accountId, headless: true }); const page = context.pages()[0] || await context.newPage(); try { await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.waitForTimeout(2500); return await detectLogin(page, accountId); } finally { await context.close(); } } async function detectLogin(page, accountId = DEFAULT_ACCOUNT_ID) { const url = page.url(); const title = await page.title().catch(() => ''); const bodyText = await page.locator('body').innerText({ timeout: 5000 }).catch(() => ''); const loginVisible = LOGIN_TEXT.test(bodyText); const creatorVisible = CREATOR_TEXT.test(bodyText); const loggedIn = creatorVisible || (!loginVisible && /creator\.douyin\.com/.test(url)); return { loggedIn, status: loggedIn ? 'SUCCESS' : 'FAILED', message: loggedIn ? 'Douyin creator profile is logged in.' : 'Douyin creator profile is not logged in.', url, title, profileDir: profileDir(accountId) }; } async function collectMetrics(task) { const payload = task.payload || {}; const accountId = task.accountId || DEFAULT_ACCOUNT_ID; const targetUrl = payload.metricsUrl || payload.contentUrl || DEFAULT_CONTENT_URL; const headless = payload.headless === false ? false : true; const context = await launchDouyinContext({ accountId, headless }); const page = context.pages()[0] || await context.newPage(); try { await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); await page.waitForTimeout(Number(payload.waitMs || 5000)); const status = await detectLogin(page, accountId); if (!status.loggedIn) { return { status: 'FAILED', errorMessage: status.message, result: { action: 'sync_metrics', loggedIn: false, ...status } }; } const text = await page.locator('body').innerText({ timeout: 15000 }).catch(() => ''); const screenshot = await saveScreenshot(page, task.id); const videos = await collectVideoMetrics(page, Number(payload.maxVideos || 20)); const metrics = videos.length > 0 ? summarizeVideos(videos) : extractMetrics(text); return { status: 'SUCCESS', errorMessage: null, result: { action: 'sync_metrics', loggedIn: true, url: page.url(), title: await page.title().catch(() => ''), metrics, videos, videoCount: videos.length, extractionMode: videos.length > 0 ? 'video_list' : 'page_summary', screenshot, pageTextPreview: compactText(text).slice(0, 2000) } }; } finally { await context.close().catch(() => {}); } } async function saveScreenshot(page, taskId) { const dir = path.resolve('target/media/douyin'); ensureDir(dir); const file = path.join(dir, `${taskId || Date.now()}-metrics.png`); await page.screenshot({ path: file, fullPage: true }).catch(() => {}); return file; } function compactText(text) { return (text || '').replace(/\s+/g, ' ').trim(); } function extractMetrics(text) { const compact = compactText(text); return { views: findMetric(compact, ['\\u64ad\\u653e', '\\u6d4f\\u89c8', '\\u89c2\\u770b', '\\u5c55\\u73b0']), likes: findMetric(compact, ['\\u70b9\\u8d5e']), comments: findMetric(compact, ['\\u8bc4\\u8bba']), collects: findMetric(compact, ['\\u6536\\u85cf']), shares: findMetric(compact, ['\\u5206\\u4eab', '\\u8f6c\\u53d1']) }; } async function collectVideoMetrics(page, limit = 20) { await autoScroll(page); const blocks = await page.evaluate(maxBlocks => { const selectors = [ 'tr', 'article', 'li', '[class*="card"]', '[class*="item"]', '[class*="work"]', '[class*="video"]', '[class*="content"]' ]; const seen = new Set(); const rows = []; for (const selector of selectors) { for (const element of document.querySelectorAll(selector)) { if (rows.length >= maxBlocks) { break; } const rect = element.getBoundingClientRect(); if (rect.width < 160 || rect.height < 40) { continue; } const text = (element.innerText || '').replace(/\s+/g, ' ').trim(); if (text.length < 12 || text.length > 1200) { continue; } const signature = text.slice(0, 120); if (seen.has(signature)) { continue; } seen.add(signature); const link = element.querySelector('a[href]'); rows.push({ text, href: link ? link.href : '', rect: { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) } }); } } return rows; }, Math.max(limit * 8, 80)); const videos = []; const seen = new Set(); for (const block of blocks) { const video = parseVideoBlock(block); if (!video) { continue; } const key = `${video.title}|${video.views}|${video.likes}|${video.comments}|${video.href}`; if (seen.has(key)) { continue; } seen.add(key); videos.push(video); if (videos.length >= limit) { break; } } return videos; } async function autoScroll(page) { await page.evaluate(async () => { await new Promise(resolve => { let count = 0; const timer = setInterval(() => { window.scrollBy(0, Math.floor(window.innerHeight * 0.8)); count += 1; if (count >= 3) { clearInterval(timer); resolve(); } }, 500); }); }).catch(() => {}); await page.waitForTimeout(800).catch(() => {}); } function parseVideoBlock(block) { const text = compactText(block.text); const metrics = extractMetrics(text); const score = metrics.views + metrics.likes + metrics.comments + metrics.collects + metrics.shares; if (score <= 0) { return null; } const title = extractVideoTitle(text); return { title, href: block.href || '', views: metrics.views, likes: metrics.likes, comments: metrics.comments, collects: metrics.collects, shares: metrics.shares, interactionCount: metrics.likes + metrics.comments + metrics.collects + metrics.shares, interactionRate: metrics.views <= 0 ? 0 : (metrics.likes + metrics.comments + metrics.collects + metrics.shares) / metrics.views, rawTextPreview: text.slice(0, 500) }; } function extractVideoTitle(text) { const cleaned = text .replace(/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}.*/g, '') .replace(/(\u64ad\u653e|\u6d4f\u89c8|\u89c2\u770b|\u5c55\u73b0|\u70b9\u8d5e|\u8bc4\u8bba|\u6536\u85cf|\u5206\u4eab|\u8f6c\u53d1).*/g, '') .trim(); if (cleaned.length >= 2) { return cleaned.slice(0, 120); } return text.slice(0, 80); } function summarizeVideos(videos) { const summary = { views: 0, likes: 0, comments: 0, collects: 0, shares: 0 }; for (const video of videos) { summary.views += Number(video.views || 0); summary.likes += Number(video.likes || 0); summary.comments += Number(video.comments || 0); summary.collects += Number(video.collects || 0); summary.shares += Number(video.shares || 0); } return summary; } function findMetric(text, labels) { for (const label of labels) { const labelPattern = new RegExp(label); const decodedLabel = label.replace(/\\u([0-9a-fA-F]{4})/g, (_, code) => String.fromCharCode(parseInt(code, 16))); const after = new RegExp(`${label}[^0-9\\d]{0,16}([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)`); const before = new RegExp(`([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)[^0-9\\d]{0,16}${label}`); if (labelPattern.test(text)) { const afterMatch = text.match(after); if (afterMatch) { return parseMetricNumber(afterMatch[1], afterMatch[2]); } const beforeMatch = text.match(before); if (beforeMatch) { return parseMetricNumber(beforeMatch[1], beforeMatch[2]); } const looseIndex = text.indexOf(decodedLabel); if (looseIndex >= 0) { const around = text.slice(Math.max(0, looseIndex - 30), looseIndex + 60); const number = around.match(/([0-9][0-9,.]*)(\s*[\u4e07\u5343wWkK]?)/); if (number) { return parseMetricNumber(number[1], number[2]); } } } } return 0; } function parseMetricNumber(raw, unit = '') { const value = Number(String(raw).replace(/,/g, '')); if (!Number.isFinite(value)) { return 0; } const normalizedUnit = unit.trim().toLowerCase(); if (normalizedUnit.includes('\u4e07') || normalizedUnit.includes('w')) { return Math.round(value * 10000); } if (normalizedUnit.includes('\u5343') || normalizedUnit.includes('k')) { return Math.round(value * 1000); } return Math.round(value); } async function pollTasks() { const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Worker-Token': DEFAULT_WORKER_TOKEN }, body: JSON.stringify({ workerId: DEFAULT_WORKER_ID, platforms: ['douyin'], maxTasks: 5 }) }); if (!response.ok) { throw new Error(`Poll failed: ${response.status} ${await response.text()}`); } return response.json(); } async function submitResult(taskId, body) { const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Worker-Token': DEFAULT_WORKER_TOKEN }, body: JSON.stringify(body) }); if (!response.ok) { throw new Error(`Submit result failed: ${response.status} ${await response.text()}`); } return response.json(); } async function handleTask(task) { const payload = task.payload || {}; const action = payload.action || (task.taskType === 'SYNC_METRICS' ? 'sync_metrics' : 'check_login'); if (action === 'check_login') { const status = await checkLogin(task.accountId || DEFAULT_ACCOUNT_ID); return submitResult(task.id, { status: status.loggedIn ? 'SUCCESS' : 'FAILED', errorMessage: status.loggedIn ? null : status.message, result: { action, loggedIn: status.loggedIn, message: status.message, url: status.url, title: status.title, profileDir: profileDir(task.accountId || DEFAULT_ACCOUNT_ID) } }); } if (action === 'sync_metrics') { const result = await collectMetrics(task); return submitResult(task.id, result); } return submitResult(task.id, { status: 'FAILED', errorMessage: `Unsupported local Douyin worker action: ${action}`, result: { action } }); } async function workerOnce() { const tasks = await pollTasks(); if (!Array.isArray(tasks) || tasks.length === 0) { console.log('No pending Douyin tasks.'); return; } for (const task of tasks) { const result = await handleTask(task); console.log(`Completed ${task.id}: ${result.status}`); } } function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } async function workerLoop() { const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000); console.log(`Douyin worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`); while (true) { try { await workerOnce(); } catch (error) { console.error(error.stack || error.message); } await sleep(intervalMs); } } async function main() { const mode = process.argv[2]; if (mode === 'login') { await login(); return; } if (mode === 'check-login') { const status = await checkLogin(process.argv[3] || DEFAULT_ACCOUNT_ID); console.log(JSON.stringify(status, null, 2)); process.exit(status.loggedIn ? 0 : 2); } if (mode === 'metrics') { const task = { id: `manual_${Date.now()}`, taskType: 'SYNC_METRICS', platform: 'douyin', accountId: process.argv[3] || DEFAULT_ACCOUNT_ID, payload: { action: 'sync_metrics', headless: false } }; console.log(JSON.stringify((await collectMetrics(task)).result, null, 2)); return; } if (mode === 'worker') { await workerOnce(); return; } if (mode === 'worker-loop') { await workerLoop(); return; } throw new Error('Usage: node scripts/local_douyin_worker.js [accountId]'); } main().catch(error => { console.error(error.stack || error.message); process.exit(1); });