508 lines
16 KiB
JavaScript
508 lines
16 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const readline = require('readline');
|
|
|
|
const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880';
|
|
const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-douyin-worker';
|
|
const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token';
|
|
const DEFAULT_ACCOUNT_ID = process.env.DOUYIN_ACCOUNT_ID || 'douyin_company';
|
|
const DEFAULT_CREATOR_URL = process.env.DOUYIN_CREATOR_URL || 'https://creator.douyin.com';
|
|
const DEFAULT_CONTENT_URL = process.env.DOUYIN_CONTENT_URL || 'https://creator.douyin.com/creator-micro/content/manage';
|
|
const DEFAULT_PROFILE_ROOT = process.env.DOUYIN_PROFILE_ROOT || path.resolve('data/platform-profiles');
|
|
|
|
const LOGIN_TEXT = new RegExp([
|
|
'\\u626b\\u7801\\u767b\\u5f55',
|
|
'\\u9a8c\\u8bc1\\u7801',
|
|
'\\u624b\\u673a\\u53f7',
|
|
'\\u767b\\u5f55'
|
|
].join('|'));
|
|
|
|
const CREATOR_TEXT = new RegExp([
|
|
'\\u521b\\u4f5c\\u670d\\u52a1',
|
|
'\\u53d1\\u5e03\\u4f5c\\u54c1',
|
|
'\\u5185\\u5bb9\\u7ba1\\u7406',
|
|
'\\u6570\\u636e\\u4e2d\\u5fc3',
|
|
'\\u4f5c\\u54c1\\u7ba1\\u7406',
|
|
'\\u53d1\\u5e03\\u89c6\\u9891'
|
|
].join('|'));
|
|
|
|
function requirePlaywright() {
|
|
try {
|
|
return require('playwright');
|
|
} catch (error) {
|
|
throw new Error('Playwright is not installed. Run npm.cmd install in the project root first.');
|
|
}
|
|
}
|
|
|
|
function profileDir(accountId = DEFAULT_ACCOUNT_ID) {
|
|
return path.resolve(DEFAULT_PROFILE_ROOT, accountId);
|
|
}
|
|
|
|
function ensureDir(dir) {
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
}
|
|
|
|
function browserChannel() {
|
|
if (process.env.DOUYIN_BROWSER_CHANNEL) {
|
|
return process.env.DOUYIN_BROWSER_CHANNEL;
|
|
}
|
|
return process.platform === 'win32' ? 'msedge' : undefined;
|
|
}
|
|
|
|
async function launchDouyinContext({ accountId = DEFAULT_ACCOUNT_ID, headless = false } = {}) {
|
|
const { chromium } = requirePlaywright();
|
|
const dir = profileDir(accountId);
|
|
ensureDir(dir);
|
|
const channel = browserChannel();
|
|
const options = {
|
|
headless,
|
|
viewport: { width: 1440, height: 1000 },
|
|
acceptDownloads: true,
|
|
args: [
|
|
'--disable-blink-features=AutomationControlled',
|
|
'--disable-dev-shm-usage'
|
|
]
|
|
};
|
|
if (channel) {
|
|
options.channel = channel;
|
|
}
|
|
return chromium.launchPersistentContext(dir, options);
|
|
}
|
|
|
|
async function waitForEnter(message) {
|
|
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
return new Promise(resolve => {
|
|
rl.question(message, () => {
|
|
rl.close();
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
async function login() {
|
|
const accountId = process.argv[3] || DEFAULT_ACCOUNT_ID;
|
|
const context = await launchDouyinContext({ accountId, headless: false });
|
|
const page = context.pages()[0] || await context.newPage();
|
|
await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
|
console.log(`Douyin login profile: ${profileDir(accountId)}`);
|
|
console.log('Please scan/login in the opened browser. Keep this window open until login succeeds.');
|
|
await waitForEnter('Press Enter here after Douyin creator center is logged in...');
|
|
const status = await detectLogin(page, accountId);
|
|
await context.close();
|
|
console.log(JSON.stringify(status, null, 2));
|
|
}
|
|
|
|
async function checkLogin(accountId = DEFAULT_ACCOUNT_ID) {
|
|
const context = await launchDouyinContext({ accountId, headless: true });
|
|
const page = context.pages()[0] || await context.newPage();
|
|
try {
|
|
await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
|
await page.waitForTimeout(2500);
|
|
return await detectLogin(page, accountId);
|
|
} finally {
|
|
await context.close();
|
|
}
|
|
}
|
|
|
|
async function detectLogin(page, accountId = DEFAULT_ACCOUNT_ID) {
|
|
const url = page.url();
|
|
const title = await page.title().catch(() => '');
|
|
const bodyText = await page.locator('body').innerText({ timeout: 5000 }).catch(() => '');
|
|
const loginVisible = LOGIN_TEXT.test(bodyText);
|
|
const creatorVisible = CREATOR_TEXT.test(bodyText);
|
|
const loggedIn = creatorVisible || (!loginVisible && /creator\.douyin\.com/.test(url));
|
|
return {
|
|
loggedIn,
|
|
status: loggedIn ? 'SUCCESS' : 'FAILED',
|
|
message: loggedIn ? 'Douyin creator profile is logged in.' : 'Douyin creator profile is not logged in.',
|
|
url,
|
|
title,
|
|
profileDir: profileDir(accountId)
|
|
};
|
|
}
|
|
|
|
async function collectMetrics(task) {
|
|
const payload = task.payload || {};
|
|
const accountId = task.accountId || DEFAULT_ACCOUNT_ID;
|
|
const targetUrl = payload.metricsUrl || payload.contentUrl || DEFAULT_CONTENT_URL;
|
|
const headless = payload.headless === false ? false : true;
|
|
const context = await launchDouyinContext({ accountId, headless });
|
|
const page = context.pages()[0] || await context.newPage();
|
|
try {
|
|
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
|
await page.waitForTimeout(Number(payload.waitMs || 5000));
|
|
const status = await detectLogin(page, accountId);
|
|
if (!status.loggedIn) {
|
|
return {
|
|
status: 'FAILED',
|
|
errorMessage: status.message,
|
|
result: {
|
|
action: 'sync_metrics',
|
|
loggedIn: false,
|
|
...status
|
|
}
|
|
};
|
|
}
|
|
|
|
const text = await page.locator('body').innerText({ timeout: 15000 }).catch(() => '');
|
|
const screenshot = await saveScreenshot(page, task.id);
|
|
const videos = await collectVideoMetrics(page, Number(payload.maxVideos || 20));
|
|
const metrics = videos.length > 0 ? summarizeVideos(videos) : extractMetrics(text);
|
|
return {
|
|
status: 'SUCCESS',
|
|
errorMessage: null,
|
|
result: {
|
|
action: 'sync_metrics',
|
|
loggedIn: true,
|
|
url: page.url(),
|
|
title: await page.title().catch(() => ''),
|
|
metrics,
|
|
videos,
|
|
videoCount: videos.length,
|
|
extractionMode: videos.length > 0 ? 'video_list' : 'page_summary',
|
|
screenshot,
|
|
pageTextPreview: compactText(text).slice(0, 2000)
|
|
}
|
|
};
|
|
} finally {
|
|
await context.close().catch(() => {});
|
|
}
|
|
}
|
|
|
|
async function saveScreenshot(page, taskId) {
|
|
const dir = path.resolve('target/media/douyin');
|
|
ensureDir(dir);
|
|
const file = path.join(dir, `${taskId || Date.now()}-metrics.png`);
|
|
await page.screenshot({ path: file, fullPage: true }).catch(() => {});
|
|
return file;
|
|
}
|
|
|
|
function compactText(text) {
|
|
return (text || '').replace(/\s+/g, ' ').trim();
|
|
}
|
|
|
|
function extractMetrics(text) {
|
|
const compact = compactText(text);
|
|
return {
|
|
views: findMetric(compact, ['\\u64ad\\u653e', '\\u6d4f\\u89c8', '\\u89c2\\u770b', '\\u5c55\\u73b0']),
|
|
likes: findMetric(compact, ['\\u70b9\\u8d5e']),
|
|
comments: findMetric(compact, ['\\u8bc4\\u8bba']),
|
|
collects: findMetric(compact, ['\\u6536\\u85cf']),
|
|
shares: findMetric(compact, ['\\u5206\\u4eab', '\\u8f6c\\u53d1'])
|
|
};
|
|
}
|
|
|
|
async function collectVideoMetrics(page, limit = 20) {
|
|
await autoScroll(page);
|
|
const blocks = await page.evaluate(maxBlocks => {
|
|
const selectors = [
|
|
'tr',
|
|
'article',
|
|
'li',
|
|
'[class*="card"]',
|
|
'[class*="item"]',
|
|
'[class*="work"]',
|
|
'[class*="video"]',
|
|
'[class*="content"]'
|
|
];
|
|
const seen = new Set();
|
|
const rows = [];
|
|
for (const selector of selectors) {
|
|
for (const element of document.querySelectorAll(selector)) {
|
|
if (rows.length >= maxBlocks) {
|
|
break;
|
|
}
|
|
const rect = element.getBoundingClientRect();
|
|
if (rect.width < 160 || rect.height < 40) {
|
|
continue;
|
|
}
|
|
const text = (element.innerText || '').replace(/\s+/g, ' ').trim();
|
|
if (text.length < 12 || text.length > 1200) {
|
|
continue;
|
|
}
|
|
const signature = text.slice(0, 120);
|
|
if (seen.has(signature)) {
|
|
continue;
|
|
}
|
|
seen.add(signature);
|
|
const link = element.querySelector('a[href]');
|
|
rows.push({
|
|
text,
|
|
href: link ? link.href : '',
|
|
rect: {
|
|
x: Math.round(rect.x),
|
|
y: Math.round(rect.y),
|
|
width: Math.round(rect.width),
|
|
height: Math.round(rect.height)
|
|
}
|
|
});
|
|
}
|
|
}
|
|
return rows;
|
|
}, Math.max(limit * 8, 80));
|
|
|
|
const videos = [];
|
|
const seen = new Set();
|
|
for (const block of blocks) {
|
|
const video = parseVideoBlock(block);
|
|
if (!video) {
|
|
continue;
|
|
}
|
|
const key = `${video.title}|${video.views}|${video.likes}|${video.comments}|${video.href}`;
|
|
if (seen.has(key)) {
|
|
continue;
|
|
}
|
|
seen.add(key);
|
|
videos.push(video);
|
|
if (videos.length >= limit) {
|
|
break;
|
|
}
|
|
}
|
|
return videos;
|
|
}
|
|
|
|
async function autoScroll(page) {
|
|
await page.evaluate(async () => {
|
|
await new Promise(resolve => {
|
|
let count = 0;
|
|
const timer = setInterval(() => {
|
|
window.scrollBy(0, Math.floor(window.innerHeight * 0.8));
|
|
count += 1;
|
|
if (count >= 3) {
|
|
clearInterval(timer);
|
|
resolve();
|
|
}
|
|
}, 500);
|
|
});
|
|
}).catch(() => {});
|
|
await page.waitForTimeout(800).catch(() => {});
|
|
}
|
|
|
|
function parseVideoBlock(block) {
|
|
const text = compactText(block.text);
|
|
const metrics = extractMetrics(text);
|
|
const score = metrics.views + metrics.likes + metrics.comments + metrics.collects + metrics.shares;
|
|
if (score <= 0) {
|
|
return null;
|
|
}
|
|
const title = extractVideoTitle(text);
|
|
return {
|
|
title,
|
|
href: block.href || '',
|
|
views: metrics.views,
|
|
likes: metrics.likes,
|
|
comments: metrics.comments,
|
|
collects: metrics.collects,
|
|
shares: metrics.shares,
|
|
interactionCount: metrics.likes + metrics.comments + metrics.collects + metrics.shares,
|
|
interactionRate: metrics.views <= 0
|
|
? 0
|
|
: (metrics.likes + metrics.comments + metrics.collects + metrics.shares) / metrics.views,
|
|
rawTextPreview: text.slice(0, 500)
|
|
};
|
|
}
|
|
|
|
function extractVideoTitle(text) {
|
|
const cleaned = text
|
|
.replace(/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}.*/g, '')
|
|
.replace(/(\u64ad\u653e|\u6d4f\u89c8|\u89c2\u770b|\u5c55\u73b0|\u70b9\u8d5e|\u8bc4\u8bba|\u6536\u85cf|\u5206\u4eab|\u8f6c\u53d1).*/g, '')
|
|
.trim();
|
|
if (cleaned.length >= 2) {
|
|
return cleaned.slice(0, 120);
|
|
}
|
|
return text.slice(0, 80);
|
|
}
|
|
|
|
function summarizeVideos(videos) {
|
|
const summary = {
|
|
views: 0,
|
|
likes: 0,
|
|
comments: 0,
|
|
collects: 0,
|
|
shares: 0
|
|
};
|
|
for (const video of videos) {
|
|
summary.views += Number(video.views || 0);
|
|
summary.likes += Number(video.likes || 0);
|
|
summary.comments += Number(video.comments || 0);
|
|
summary.collects += Number(video.collects || 0);
|
|
summary.shares += Number(video.shares || 0);
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
function findMetric(text, labels) {
|
|
for (const label of labels) {
|
|
const labelPattern = new RegExp(label);
|
|
const decodedLabel = label.replace(/\\u([0-9a-fA-F]{4})/g, (_, code) => String.fromCharCode(parseInt(code, 16)));
|
|
const after = new RegExp(`${label}[^0-9\\d]{0,16}([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)`);
|
|
const before = new RegExp(`([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)[^0-9\\d]{0,16}${label}`);
|
|
if (labelPattern.test(text)) {
|
|
const afterMatch = text.match(after);
|
|
if (afterMatch) {
|
|
return parseMetricNumber(afterMatch[1], afterMatch[2]);
|
|
}
|
|
const beforeMatch = text.match(before);
|
|
if (beforeMatch) {
|
|
return parseMetricNumber(beforeMatch[1], beforeMatch[2]);
|
|
}
|
|
const looseIndex = text.indexOf(decodedLabel);
|
|
if (looseIndex >= 0) {
|
|
const around = text.slice(Math.max(0, looseIndex - 30), looseIndex + 60);
|
|
const number = around.match(/([0-9][0-9,.]*)(\s*[\u4e07\u5343wWkK]?)/);
|
|
if (number) {
|
|
return parseMetricNumber(number[1], number[2]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
function parseMetricNumber(raw, unit = '') {
|
|
const value = Number(String(raw).replace(/,/g, ''));
|
|
if (!Number.isFinite(value)) {
|
|
return 0;
|
|
}
|
|
const normalizedUnit = unit.trim().toLowerCase();
|
|
if (normalizedUnit.includes('\u4e07') || normalizedUnit.includes('w')) {
|
|
return Math.round(value * 10000);
|
|
}
|
|
if (normalizedUnit.includes('\u5343') || normalizedUnit.includes('k')) {
|
|
return Math.round(value * 1000);
|
|
}
|
|
return Math.round(value);
|
|
}
|
|
|
|
async function pollTasks() {
|
|
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
|
},
|
|
body: JSON.stringify({
|
|
workerId: DEFAULT_WORKER_ID,
|
|
platforms: ['douyin'],
|
|
maxTasks: 5
|
|
})
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`Poll failed: ${response.status} ${await response.text()}`);
|
|
}
|
|
return response.json();
|
|
}
|
|
|
|
async function submitResult(taskId, body) {
|
|
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
|
},
|
|
body: JSON.stringify(body)
|
|
});
|
|
if (!response.ok) {
|
|
throw new Error(`Submit result failed: ${response.status} ${await response.text()}`);
|
|
}
|
|
return response.json();
|
|
}
|
|
|
|
async function handleTask(task) {
|
|
const payload = task.payload || {};
|
|
const action = payload.action || (task.taskType === 'SYNC_METRICS' ? 'sync_metrics' : 'check_login');
|
|
if (action === 'check_login') {
|
|
const status = await checkLogin(task.accountId || DEFAULT_ACCOUNT_ID);
|
|
return submitResult(task.id, {
|
|
status: status.loggedIn ? 'SUCCESS' : 'FAILED',
|
|
errorMessage: status.loggedIn ? null : status.message,
|
|
result: {
|
|
action,
|
|
loggedIn: status.loggedIn,
|
|
message: status.message,
|
|
url: status.url,
|
|
title: status.title,
|
|
profileDir: profileDir(task.accountId || DEFAULT_ACCOUNT_ID)
|
|
}
|
|
});
|
|
}
|
|
|
|
if (action === 'sync_metrics') {
|
|
const result = await collectMetrics(task);
|
|
return submitResult(task.id, result);
|
|
}
|
|
|
|
return submitResult(task.id, {
|
|
status: 'FAILED',
|
|
errorMessage: `Unsupported local Douyin worker action: ${action}`,
|
|
result: { action }
|
|
});
|
|
}
|
|
|
|
async function workerOnce() {
|
|
const tasks = await pollTasks();
|
|
if (!Array.isArray(tasks) || tasks.length === 0) {
|
|
console.log('No pending Douyin tasks.');
|
|
return;
|
|
}
|
|
for (const task of tasks) {
|
|
const result = await handleTask(task);
|
|
console.log(`Completed ${task.id}: ${result.status}`);
|
|
}
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
|
|
async function workerLoop() {
|
|
const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000);
|
|
console.log(`Douyin worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`);
|
|
while (true) {
|
|
try {
|
|
await workerOnce();
|
|
} catch (error) {
|
|
console.error(error.stack || error.message);
|
|
}
|
|
await sleep(intervalMs);
|
|
}
|
|
}
|
|
|
|
async function main() {
|
|
const mode = process.argv[2];
|
|
if (mode === 'login') {
|
|
await login();
|
|
return;
|
|
}
|
|
if (mode === 'check-login') {
|
|
const status = await checkLogin(process.argv[3] || DEFAULT_ACCOUNT_ID);
|
|
console.log(JSON.stringify(status, null, 2));
|
|
process.exit(status.loggedIn ? 0 : 2);
|
|
}
|
|
if (mode === 'metrics') {
|
|
const task = {
|
|
id: `manual_${Date.now()}`,
|
|
taskType: 'SYNC_METRICS',
|
|
platform: 'douyin',
|
|
accountId: process.argv[3] || DEFAULT_ACCOUNT_ID,
|
|
payload: { action: 'sync_metrics', headless: false }
|
|
};
|
|
console.log(JSON.stringify((await collectMetrics(task)).result, null, 2));
|
|
return;
|
|
}
|
|
if (mode === 'worker') {
|
|
await workerOnce();
|
|
return;
|
|
}
|
|
if (mode === 'worker-loop') {
|
|
await workerLoop();
|
|
return;
|
|
}
|
|
throw new Error('Usage: node scripts/local_douyin_worker.js <login|check-login|metrics|worker|worker-loop> [accountId]');
|
|
}
|
|
|
|
main().catch(error => {
|
|
console.error(error.stack || error.message);
|
|
process.exit(1);
|
|
});
|