xiaohongshu
This commit is contained in:
parent
b587576e85
commit
7443e12f66
@ -7,7 +7,7 @@ Before first use, install:
|
||||
How to start:
|
||||
1. Unzip this package to a normal folder, for example Desktop\agent-foundation-client.
|
||||
2. Double-click start_yuedong_client.bat.
|
||||
3. If a Douyin login page opens, login in the browser, then press Enter in the terminal window.
|
||||
3. If a Douyin or Xiaohongshu login page opens, login in the browser, then press Enter in the terminal window.
|
||||
4. Keep the worker terminal window open while using the system.
|
||||
|
||||
Server API:
|
||||
|
||||
BIN
dist/agent-foundation-client-yuedong.zip
vendored
BIN
dist/agent-foundation-client-yuedong.zip
vendored
Binary file not shown.
@ -7,7 +7,7 @@ Before first use, install:
|
||||
How to start:
|
||||
1. Unzip this package to a normal folder, for example Desktop\agent-foundation-client.
|
||||
2. Double-click start_yuedong_client.bat.
|
||||
3. If a Douyin login page opens, login in the browser, then press Enter in the terminal window.
|
||||
3. If a Douyin or Xiaohongshu login page opens, login in the browser, then press Enter in the terminal window.
|
||||
4. Keep the worker terminal window open while using the system.
|
||||
|
||||
Server API:
|
||||
|
||||
@ -7,7 +7,14 @@
|
||||
"douyin:worker": "node scripts/local_douyin_worker.js worker",
|
||||
"douyin:worker:loop": "node scripts/local_douyin_worker.js worker-loop",
|
||||
"douyin:check": "node scripts/local_douyin_worker.js check-login",
|
||||
"douyin:metrics": "node scripts/local_douyin_worker.js metrics"
|
||||
"douyin:metrics": "node scripts/local_douyin_worker.js metrics",
|
||||
"xhs:login": "node scripts/local_xiaohongshu_worker.js login",
|
||||
"xhs:worker": "node scripts/local_xiaohongshu_worker.js worker",
|
||||
"xhs:worker:loop": "node scripts/local_xiaohongshu_worker.js worker-loop",
|
||||
"xhs:check": "node scripts/local_xiaohongshu_worker.js check-login",
|
||||
"xhs:metrics": "node scripts/local_xiaohongshu_worker.js metrics",
|
||||
"media:worker": "node scripts/local_media_matrix_worker.js worker",
|
||||
"media:worker:loop": "node scripts/local_media_matrix_worker.js worker-loop"
|
||||
},
|
||||
"dependencies": {
|
||||
"playwright": "^1.61.0"
|
||||
|
||||
@ -137,8 +137,9 @@ async function collectMetrics(task) {
|
||||
status: 'FAILED',
|
||||
errorMessage: status.message,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
loggedIn: false,
|
||||
action: 'sync_metrics',
|
||||
platform: 'douyin',
|
||||
loggedIn: false,
|
||||
...status
|
||||
}
|
||||
};
|
||||
@ -153,6 +154,7 @@ async function collectMetrics(task) {
|
||||
errorMessage: null,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'douyin',
|
||||
loggedIn: true,
|
||||
url: page.url(),
|
||||
title: await page.title().catch(() => ''),
|
||||
@ -418,6 +420,7 @@ async function handleTask(task) {
|
||||
errorMessage: status.loggedIn ? null : status.message,
|
||||
result: {
|
||||
action,
|
||||
platform: 'douyin',
|
||||
loggedIn: status.loggedIn,
|
||||
message: status.message,
|
||||
url: status.url,
|
||||
@ -501,7 +504,18 @@ async function main() {
|
||||
throw new Error('Usage: node scripts/local_douyin_worker.js <login|check-login|metrics|worker|worker-loop> [accountId]');
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
module.exports = {
|
||||
checkLogin,
|
||||
collectMetrics,
|
||||
login,
|
||||
profileDir,
|
||||
workerLoop,
|
||||
workerOnce
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
217
dist/agent-foundation-client-yuedong/scripts/local_media_matrix_worker.js
vendored
Normal file
217
dist/agent-foundation-client-yuedong/scripts/local_media_matrix_worker.js
vendored
Normal file
@ -0,0 +1,217 @@
|
||||
const douyin = require('./local_douyin_worker');
|
||||
const xiaohongshu = require('./local_xiaohongshu_worker');
|
||||
|
||||
const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880';
|
||||
const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-media-matrix-worker';
|
||||
const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token';
|
||||
const DEFAULT_DOUYIN_ACCOUNT_ID = process.env.DOUYIN_ACCOUNT_ID || 'douyin_company';
|
||||
const DEFAULT_XHS_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company';
|
||||
|
||||
const PLATFORM_HANDLERS = {
|
||||
douyin: {
|
||||
accountId: DEFAULT_DOUYIN_ACCOUNT_ID,
|
||||
collectMetrics: douyin.collectMetrics,
|
||||
checkLogin: douyin.checkLogin
|
||||
},
|
||||
xiaohongshu: {
|
||||
accountId: DEFAULT_XHS_ACCOUNT_ID,
|
||||
collectMetrics: xiaohongshu.collectMetrics,
|
||||
checkLogin: xiaohongshu.checkLogin
|
||||
}
|
||||
};
|
||||
|
||||
function normalizePlatform(platform) {
|
||||
const value = String(platform || '').trim().toLowerCase();
|
||||
if (value === 'xhs' || value === 'rednote') {
|
||||
return 'xiaohongshu';
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requestedPlatforms(task) {
|
||||
const payload = task.payload || {};
|
||||
const raw = Array.isArray(payload.platforms) && payload.platforms.length > 0
|
||||
? payload.platforms
|
||||
: ['douyin', 'xiaohongshu'];
|
||||
return raw.map(normalizePlatform).filter(platform => PLATFORM_HANDLERS[platform]);
|
||||
}
|
||||
|
||||
function accountIdFor(task, platform) {
|
||||
const payload = task.payload || {};
|
||||
const accountIds = payload.accountIds || {};
|
||||
if (accountIds && typeof accountIds === 'object' && accountIds[platform]) {
|
||||
return accountIds[platform];
|
||||
}
|
||||
return PLATFORM_HANDLERS[platform].accountId;
|
||||
}
|
||||
|
||||
function payloadForPlatform(task, platform) {
|
||||
const payload = task.payload || {};
|
||||
const platformPayloads = payload.platformPayloads || {};
|
||||
return {
|
||||
...payload,
|
||||
...(platformPayloads[platform] || {}),
|
||||
platforms: undefined,
|
||||
accountIds: undefined,
|
||||
platformPayloads: undefined
|
||||
};
|
||||
}
|
||||
|
||||
async function pollTasks() {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workerId: DEFAULT_WORKER_ID,
|
||||
platforms: ['multi', 'matrix', 'all'],
|
||||
maxTasks: 3
|
||||
})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Poll failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function submitResult(taskId, body) {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Submit result failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function handleTask(task) {
|
||||
const platforms = requestedPlatforms(task);
|
||||
if (platforms.length === 0) {
|
||||
return submitResult(task.id, {
|
||||
status: 'FAILED',
|
||||
errorMessage: 'No supported platforms requested.',
|
||||
result: { action: 'sync_metrics', platform: 'multi', platformResults: {} }
|
||||
});
|
||||
}
|
||||
|
||||
const platformResults = {};
|
||||
const videos = [];
|
||||
const metrics = emptyMetrics();
|
||||
const errors = [];
|
||||
|
||||
for (const platform of platforms) {
|
||||
const handler = PLATFORM_HANDLERS[platform];
|
||||
const platformTask = {
|
||||
...task,
|
||||
platform,
|
||||
accountId: accountIdFor(task, platform),
|
||||
payload: payloadForPlatform(task, platform)
|
||||
};
|
||||
try {
|
||||
const response = await handler.collectMetrics(platformTask);
|
||||
const result = response.result || {};
|
||||
platformResults[platform] = {
|
||||
...result,
|
||||
platform,
|
||||
accountId: platformTask.accountId,
|
||||
status: response.status
|
||||
};
|
||||
addMetrics(metrics, result.metrics || {});
|
||||
for (const video of result.videos || []) {
|
||||
videos.push({ ...video, platform, accountId: platformTask.accountId });
|
||||
}
|
||||
if (response.status !== 'SUCCESS') {
|
||||
errors.push(`${platform}: ${response.errorMessage || 'failed'}`);
|
||||
}
|
||||
} catch (error) {
|
||||
platformResults[platform] = {
|
||||
platform,
|
||||
accountId: platformTask.accountId,
|
||||
status: 'FAILED',
|
||||
errorMessage: error.message
|
||||
};
|
||||
errors.push(`${platform}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return submitResult(task.id, {
|
||||
status: errors.length === platforms.length ? 'FAILED' : 'SUCCESS',
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : null,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'multi',
|
||||
platforms,
|
||||
metrics,
|
||||
videos,
|
||||
videoCount: videos.length,
|
||||
platformResults,
|
||||
extractionMode: 'multi_platform'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function emptyMetrics() {
|
||||
return { views: 0, likes: 0, comments: 0, collects: 0, shares: 0 };
|
||||
}
|
||||
|
||||
function addMetrics(target, source) {
|
||||
target.views += Number(source.views || 0);
|
||||
target.likes += Number(source.likes || 0);
|
||||
target.comments += Number(source.comments || 0);
|
||||
target.collects += Number(source.collects || 0);
|
||||
target.shares += Number(source.shares || 0);
|
||||
}
|
||||
|
||||
async function workerOnce() {
|
||||
const tasks = await pollTasks();
|
||||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||||
console.log('No pending media matrix tasks.');
|
||||
return;
|
||||
}
|
||||
for (const task of tasks) {
|
||||
const result = await handleTask(task);
|
||||
console.log(`Completed ${task.id}: ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function workerLoop() {
|
||||
const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000);
|
||||
console.log(`Media matrix worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`);
|
||||
while (true) {
|
||||
try {
|
||||
await workerOnce();
|
||||
} catch (error) {
|
||||
console.error(error.stack || error.message);
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mode = process.argv[2];
|
||||
if (mode === 'worker') {
|
||||
await workerOnce();
|
||||
return;
|
||||
}
|
||||
if (mode === 'worker-loop') {
|
||||
await workerLoop();
|
||||
return;
|
||||
}
|
||||
throw new Error('Usage: node scripts/local_media_matrix_worker.js <worker|worker-loop>');
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
517
dist/agent-foundation-client-yuedong/scripts/local_xiaohongshu_worker.js
vendored
Normal file
517
dist/agent-foundation-client-yuedong/scripts/local_xiaohongshu_worker.js
vendored
Normal file
@ -0,0 +1,517 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
|
||||
const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880';
|
||||
const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-xiaohongshu-worker';
|
||||
const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token';
|
||||
const DEFAULT_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company';
|
||||
const DEFAULT_CREATOR_URL = process.env.XHS_CREATOR_URL || 'https://creator.xiaohongshu.com';
|
||||
const DEFAULT_CONTENT_URL = process.env.XHS_CONTENT_URL || 'https://creator.xiaohongshu.com';
|
||||
const DEFAULT_PROFILE_ROOT = process.env.XHS_PROFILE_ROOT || path.resolve('data/platform-profiles');
|
||||
|
||||
const LOGIN_TEXT = new RegExp([
|
||||
'\\u626b\\u7801\\u767b\\u5f55',
|
||||
'\\u9a8c\\u8bc1\\u7801',
|
||||
'\\u624b\\u673a\\u53f7',
|
||||
'\\u767b\\u5f55',
|
||||
'\\u6ce8\\u518c'
|
||||
].join('|'));
|
||||
|
||||
const CREATOR_TEXT = new RegExp([
|
||||
'\\u5c0f\\u7ea2\\u4e66',
|
||||
'\\u521b\\u4f5c\\u670d\\u52a1',
|
||||
'\\u7b14\\u8bb0',
|
||||
'\\u6570\\u636e',
|
||||
'\\u4f5c\\u54c1',
|
||||
'\\u53d1\\u5e03'
|
||||
].join('|'));
|
||||
|
||||
function requirePlaywright() {
|
||||
try {
|
||||
return require('playwright');
|
||||
} catch (error) {
|
||||
throw new Error('Playwright is not installed. Run npm.cmd install in the project root first.');
|
||||
}
|
||||
}
|
||||
|
||||
function profileDir(accountId = DEFAULT_ACCOUNT_ID) {
|
||||
return path.resolve(DEFAULT_PROFILE_ROOT, accountId);
|
||||
}
|
||||
|
||||
function ensureDir(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function browserChannel() {
|
||||
if (process.env.XHS_BROWSER_CHANNEL) {
|
||||
return process.env.XHS_BROWSER_CHANNEL;
|
||||
}
|
||||
return process.platform === 'win32' ? 'msedge' : undefined;
|
||||
}
|
||||
|
||||
async function launchContext({ accountId = DEFAULT_ACCOUNT_ID, headless = false } = {}) {
|
||||
const { chromium } = requirePlaywright();
|
||||
const dir = profileDir(accountId);
|
||||
ensureDir(dir);
|
||||
const channel = browserChannel();
|
||||
const options = {
|
||||
headless,
|
||||
viewport: { width: 1440, height: 1000 },
|
||||
acceptDownloads: true,
|
||||
args: [
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-dev-shm-usage'
|
||||
]
|
||||
};
|
||||
if (channel) {
|
||||
options.channel = channel;
|
||||
}
|
||||
return chromium.launchPersistentContext(dir, options);
|
||||
}
|
||||
|
||||
async function waitForEnter(message) {
|
||||
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
||||
return new Promise(resolve => {
|
||||
rl.question(message, () => {
|
||||
rl.close();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function login() {
|
||||
const accountId = process.argv[3] || DEFAULT_ACCOUNT_ID;
|
||||
const context = await launchContext({ accountId, headless: false });
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
console.log(`Xiaohongshu login profile: ${profileDir(accountId)}`);
|
||||
console.log('Please login in the opened browser. Keep this window open until login succeeds.');
|
||||
await waitForEnter('Press Enter here after Xiaohongshu creator center is logged in...');
|
||||
const status = await detectLogin(page, accountId);
|
||||
await context.close();
|
||||
console.log(JSON.stringify(status, null, 2));
|
||||
}
|
||||
|
||||
async function checkLogin(accountId = DEFAULT_ACCOUNT_ID) {
|
||||
const context = await launchContext({ accountId, headless: true });
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
try {
|
||||
await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForTimeout(2500);
|
||||
return await detectLogin(page, accountId);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function detectLogin(page, accountId = DEFAULT_ACCOUNT_ID) {
|
||||
const url = page.url();
|
||||
const title = await page.title().catch(() => '');
|
||||
const bodyText = await page.locator('body').innerText({ timeout: 5000 }).catch(() => '');
|
||||
const loginVisible = LOGIN_TEXT.test(bodyText);
|
||||
const creatorVisible = CREATOR_TEXT.test(bodyText);
|
||||
const loggedIn = creatorVisible || (!loginVisible && /xiaohongshu\.com/.test(url));
|
||||
return {
|
||||
loggedIn,
|
||||
status: loggedIn ? 'SUCCESS' : 'FAILED',
|
||||
message: loggedIn ? 'Xiaohongshu creator profile is logged in.' : 'Xiaohongshu creator profile is not logged in.',
|
||||
url,
|
||||
title,
|
||||
profileDir: profileDir(accountId)
|
||||
};
|
||||
}
|
||||
|
||||
async function collectMetrics(task) {
|
||||
const payload = task.payload || {};
|
||||
const accountId = task.accountId || DEFAULT_ACCOUNT_ID;
|
||||
const targetUrl = payload.metricsUrl || payload.contentUrl || DEFAULT_CONTENT_URL;
|
||||
const headless = payload.headless === false ? false : true;
|
||||
const context = await launchContext({ accountId, headless });
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
try {
|
||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForTimeout(Number(payload.waitMs || 6000));
|
||||
const status = await detectLogin(page, accountId);
|
||||
if (!status.loggedIn) {
|
||||
return {
|
||||
status: 'FAILED',
|
||||
errorMessage: status.message,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'xiaohongshu',
|
||||
loggedIn: false,
|
||||
...status
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const text = await page.locator('body').innerText({ timeout: 15000 }).catch(() => '');
|
||||
const screenshot = await saveScreenshot(page, task.id);
|
||||
const videos = await collectVideoMetrics(page, Number(payload.maxVideos || 20));
|
||||
const metrics = videos.length > 0 ? summarizeVideos(videos) : extractMetrics(text);
|
||||
return {
|
||||
status: 'SUCCESS',
|
||||
errorMessage: null,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'xiaohongshu',
|
||||
loggedIn: true,
|
||||
url: page.url(),
|
||||
title: await page.title().catch(() => ''),
|
||||
metrics,
|
||||
videos,
|
||||
videoCount: videos.length,
|
||||
extractionMode: videos.length > 0 ? 'video_list' : 'page_summary',
|
||||
screenshot,
|
||||
pageTextPreview: compactText(text).slice(0, 2000)
|
||||
}
|
||||
};
|
||||
} finally {
|
||||
await context.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function saveScreenshot(page, taskId) {
|
||||
const dir = path.resolve('target/media/xiaohongshu');
|
||||
ensureDir(dir);
|
||||
const file = path.join(dir, `${taskId || Date.now()}-metrics.png`);
|
||||
await page.screenshot({ path: file, fullPage: true }).catch(() => {});
|
||||
return file;
|
||||
}
|
||||
|
||||
function compactText(text) {
|
||||
return (text || '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function extractMetrics(text) {
|
||||
const compact = compactText(text);
|
||||
return {
|
||||
views: findMetric(compact, ['\\u6d4f\\u89c8', '\\u89c2\\u770b', '\\u9605\\u8bfb', '\\u66dd\\u5149']),
|
||||
likes: findMetric(compact, ['\\u70b9\\u8d5e', '\\u8d5e']),
|
||||
comments: findMetric(compact, ['\\u8bc4\\u8bba']),
|
||||
collects: findMetric(compact, ['\\u6536\\u85cf']),
|
||||
shares: findMetric(compact, ['\\u5206\\u4eab', '\\u8f6c\\u53d1'])
|
||||
};
|
||||
}
|
||||
|
||||
async function collectVideoMetrics(page, limit = 20) {
|
||||
await autoScroll(page);
|
||||
const blocks = await page.evaluate(maxBlocks => {
|
||||
const selectors = [
|
||||
'tr',
|
||||
'article',
|
||||
'li',
|
||||
'[class*="note"]',
|
||||
'[class*="card"]',
|
||||
'[class*="item"]',
|
||||
'[class*="work"]',
|
||||
'[class*="video"]',
|
||||
'[class*="content"]'
|
||||
];
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const selector of selectors) {
|
||||
for (const element of document.querySelectorAll(selector)) {
|
||||
if (rows.length >= maxBlocks) {
|
||||
break;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (rect.width < 140 || rect.height < 40) {
|
||||
continue;
|
||||
}
|
||||
const text = (element.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
if (text.length < 10 || text.length > 1200) {
|
||||
continue;
|
||||
}
|
||||
const signature = text.slice(0, 120);
|
||||
if (seen.has(signature)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(signature);
|
||||
const link = element.querySelector('a[href]');
|
||||
rows.push({
|
||||
text,
|
||||
href: link ? link.href : ''
|
||||
});
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}, Math.max(limit * 8, 80));
|
||||
|
||||
const videos = [];
|
||||
const seen = new Set();
|
||||
for (const block of blocks) {
|
||||
const video = parseVideoBlock(block);
|
||||
if (!video) {
|
||||
continue;
|
||||
}
|
||||
const key = `${video.title}|${video.views}|${video.likes}|${video.comments}|${video.href}`;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
videos.push(video);
|
||||
if (videos.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return videos;
|
||||
}
|
||||
|
||||
async function autoScroll(page) {
|
||||
await page.evaluate(async () => {
|
||||
await new Promise(resolve => {
|
||||
let count = 0;
|
||||
const timer = setInterval(() => {
|
||||
window.scrollBy(0, Math.floor(window.innerHeight * 0.8));
|
||||
count += 1;
|
||||
if (count >= 3) {
|
||||
clearInterval(timer);
|
||||
resolve();
|
||||
}
|
||||
}, 500);
|
||||
});
|
||||
}).catch(() => {});
|
||||
await page.waitForTimeout(800).catch(() => {});
|
||||
}
|
||||
|
||||
function parseVideoBlock(block) {
|
||||
const text = compactText(block.text);
|
||||
const metrics = extractMetrics(text);
|
||||
const score = metrics.views + metrics.likes + metrics.comments + metrics.collects + metrics.shares;
|
||||
if (score <= 0) {
|
||||
return null;
|
||||
}
|
||||
const title = extractTitle(text);
|
||||
return {
|
||||
title,
|
||||
href: block.href || '',
|
||||
views: metrics.views,
|
||||
likes: metrics.likes,
|
||||
comments: metrics.comments,
|
||||
collects: metrics.collects,
|
||||
shares: metrics.shares,
|
||||
interactionCount: metrics.likes + metrics.comments + metrics.collects + metrics.shares,
|
||||
interactionRate: metrics.views <= 0
|
||||
? 0
|
||||
: (metrics.likes + metrics.comments + metrics.collects + metrics.shares) / metrics.views,
|
||||
rawTextPreview: text.slice(0, 500)
|
||||
};
|
||||
}
|
||||
|
||||
function extractTitle(text) {
|
||||
const cleaned = text
|
||||
.replace(/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}.*/g, '')
|
||||
.replace(/(\u6d4f\u89c8|\u89c2\u770b|\u9605\u8bfb|\u66dd\u5149|\u70b9\u8d5e|\u8d5e|\u8bc4\u8bba|\u6536\u85cf|\u5206\u4eab|\u8f6c\u53d1).*/g, '')
|
||||
.trim();
|
||||
if (cleaned.length >= 2) {
|
||||
return cleaned.slice(0, 120);
|
||||
}
|
||||
return text.slice(0, 80);
|
||||
}
|
||||
|
||||
function summarizeVideos(videos) {
|
||||
const summary = {
|
||||
views: 0,
|
||||
likes: 0,
|
||||
comments: 0,
|
||||
collects: 0,
|
||||
shares: 0
|
||||
};
|
||||
for (const video of videos) {
|
||||
summary.views += Number(video.views || 0);
|
||||
summary.likes += Number(video.likes || 0);
|
||||
summary.comments += Number(video.comments || 0);
|
||||
summary.collects += Number(video.collects || 0);
|
||||
summary.shares += Number(video.shares || 0);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
function findMetric(text, labels) {
|
||||
for (const label of labels) {
|
||||
const labelPattern = new RegExp(label);
|
||||
const decodedLabel = label.replace(/\\u([0-9a-fA-F]{4})/g, (_, code) => String.fromCharCode(parseInt(code, 16)));
|
||||
const after = new RegExp(`${label}[^0-9\\d]{0,16}([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)`);
|
||||
const before = new RegExp(`([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)[^0-9\\d]{0,16}${label}`);
|
||||
if (labelPattern.test(text)) {
|
||||
const afterMatch = text.match(after);
|
||||
if (afterMatch) {
|
||||
return parseMetricNumber(afterMatch[1], afterMatch[2]);
|
||||
}
|
||||
const beforeMatch = text.match(before);
|
||||
if (beforeMatch) {
|
||||
return parseMetricNumber(beforeMatch[1], beforeMatch[2]);
|
||||
}
|
||||
const looseIndex = text.indexOf(decodedLabel);
|
||||
if (looseIndex >= 0) {
|
||||
const around = text.slice(Math.max(0, looseIndex - 30), looseIndex + 60);
|
||||
const number = around.match(/([0-9][0-9,.]*)(\s*[\u4e07\u5343wWkK]?)/);
|
||||
if (number) {
|
||||
return parseMetricNumber(number[1], number[2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function parseMetricNumber(raw, unit = '') {
|
||||
const value = Number(String(raw).replace(/,/g, ''));
|
||||
if (!Number.isFinite(value)) {
|
||||
return 0;
|
||||
}
|
||||
const normalizedUnit = unit.trim().toLowerCase();
|
||||
if (normalizedUnit.includes('\u4e07') || normalizedUnit.includes('w')) {
|
||||
return Math.round(value * 10000);
|
||||
}
|
||||
if (normalizedUnit.includes('\u5343') || normalizedUnit.includes('k')) {
|
||||
return Math.round(value * 1000);
|
||||
}
|
||||
return Math.round(value);
|
||||
}
|
||||
|
||||
async function pollTasks() {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workerId: DEFAULT_WORKER_ID,
|
||||
platforms: ['xiaohongshu'],
|
||||
maxTasks: 5
|
||||
})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Poll failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function submitResult(taskId, body) {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Submit result failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function handleTask(task) {
|
||||
const payload = task.payload || {};
|
||||
const action = payload.action || (task.taskType === 'SYNC_METRICS' ? 'sync_metrics' : 'check_login');
|
||||
if (action === 'check_login') {
|
||||
const status = await checkLogin(task.accountId || DEFAULT_ACCOUNT_ID);
|
||||
return submitResult(task.id, {
|
||||
status: status.loggedIn ? 'SUCCESS' : 'FAILED',
|
||||
errorMessage: status.loggedIn ? null : status.message,
|
||||
result: {
|
||||
action,
|
||||
platform: 'xiaohongshu',
|
||||
loggedIn: status.loggedIn,
|
||||
message: status.message,
|
||||
url: status.url,
|
||||
title: status.title,
|
||||
profileDir: profileDir(task.accountId || DEFAULT_ACCOUNT_ID)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (action === 'sync_metrics') {
|
||||
const result = await collectMetrics(task);
|
||||
return submitResult(task.id, result);
|
||||
}
|
||||
|
||||
return submitResult(task.id, {
|
||||
status: 'FAILED',
|
||||
errorMessage: `Unsupported local Xiaohongshu worker action: ${action}`,
|
||||
result: { action, platform: 'xiaohongshu' }
|
||||
});
|
||||
}
|
||||
|
||||
async function workerOnce() {
|
||||
const tasks = await pollTasks();
|
||||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||||
console.log('No pending Xiaohongshu tasks.');
|
||||
return;
|
||||
}
|
||||
for (const task of tasks) {
|
||||
const result = await handleTask(task);
|
||||
console.log(`Completed ${task.id}: ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function workerLoop() {
|
||||
const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000);
|
||||
console.log(`Xiaohongshu worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`);
|
||||
while (true) {
|
||||
try {
|
||||
await workerOnce();
|
||||
} catch (error) {
|
||||
console.error(error.stack || error.message);
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mode = process.argv[2];
|
||||
if (mode === 'login') {
|
||||
await login();
|
||||
return;
|
||||
}
|
||||
if (mode === 'check-login') {
|
||||
const status = await checkLogin(process.argv[3] || DEFAULT_ACCOUNT_ID);
|
||||
console.log(JSON.stringify(status, null, 2));
|
||||
process.exit(status.loggedIn ? 0 : 2);
|
||||
}
|
||||
if (mode === 'metrics') {
|
||||
const task = {
|
||||
id: `manual_${Date.now()}`,
|
||||
taskType: 'SYNC_METRICS',
|
||||
platform: 'xiaohongshu',
|
||||
accountId: process.argv[3] || DEFAULT_ACCOUNT_ID,
|
||||
payload: { action: 'sync_metrics', headless: false }
|
||||
};
|
||||
console.log(JSON.stringify((await collectMetrics(task)).result, null, 2));
|
||||
return;
|
||||
}
|
||||
if (mode === 'worker') {
|
||||
await workerOnce();
|
||||
return;
|
||||
}
|
||||
if (mode === 'worker-loop') {
|
||||
await workerLoop();
|
||||
return;
|
||||
}
|
||||
throw new Error('Usage: node scripts/local_xiaohongshu_worker.js <login|check-login|metrics|worker|worker-loop> [accountId]');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
checkLogin,
|
||||
collectMetrics,
|
||||
login,
|
||||
profileDir,
|
||||
workerLoop,
|
||||
workerOnce
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@ -1,8 +1,10 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://localhost:8880",
|
||||
[string]$AccountId = "douyin_company",
|
||||
[string]$XhsAccountId = "xiaohongshu_company",
|
||||
[int]$WorkerIntervalSeconds = 30,
|
||||
[switch]$RemoteBackend
|
||||
[switch]$RemoteBackend,
|
||||
[string]$Platforms = "douyin,xiaohongshu"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@ -115,25 +117,62 @@ if ($RemoteBackend -or -not (Test-LocalBaseUrl $BaseUrl)) {
|
||||
Write-Step "Checking Douyin login"
|
||||
$env:MEDIA_BASE_URL = $BaseUrl
|
||||
$env:DOUYIN_ACCOUNT_ID = $AccountId
|
||||
$checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host $checkOutput
|
||||
Write-Host ""
|
||||
Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window."
|
||||
& npm.cmd run douyin:login -- $AccountId
|
||||
$platformList = $Platforms.Split(",") | ForEach-Object { $_.Trim().ToLower() } | Where-Object { $_ }
|
||||
|
||||
if ($platformList -contains "douyin") {
|
||||
$checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Douyin login failed."
|
||||
Write-Host $checkOutput
|
||||
Write-Host ""
|
||||
Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window."
|
||||
& npm.cmd run douyin:login -- $AccountId
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Douyin login failed."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Douyin login profile is ready."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Douyin login profile is ready."
|
||||
}
|
||||
|
||||
Write-Step "Starting Douyin worker loop"
|
||||
$workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal
|
||||
if ($platformList -contains "xiaohongshu") {
|
||||
Write-Step "Checking Xiaohongshu login"
|
||||
$env:XHS_ACCOUNT_ID = $XhsAccountId
|
||||
$xhsCheckOutput = & npm.cmd run xhs:check -- $XhsAccountId 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host $xhsCheckOutput
|
||||
Write-Host ""
|
||||
Write-Host "Xiaohongshu login is required. A browser will open. Login, then press Enter in that window."
|
||||
& npm.cmd run xhs:login -- $XhsAccountId
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Xiaohongshu login failed."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Xiaohongshu login profile is ready."
|
||||
}
|
||||
}
|
||||
|
||||
if ($platformList -contains "douyin") {
|
||||
Write-Step "Starting Douyin worker loop"
|
||||
$workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal
|
||||
}
|
||||
|
||||
if ($platformList -contains "xiaohongshu") {
|
||||
Write-Step "Starting Xiaohongshu worker loop"
|
||||
$xhsWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run xhs:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $xhsWorkerCommand) -WindowStyle Normal
|
||||
}
|
||||
|
||||
if (($platformList -contains "douyin") -and ($platformList -contains "xiaohongshu")) {
|
||||
Write-Step "Starting media matrix worker loop"
|
||||
$matrixWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run media:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $matrixWorkerCommand) -WindowStyle Normal
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Local client is ready."
|
||||
Write-Host "Backend: $BaseUrl"
|
||||
Write-Host "Account: $AccountId"
|
||||
Write-Host "Douyin account: $AccountId"
|
||||
Write-Host "Xiaohongshu account: $XhsAccountId"
|
||||
Write-Host "Platforms: $($platformList -join ', ')"
|
||||
Write-Host "Worker polling interval: $WorkerIntervalSeconds seconds"
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0"
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend -Platforms "douyin,xiaohongshu"
|
||||
pause
|
||||
|
||||
@ -7,7 +7,14 @@
|
||||
"douyin:worker": "node scripts/local_douyin_worker.js worker",
|
||||
"douyin:worker:loop": "node scripts/local_douyin_worker.js worker-loop",
|
||||
"douyin:check": "node scripts/local_douyin_worker.js check-login",
|
||||
"douyin:metrics": "node scripts/local_douyin_worker.js metrics"
|
||||
"douyin:metrics": "node scripts/local_douyin_worker.js metrics",
|
||||
"xhs:login": "node scripts/local_xiaohongshu_worker.js login",
|
||||
"xhs:worker": "node scripts/local_xiaohongshu_worker.js worker",
|
||||
"xhs:worker:loop": "node scripts/local_xiaohongshu_worker.js worker-loop",
|
||||
"xhs:check": "node scripts/local_xiaohongshu_worker.js check-login",
|
||||
"xhs:metrics": "node scripts/local_xiaohongshu_worker.js metrics",
|
||||
"media:worker": "node scripts/local_media_matrix_worker.js worker",
|
||||
"media:worker:loop": "node scripts/local_media_matrix_worker.js worker-loop"
|
||||
},
|
||||
"dependencies": {
|
||||
"playwright": "^1.61.0"
|
||||
|
||||
@ -137,8 +137,9 @@ async function collectMetrics(task) {
|
||||
status: 'FAILED',
|
||||
errorMessage: status.message,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
loggedIn: false,
|
||||
action: 'sync_metrics',
|
||||
platform: 'douyin',
|
||||
loggedIn: false,
|
||||
...status
|
||||
}
|
||||
};
|
||||
@ -153,6 +154,7 @@ async function collectMetrics(task) {
|
||||
errorMessage: null,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'douyin',
|
||||
loggedIn: true,
|
||||
url: page.url(),
|
||||
title: await page.title().catch(() => ''),
|
||||
@ -418,6 +420,7 @@ async function handleTask(task) {
|
||||
errorMessage: status.loggedIn ? null : status.message,
|
||||
result: {
|
||||
action,
|
||||
platform: 'douyin',
|
||||
loggedIn: status.loggedIn,
|
||||
message: status.message,
|
||||
url: status.url,
|
||||
@ -501,7 +504,18 @@ async function main() {
|
||||
throw new Error('Usage: node scripts/local_douyin_worker.js <login|check-login|metrics|worker|worker-loop> [accountId]');
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
module.exports = {
|
||||
checkLogin,
|
||||
collectMetrics,
|
||||
login,
|
||||
profileDir,
|
||||
workerLoop,
|
||||
workerOnce
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
217
scripts/local_media_matrix_worker.js
Normal file
217
scripts/local_media_matrix_worker.js
Normal file
@ -0,0 +1,217 @@
|
||||
const douyin = require('./local_douyin_worker');
|
||||
const xiaohongshu = require('./local_xiaohongshu_worker');
|
||||
|
||||
const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880';
|
||||
const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-media-matrix-worker';
|
||||
const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token';
|
||||
const DEFAULT_DOUYIN_ACCOUNT_ID = process.env.DOUYIN_ACCOUNT_ID || 'douyin_company';
|
||||
const DEFAULT_XHS_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company';
|
||||
|
||||
const PLATFORM_HANDLERS = {
|
||||
douyin: {
|
||||
accountId: DEFAULT_DOUYIN_ACCOUNT_ID,
|
||||
collectMetrics: douyin.collectMetrics,
|
||||
checkLogin: douyin.checkLogin
|
||||
},
|
||||
xiaohongshu: {
|
||||
accountId: DEFAULT_XHS_ACCOUNT_ID,
|
||||
collectMetrics: xiaohongshu.collectMetrics,
|
||||
checkLogin: xiaohongshu.checkLogin
|
||||
}
|
||||
};
|
||||
|
||||
function normalizePlatform(platform) {
|
||||
const value = String(platform || '').trim().toLowerCase();
|
||||
if (value === 'xhs' || value === 'rednote') {
|
||||
return 'xiaohongshu';
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requestedPlatforms(task) {
|
||||
const payload = task.payload || {};
|
||||
const raw = Array.isArray(payload.platforms) && payload.platforms.length > 0
|
||||
? payload.platforms
|
||||
: ['douyin', 'xiaohongshu'];
|
||||
return raw.map(normalizePlatform).filter(platform => PLATFORM_HANDLERS[platform]);
|
||||
}
|
||||
|
||||
function accountIdFor(task, platform) {
|
||||
const payload = task.payload || {};
|
||||
const accountIds = payload.accountIds || {};
|
||||
if (accountIds && typeof accountIds === 'object' && accountIds[platform]) {
|
||||
return accountIds[platform];
|
||||
}
|
||||
return PLATFORM_HANDLERS[platform].accountId;
|
||||
}
|
||||
|
||||
function payloadForPlatform(task, platform) {
|
||||
const payload = task.payload || {};
|
||||
const platformPayloads = payload.platformPayloads || {};
|
||||
return {
|
||||
...payload,
|
||||
...(platformPayloads[platform] || {}),
|
||||
platforms: undefined,
|
||||
accountIds: undefined,
|
||||
platformPayloads: undefined
|
||||
};
|
||||
}
|
||||
|
||||
async function pollTasks() {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workerId: DEFAULT_WORKER_ID,
|
||||
platforms: ['multi', 'matrix', 'all'],
|
||||
maxTasks: 3
|
||||
})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Poll failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function submitResult(taskId, body) {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Submit result failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function handleTask(task) {
|
||||
const platforms = requestedPlatforms(task);
|
||||
if (platforms.length === 0) {
|
||||
return submitResult(task.id, {
|
||||
status: 'FAILED',
|
||||
errorMessage: 'No supported platforms requested.',
|
||||
result: { action: 'sync_metrics', platform: 'multi', platformResults: {} }
|
||||
});
|
||||
}
|
||||
|
||||
const platformResults = {};
|
||||
const videos = [];
|
||||
const metrics = emptyMetrics();
|
||||
const errors = [];
|
||||
|
||||
for (const platform of platforms) {
|
||||
const handler = PLATFORM_HANDLERS[platform];
|
||||
const platformTask = {
|
||||
...task,
|
||||
platform,
|
||||
accountId: accountIdFor(task, platform),
|
||||
payload: payloadForPlatform(task, platform)
|
||||
};
|
||||
try {
|
||||
const response = await handler.collectMetrics(platformTask);
|
||||
const result = response.result || {};
|
||||
platformResults[platform] = {
|
||||
...result,
|
||||
platform,
|
||||
accountId: platformTask.accountId,
|
||||
status: response.status
|
||||
};
|
||||
addMetrics(metrics, result.metrics || {});
|
||||
for (const video of result.videos || []) {
|
||||
videos.push({ ...video, platform, accountId: platformTask.accountId });
|
||||
}
|
||||
if (response.status !== 'SUCCESS') {
|
||||
errors.push(`${platform}: ${response.errorMessage || 'failed'}`);
|
||||
}
|
||||
} catch (error) {
|
||||
platformResults[platform] = {
|
||||
platform,
|
||||
accountId: platformTask.accountId,
|
||||
status: 'FAILED',
|
||||
errorMessage: error.message
|
||||
};
|
||||
errors.push(`${platform}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return submitResult(task.id, {
|
||||
status: errors.length === platforms.length ? 'FAILED' : 'SUCCESS',
|
||||
errorMessage: errors.length > 0 ? errors.join('; ') : null,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'multi',
|
||||
platforms,
|
||||
metrics,
|
||||
videos,
|
||||
videoCount: videos.length,
|
||||
platformResults,
|
||||
extractionMode: 'multi_platform'
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function emptyMetrics() {
|
||||
return { views: 0, likes: 0, comments: 0, collects: 0, shares: 0 };
|
||||
}
|
||||
|
||||
function addMetrics(target, source) {
|
||||
target.views += Number(source.views || 0);
|
||||
target.likes += Number(source.likes || 0);
|
||||
target.comments += Number(source.comments || 0);
|
||||
target.collects += Number(source.collects || 0);
|
||||
target.shares += Number(source.shares || 0);
|
||||
}
|
||||
|
||||
async function workerOnce() {
|
||||
const tasks = await pollTasks();
|
||||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||||
console.log('No pending media matrix tasks.');
|
||||
return;
|
||||
}
|
||||
for (const task of tasks) {
|
||||
const result = await handleTask(task);
|
||||
console.log(`Completed ${task.id}: ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function workerLoop() {
|
||||
const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000);
|
||||
console.log(`Media matrix worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`);
|
||||
while (true) {
|
||||
try {
|
||||
await workerOnce();
|
||||
} catch (error) {
|
||||
console.error(error.stack || error.message);
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mode = process.argv[2];
|
||||
if (mode === 'worker') {
|
||||
await workerOnce();
|
||||
return;
|
||||
}
|
||||
if (mode === 'worker-loop') {
|
||||
await workerLoop();
|
||||
return;
|
||||
}
|
||||
throw new Error('Usage: node scripts/local_media_matrix_worker.js <worker|worker-loop>');
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
517
scripts/local_xiaohongshu_worker.js
Normal file
517
scripts/local_xiaohongshu_worker.js
Normal file
@ -0,0 +1,517 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
|
||||
const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880';
|
||||
const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-xiaohongshu-worker';
|
||||
const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token';
|
||||
const DEFAULT_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company';
|
||||
const DEFAULT_CREATOR_URL = process.env.XHS_CREATOR_URL || 'https://creator.xiaohongshu.com';
|
||||
const DEFAULT_CONTENT_URL = process.env.XHS_CONTENT_URL || 'https://creator.xiaohongshu.com';
|
||||
const DEFAULT_PROFILE_ROOT = process.env.XHS_PROFILE_ROOT || path.resolve('data/platform-profiles');
|
||||
|
||||
const LOGIN_TEXT = new RegExp([
|
||||
'\\u626b\\u7801\\u767b\\u5f55',
|
||||
'\\u9a8c\\u8bc1\\u7801',
|
||||
'\\u624b\\u673a\\u53f7',
|
||||
'\\u767b\\u5f55',
|
||||
'\\u6ce8\\u518c'
|
||||
].join('|'));
|
||||
|
||||
const CREATOR_TEXT = new RegExp([
|
||||
'\\u5c0f\\u7ea2\\u4e66',
|
||||
'\\u521b\\u4f5c\\u670d\\u52a1',
|
||||
'\\u7b14\\u8bb0',
|
||||
'\\u6570\\u636e',
|
||||
'\\u4f5c\\u54c1',
|
||||
'\\u53d1\\u5e03'
|
||||
].join('|'));
|
||||
|
||||
function requirePlaywright() {
|
||||
try {
|
||||
return require('playwright');
|
||||
} catch (error) {
|
||||
throw new Error('Playwright is not installed. Run npm.cmd install in the project root first.');
|
||||
}
|
||||
}
|
||||
|
||||
function profileDir(accountId = DEFAULT_ACCOUNT_ID) {
|
||||
return path.resolve(DEFAULT_PROFILE_ROOT, accountId);
|
||||
}
|
||||
|
||||
function ensureDir(dir) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
function browserChannel() {
|
||||
if (process.env.XHS_BROWSER_CHANNEL) {
|
||||
return process.env.XHS_BROWSER_CHANNEL;
|
||||
}
|
||||
return process.platform === 'win32' ? 'msedge' : undefined;
|
||||
}
|
||||
|
||||
async function launchContext({ accountId = DEFAULT_ACCOUNT_ID, headless = false } = {}) {
|
||||
const { chromium } = requirePlaywright();
|
||||
const dir = profileDir(accountId);
|
||||
ensureDir(dir);
|
||||
const channel = browserChannel();
|
||||
const options = {
|
||||
headless,
|
||||
viewport: { width: 1440, height: 1000 },
|
||||
acceptDownloads: true,
|
||||
args: [
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-dev-shm-usage'
|
||||
]
|
||||
};
|
||||
if (channel) {
|
||||
options.channel = channel;
|
||||
}
|
||||
return chromium.launchPersistentContext(dir, options);
|
||||
}
|
||||
|
||||
async function waitForEnter(message) {
|
||||
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
||||
return new Promise(resolve => {
|
||||
rl.question(message, () => {
|
||||
rl.close();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function login() {
|
||||
const accountId = process.argv[3] || DEFAULT_ACCOUNT_ID;
|
||||
const context = await launchContext({ accountId, headless: false });
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
console.log(`Xiaohongshu login profile: ${profileDir(accountId)}`);
|
||||
console.log('Please login in the opened browser. Keep this window open until login succeeds.');
|
||||
await waitForEnter('Press Enter here after Xiaohongshu creator center is logged in...');
|
||||
const status = await detectLogin(page, accountId);
|
||||
await context.close();
|
||||
console.log(JSON.stringify(status, null, 2));
|
||||
}
|
||||
|
||||
async function checkLogin(accountId = DEFAULT_ACCOUNT_ID) {
|
||||
const context = await launchContext({ accountId, headless: true });
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
try {
|
||||
await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForTimeout(2500);
|
||||
return await detectLogin(page, accountId);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function detectLogin(page, accountId = DEFAULT_ACCOUNT_ID) {
|
||||
const url = page.url();
|
||||
const title = await page.title().catch(() => '');
|
||||
const bodyText = await page.locator('body').innerText({ timeout: 5000 }).catch(() => '');
|
||||
const loginVisible = LOGIN_TEXT.test(bodyText);
|
||||
const creatorVisible = CREATOR_TEXT.test(bodyText);
|
||||
const loggedIn = creatorVisible || (!loginVisible && /xiaohongshu\.com/.test(url));
|
||||
return {
|
||||
loggedIn,
|
||||
status: loggedIn ? 'SUCCESS' : 'FAILED',
|
||||
message: loggedIn ? 'Xiaohongshu creator profile is logged in.' : 'Xiaohongshu creator profile is not logged in.',
|
||||
url,
|
||||
title,
|
||||
profileDir: profileDir(accountId)
|
||||
};
|
||||
}
|
||||
|
||||
async function collectMetrics(task) {
|
||||
const payload = task.payload || {};
|
||||
const accountId = task.accountId || DEFAULT_ACCOUNT_ID;
|
||||
const targetUrl = payload.metricsUrl || payload.contentUrl || DEFAULT_CONTENT_URL;
|
||||
const headless = payload.headless === false ? false : true;
|
||||
const context = await launchContext({ accountId, headless });
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
try {
|
||||
await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
|
||||
await page.waitForTimeout(Number(payload.waitMs || 6000));
|
||||
const status = await detectLogin(page, accountId);
|
||||
if (!status.loggedIn) {
|
||||
return {
|
||||
status: 'FAILED',
|
||||
errorMessage: status.message,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'xiaohongshu',
|
||||
loggedIn: false,
|
||||
...status
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const text = await page.locator('body').innerText({ timeout: 15000 }).catch(() => '');
|
||||
const screenshot = await saveScreenshot(page, task.id);
|
||||
const videos = await collectVideoMetrics(page, Number(payload.maxVideos || 20));
|
||||
const metrics = videos.length > 0 ? summarizeVideos(videos) : extractMetrics(text);
|
||||
return {
|
||||
status: 'SUCCESS',
|
||||
errorMessage: null,
|
||||
result: {
|
||||
action: 'sync_metrics',
|
||||
platform: 'xiaohongshu',
|
||||
loggedIn: true,
|
||||
url: page.url(),
|
||||
title: await page.title().catch(() => ''),
|
||||
metrics,
|
||||
videos,
|
||||
videoCount: videos.length,
|
||||
extractionMode: videos.length > 0 ? 'video_list' : 'page_summary',
|
||||
screenshot,
|
||||
pageTextPreview: compactText(text).slice(0, 2000)
|
||||
}
|
||||
};
|
||||
} finally {
|
||||
await context.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function saveScreenshot(page, taskId) {
|
||||
const dir = path.resolve('target/media/xiaohongshu');
|
||||
ensureDir(dir);
|
||||
const file = path.join(dir, `${taskId || Date.now()}-metrics.png`);
|
||||
await page.screenshot({ path: file, fullPage: true }).catch(() => {});
|
||||
return file;
|
||||
}
|
||||
|
||||
function compactText(text) {
|
||||
return (text || '').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function extractMetrics(text) {
|
||||
const compact = compactText(text);
|
||||
return {
|
||||
views: findMetric(compact, ['\\u6d4f\\u89c8', '\\u89c2\\u770b', '\\u9605\\u8bfb', '\\u66dd\\u5149']),
|
||||
likes: findMetric(compact, ['\\u70b9\\u8d5e', '\\u8d5e']),
|
||||
comments: findMetric(compact, ['\\u8bc4\\u8bba']),
|
||||
collects: findMetric(compact, ['\\u6536\\u85cf']),
|
||||
shares: findMetric(compact, ['\\u5206\\u4eab', '\\u8f6c\\u53d1'])
|
||||
};
|
||||
}
|
||||
|
||||
async function collectVideoMetrics(page, limit = 20) {
|
||||
await autoScroll(page);
|
||||
const blocks = await page.evaluate(maxBlocks => {
|
||||
const selectors = [
|
||||
'tr',
|
||||
'article',
|
||||
'li',
|
||||
'[class*="note"]',
|
||||
'[class*="card"]',
|
||||
'[class*="item"]',
|
||||
'[class*="work"]',
|
||||
'[class*="video"]',
|
||||
'[class*="content"]'
|
||||
];
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const selector of selectors) {
|
||||
for (const element of document.querySelectorAll(selector)) {
|
||||
if (rows.length >= maxBlocks) {
|
||||
break;
|
||||
}
|
||||
const rect = element.getBoundingClientRect();
|
||||
if (rect.width < 140 || rect.height < 40) {
|
||||
continue;
|
||||
}
|
||||
const text = (element.innerText || '').replace(/\s+/g, ' ').trim();
|
||||
if (text.length < 10 || text.length > 1200) {
|
||||
continue;
|
||||
}
|
||||
const signature = text.slice(0, 120);
|
||||
if (seen.has(signature)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(signature);
|
||||
const link = element.querySelector('a[href]');
|
||||
rows.push({
|
||||
text,
|
||||
href: link ? link.href : ''
|
||||
});
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}, Math.max(limit * 8, 80));
|
||||
|
||||
const videos = [];
|
||||
const seen = new Set();
|
||||
for (const block of blocks) {
|
||||
const video = parseVideoBlock(block);
|
||||
if (!video) {
|
||||
continue;
|
||||
}
|
||||
const key = `${video.title}|${video.views}|${video.likes}|${video.comments}|${video.href}`;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
videos.push(video);
|
||||
if (videos.length >= limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return videos;
|
||||
}
|
||||
|
||||
async function autoScroll(page) {
|
||||
await page.evaluate(async () => {
|
||||
await new Promise(resolve => {
|
||||
let count = 0;
|
||||
const timer = setInterval(() => {
|
||||
window.scrollBy(0, Math.floor(window.innerHeight * 0.8));
|
||||
count += 1;
|
||||
if (count >= 3) {
|
||||
clearInterval(timer);
|
||||
resolve();
|
||||
}
|
||||
}, 500);
|
||||
});
|
||||
}).catch(() => {});
|
||||
await page.waitForTimeout(800).catch(() => {});
|
||||
}
|
||||
|
||||
function parseVideoBlock(block) {
|
||||
const text = compactText(block.text);
|
||||
const metrics = extractMetrics(text);
|
||||
const score = metrics.views + metrics.likes + metrics.comments + metrics.collects + metrics.shares;
|
||||
if (score <= 0) {
|
||||
return null;
|
||||
}
|
||||
const title = extractTitle(text);
|
||||
return {
|
||||
title,
|
||||
href: block.href || '',
|
||||
views: metrics.views,
|
||||
likes: metrics.likes,
|
||||
comments: metrics.comments,
|
||||
collects: metrics.collects,
|
||||
shares: metrics.shares,
|
||||
interactionCount: metrics.likes + metrics.comments + metrics.collects + metrics.shares,
|
||||
interactionRate: metrics.views <= 0
|
||||
? 0
|
||||
: (metrics.likes + metrics.comments + metrics.collects + metrics.shares) / metrics.views,
|
||||
rawTextPreview: text.slice(0, 500)
|
||||
};
|
||||
}
|
||||
|
||||
function extractTitle(text) {
|
||||
const cleaned = text
|
||||
.replace(/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}.*/g, '')
|
||||
.replace(/(\u6d4f\u89c8|\u89c2\u770b|\u9605\u8bfb|\u66dd\u5149|\u70b9\u8d5e|\u8d5e|\u8bc4\u8bba|\u6536\u85cf|\u5206\u4eab|\u8f6c\u53d1).*/g, '')
|
||||
.trim();
|
||||
if (cleaned.length >= 2) {
|
||||
return cleaned.slice(0, 120);
|
||||
}
|
||||
return text.slice(0, 80);
|
||||
}
|
||||
|
||||
function summarizeVideos(videos) {
|
||||
const summary = {
|
||||
views: 0,
|
||||
likes: 0,
|
||||
comments: 0,
|
||||
collects: 0,
|
||||
shares: 0
|
||||
};
|
||||
for (const video of videos) {
|
||||
summary.views += Number(video.views || 0);
|
||||
summary.likes += Number(video.likes || 0);
|
||||
summary.comments += Number(video.comments || 0);
|
||||
summary.collects += Number(video.collects || 0);
|
||||
summary.shares += Number(video.shares || 0);
|
||||
}
|
||||
return summary;
|
||||
}
|
||||
|
||||
function findMetric(text, labels) {
|
||||
for (const label of labels) {
|
||||
const labelPattern = new RegExp(label);
|
||||
const decodedLabel = label.replace(/\\u([0-9a-fA-F]{4})/g, (_, code) => String.fromCharCode(parseInt(code, 16)));
|
||||
const after = new RegExp(`${label}[^0-9\\d]{0,16}([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)`);
|
||||
const before = new RegExp(`([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)[^0-9\\d]{0,16}${label}`);
|
||||
if (labelPattern.test(text)) {
|
||||
const afterMatch = text.match(after);
|
||||
if (afterMatch) {
|
||||
return parseMetricNumber(afterMatch[1], afterMatch[2]);
|
||||
}
|
||||
const beforeMatch = text.match(before);
|
||||
if (beforeMatch) {
|
||||
return parseMetricNumber(beforeMatch[1], beforeMatch[2]);
|
||||
}
|
||||
const looseIndex = text.indexOf(decodedLabel);
|
||||
if (looseIndex >= 0) {
|
||||
const around = text.slice(Math.max(0, looseIndex - 30), looseIndex + 60);
|
||||
const number = around.match(/([0-9][0-9,.]*)(\s*[\u4e07\u5343wWkK]?)/);
|
||||
if (number) {
|
||||
return parseMetricNumber(number[1], number[2]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function parseMetricNumber(raw, unit = '') {
|
||||
const value = Number(String(raw).replace(/,/g, ''));
|
||||
if (!Number.isFinite(value)) {
|
||||
return 0;
|
||||
}
|
||||
const normalizedUnit = unit.trim().toLowerCase();
|
||||
if (normalizedUnit.includes('\u4e07') || normalizedUnit.includes('w')) {
|
||||
return Math.round(value * 10000);
|
||||
}
|
||||
if (normalizedUnit.includes('\u5343') || normalizedUnit.includes('k')) {
|
||||
return Math.round(value * 1000);
|
||||
}
|
||||
return Math.round(value);
|
||||
}
|
||||
|
||||
async function pollTasks() {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify({
|
||||
workerId: DEFAULT_WORKER_ID,
|
||||
platforms: ['xiaohongshu'],
|
||||
maxTasks: 5
|
||||
})
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Poll failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function submitResult(taskId, body) {
|
||||
const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Worker-Token': DEFAULT_WORKER_TOKEN
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Submit result failed: ${response.status} ${await response.text()}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function handleTask(task) {
|
||||
const payload = task.payload || {};
|
||||
const action = payload.action || (task.taskType === 'SYNC_METRICS' ? 'sync_metrics' : 'check_login');
|
||||
if (action === 'check_login') {
|
||||
const status = await checkLogin(task.accountId || DEFAULT_ACCOUNT_ID);
|
||||
return submitResult(task.id, {
|
||||
status: status.loggedIn ? 'SUCCESS' : 'FAILED',
|
||||
errorMessage: status.loggedIn ? null : status.message,
|
||||
result: {
|
||||
action,
|
||||
platform: 'xiaohongshu',
|
||||
loggedIn: status.loggedIn,
|
||||
message: status.message,
|
||||
url: status.url,
|
||||
title: status.title,
|
||||
profileDir: profileDir(task.accountId || DEFAULT_ACCOUNT_ID)
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (action === 'sync_metrics') {
|
||||
const result = await collectMetrics(task);
|
||||
return submitResult(task.id, result);
|
||||
}
|
||||
|
||||
return submitResult(task.id, {
|
||||
status: 'FAILED',
|
||||
errorMessage: `Unsupported local Xiaohongshu worker action: ${action}`,
|
||||
result: { action, platform: 'xiaohongshu' }
|
||||
});
|
||||
}
|
||||
|
||||
async function workerOnce() {
|
||||
const tasks = await pollTasks();
|
||||
if (!Array.isArray(tasks) || tasks.length === 0) {
|
||||
console.log('No pending Xiaohongshu tasks.');
|
||||
return;
|
||||
}
|
||||
for (const task of tasks) {
|
||||
const result = await handleTask(task);
|
||||
console.log(`Completed ${task.id}: ${result.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function workerLoop() {
|
||||
const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000);
|
||||
console.log(`Xiaohongshu worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`);
|
||||
while (true) {
|
||||
try {
|
||||
await workerOnce();
|
||||
} catch (error) {
|
||||
console.error(error.stack || error.message);
|
||||
}
|
||||
await sleep(intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const mode = process.argv[2];
|
||||
if (mode === 'login') {
|
||||
await login();
|
||||
return;
|
||||
}
|
||||
if (mode === 'check-login') {
|
||||
const status = await checkLogin(process.argv[3] || DEFAULT_ACCOUNT_ID);
|
||||
console.log(JSON.stringify(status, null, 2));
|
||||
process.exit(status.loggedIn ? 0 : 2);
|
||||
}
|
||||
if (mode === 'metrics') {
|
||||
const task = {
|
||||
id: `manual_${Date.now()}`,
|
||||
taskType: 'SYNC_METRICS',
|
||||
platform: 'xiaohongshu',
|
||||
accountId: process.argv[3] || DEFAULT_ACCOUNT_ID,
|
||||
payload: { action: 'sync_metrics', headless: false }
|
||||
};
|
||||
console.log(JSON.stringify((await collectMetrics(task)).result, null, 2));
|
||||
return;
|
||||
}
|
||||
if (mode === 'worker') {
|
||||
await workerOnce();
|
||||
return;
|
||||
}
|
||||
if (mode === 'worker-loop') {
|
||||
await workerLoop();
|
||||
return;
|
||||
}
|
||||
throw new Error('Usage: node scripts/local_xiaohongshu_worker.js <login|check-login|metrics|worker|worker-loop> [accountId]');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
checkLogin,
|
||||
collectMetrics,
|
||||
login,
|
||||
profileDir,
|
||||
workerLoop,
|
||||
workerOnce
|
||||
};
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(error => {
|
||||
console.error(error.stack || error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@ -1,8 +1,10 @@
|
||||
param(
|
||||
[string]$BaseUrl = "http://localhost:8880",
|
||||
[string]$AccountId = "douyin_company",
|
||||
[string]$XhsAccountId = "xiaohongshu_company",
|
||||
[int]$WorkerIntervalSeconds = 30,
|
||||
[switch]$RemoteBackend
|
||||
[switch]$RemoteBackend,
|
||||
[string]$Platforms = "douyin,xiaohongshu"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@ -115,25 +117,62 @@ if ($RemoteBackend -or -not (Test-LocalBaseUrl $BaseUrl)) {
|
||||
Write-Step "Checking Douyin login"
|
||||
$env:MEDIA_BASE_URL = $BaseUrl
|
||||
$env:DOUYIN_ACCOUNT_ID = $AccountId
|
||||
$checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host $checkOutput
|
||||
Write-Host ""
|
||||
Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window."
|
||||
& npm.cmd run douyin:login -- $AccountId
|
||||
$platformList = $Platforms.Split(",") | ForEach-Object { $_.Trim().ToLower() } | Where-Object { $_ }
|
||||
|
||||
if ($platformList -contains "douyin") {
|
||||
$checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Douyin login failed."
|
||||
Write-Host $checkOutput
|
||||
Write-Host ""
|
||||
Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window."
|
||||
& npm.cmd run douyin:login -- $AccountId
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Douyin login failed."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Douyin login profile is ready."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Douyin login profile is ready."
|
||||
}
|
||||
|
||||
Write-Step "Starting Douyin worker loop"
|
||||
$workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal
|
||||
if ($platformList -contains "xiaohongshu") {
|
||||
Write-Step "Checking Xiaohongshu login"
|
||||
$env:XHS_ACCOUNT_ID = $XhsAccountId
|
||||
$xhsCheckOutput = & npm.cmd run xhs:check -- $XhsAccountId 2>&1
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host $xhsCheckOutput
|
||||
Write-Host ""
|
||||
Write-Host "Xiaohongshu login is required. A browser will open. Login, then press Enter in that window."
|
||||
& npm.cmd run xhs:login -- $XhsAccountId
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Xiaohongshu login failed."
|
||||
}
|
||||
} else {
|
||||
Write-Host "Xiaohongshu login profile is ready."
|
||||
}
|
||||
}
|
||||
|
||||
if ($platformList -contains "douyin") {
|
||||
Write-Step "Starting Douyin worker loop"
|
||||
$workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal
|
||||
}
|
||||
|
||||
if ($platformList -contains "xiaohongshu") {
|
||||
Write-Step "Starting Xiaohongshu worker loop"
|
||||
$xhsWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run xhs:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $xhsWorkerCommand) -WindowStyle Normal
|
||||
}
|
||||
|
||||
if (($platformList -contains "douyin") -and ($platformList -contains "xiaohongshu")) {
|
||||
Write-Step "Starting media matrix worker loop"
|
||||
$matrixWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run media:worker:loop"
|
||||
Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $matrixWorkerCommand) -WindowStyle Normal
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Local client is ready."
|
||||
Write-Host "Backend: $BaseUrl"
|
||||
Write-Host "Account: $AccountId"
|
||||
Write-Host "Douyin account: $AccountId"
|
||||
Write-Host "Xiaohongshu account: $XhsAccountId"
|
||||
Write-Host "Platforms: $($platformList -join ', ')"
|
||||
Write-Host "Worker polling interval: $WorkerIntervalSeconds seconds"
|
||||
|
||||
@ -67,48 +67,47 @@ public class MediaTask {
|
||||
long shares = 0;
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
List<Map<String, Object>> videoRows = new ArrayList<>();
|
||||
Map<String, MetricBucket> byPlatform = new LinkedHashMap<>();
|
||||
for (MediaTask task : tasks) {
|
||||
if (task.status != MediaTaskStatus.SUCCESS || task.result == null) {
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> platformResults = mapValue(task.result.get("platformResults"));
|
||||
if (!platformResults.isEmpty()) {
|
||||
for (Map.Entry<String, Object> entry : platformResults.entrySet()) {
|
||||
Map<String, Object> platformResult = mapValue(entry.getValue());
|
||||
Map<String, Object> metrics = mapValue(platformResult.get("metrics"));
|
||||
if (metrics.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
MetricValues values = metricValues(metrics);
|
||||
views += values.views;
|
||||
likes += values.likes;
|
||||
comments += values.comments;
|
||||
collects += values.collects;
|
||||
shares += values.shares;
|
||||
String platform = entry.getKey();
|
||||
String accountId = stringValue(platformResult.get("accountId"));
|
||||
byPlatform.computeIfAbsent(platform, ignored -> new MetricBucket()).add(values);
|
||||
rows.add(metricRow(task.id, platform, accountId, values));
|
||||
addVideoRows(videoRows, task, platform, accountId, listOfMaps(platformResult.get("videos")));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> metrics = mapValue(task.result.get("metrics"));
|
||||
if (metrics.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
long taskViews = longValue(metrics.get("views"));
|
||||
long taskLikes = longValue(metrics.get("likes"));
|
||||
long taskComments = longValue(metrics.get("comments"));
|
||||
long taskCollects = longValue(metrics.get("collects"));
|
||||
long taskShares = longValue(metrics.get("shares"));
|
||||
views += taskViews;
|
||||
likes += taskLikes;
|
||||
comments += taskComments;
|
||||
collects += taskCollects;
|
||||
shares += taskShares;
|
||||
rows.add(Map.of(
|
||||
"taskId", task.id,
|
||||
"platform", task.platform,
|
||||
"accountId", task.accountId,
|
||||
"views", taskViews,
|
||||
"likes", taskLikes,
|
||||
"comments", taskComments,
|
||||
"collects", taskCollects,
|
||||
"shares", taskShares
|
||||
));
|
||||
for (Map<String, Object> video : listOfMaps(task.result.get("videos"))) {
|
||||
videoRows.add(Map.of(
|
||||
"taskId", task.id,
|
||||
"platform", task.platform,
|
||||
"accountId", task.accountId,
|
||||
"title", stringValue(video.get("title")),
|
||||
"href", stringValue(video.get("href")),
|
||||
"views", longValue(video.get("views")),
|
||||
"likes", longValue(video.get("likes")),
|
||||
"comments", longValue(video.get("comments")),
|
||||
"collects", longValue(video.get("collects")),
|
||||
"shares", longValue(video.get("shares"))
|
||||
));
|
||||
}
|
||||
MetricValues values = metricValues(metrics);
|
||||
views += values.views;
|
||||
likes += values.likes;
|
||||
comments += values.comments;
|
||||
collects += values.collects;
|
||||
shares += values.shares;
|
||||
byPlatform.computeIfAbsent(task.platform, ignored -> new MetricBucket())
|
||||
.add(values);
|
||||
rows.add(metricRow(task.id, task.platform, task.accountId, values));
|
||||
addVideoRows(videoRows, task, task.platform, task.accountId, listOfMaps(task.result.get("videos")));
|
||||
}
|
||||
Map<String, Object> summary = new LinkedHashMap<>();
|
||||
summary.put("views", views);
|
||||
@ -121,6 +120,7 @@ public class MediaTask {
|
||||
summary.put("rows", rows);
|
||||
summary.put("videoCount", videoRows.size());
|
||||
summary.put("videoRows", videoRows);
|
||||
summary.put("byPlatform", platformSummary(byPlatform));
|
||||
return summary;
|
||||
}
|
||||
|
||||
@ -159,4 +159,96 @@ public class MediaTask {
|
||||
private static String stringValue(Object value) {
|
||||
return value == null ? "" : String.valueOf(value);
|
||||
}
|
||||
|
||||
private static MetricValues metricValues(Map<String, Object> metrics) {
|
||||
return new MetricValues(
|
||||
longValue(metrics.get("views")),
|
||||
longValue(metrics.get("likes")),
|
||||
longValue(metrics.get("comments")),
|
||||
longValue(metrics.get("collects")),
|
||||
longValue(metrics.get("shares"))
|
||||
);
|
||||
}
|
||||
|
||||
private static Map<String, Object> metricRow(String taskId, String platform, String accountId, MetricValues values) {
|
||||
return Map.of(
|
||||
"taskId", taskId,
|
||||
"platform", platform,
|
||||
"accountId", accountId,
|
||||
"views", values.views,
|
||||
"likes", values.likes,
|
||||
"comments", values.comments,
|
||||
"collects", values.collects,
|
||||
"shares", values.shares
|
||||
);
|
||||
}
|
||||
|
||||
private static void addVideoRows(
|
||||
List<Map<String, Object>> videoRows,
|
||||
MediaTask task,
|
||||
String platform,
|
||||
String accountId,
|
||||
List<Map<String, Object>> videos
|
||||
) {
|
||||
for (Map<String, Object> video : videos) {
|
||||
videoRows.add(Map.of(
|
||||
"taskId", task.id,
|
||||
"platform", platform,
|
||||
"accountId", accountId,
|
||||
"title", stringValue(video.get("title")),
|
||||
"href", stringValue(video.get("href")),
|
||||
"views", longValue(video.get("views")),
|
||||
"likes", longValue(video.get("likes")),
|
||||
"comments", longValue(video.get("comments")),
|
||||
"collects", longValue(video.get("collects")),
|
||||
"shares", longValue(video.get("shares"))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, Object> platformSummary(Map<String, MetricBucket> buckets) {
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
for (Map.Entry<String, MetricBucket> entry : buckets.entrySet()) {
|
||||
result.put(entry.getKey(), entry.getValue().toMap());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static class MetricBucket {
|
||||
private long views;
|
||||
private long likes;
|
||||
private long comments;
|
||||
private long collects;
|
||||
private long shares;
|
||||
|
||||
private void add(MetricValues values) {
|
||||
this.views += values.views;
|
||||
this.likes += values.likes;
|
||||
this.comments += values.comments;
|
||||
this.collects += values.collects;
|
||||
this.shares += values.shares;
|
||||
}
|
||||
|
||||
private Map<String, Object> toMap() {
|
||||
long interactionCount = likes + comments + collects + shares;
|
||||
Map<String, Object> map = new LinkedHashMap<>();
|
||||
map.put("views", views);
|
||||
map.put("likes", likes);
|
||||
map.put("comments", comments);
|
||||
map.put("collects", collects);
|
||||
map.put("shares", shares);
|
||||
map.put("interactionCount", interactionCount);
|
||||
map.put("interactionRate", views <= 0 ? 0.0 : interactionCount * 1.0 / views);
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
private record MetricValues(
|
||||
long views,
|
||||
long likes,
|
||||
long comments,
|
||||
long collects,
|
||||
long shares
|
||||
) {
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0"
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend
|
||||
powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend -Platforms "douyin,xiaohongshu"
|
||||
pause
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user