diff --git a/CLIENT_README.txt b/CLIENT_README.txt index 95842fb..eb8c6e9 100644 --- a/CLIENT_README.txt +++ b/CLIENT_README.txt @@ -7,7 +7,7 @@ Before first use, install: How to start: 1. Unzip this package to a normal folder, for example Desktop\agent-foundation-client. 2. Double-click start_yuedong_client.bat. -3. If a Douyin login page opens, login in the browser, then press Enter in the terminal window. +3. If a Douyin or Xiaohongshu login page opens, login in the browser, then press Enter in the terminal window. 4. Keep the worker terminal window open while using the system. Server API: diff --git a/dist/agent-foundation-client-yuedong.zip b/dist/agent-foundation-client-yuedong.zip index 430b563..a2e52e1 100644 Binary files a/dist/agent-foundation-client-yuedong.zip and b/dist/agent-foundation-client-yuedong.zip differ diff --git a/dist/agent-foundation-client-yuedong/CLIENT_README.txt b/dist/agent-foundation-client-yuedong/CLIENT_README.txt index 95842fb..eb8c6e9 100644 --- a/dist/agent-foundation-client-yuedong/CLIENT_README.txt +++ b/dist/agent-foundation-client-yuedong/CLIENT_README.txt @@ -7,7 +7,7 @@ Before first use, install: How to start: 1. Unzip this package to a normal folder, for example Desktop\agent-foundation-client. 2. Double-click start_yuedong_client.bat. -3. If a Douyin login page opens, login in the browser, then press Enter in the terminal window. +3. If a Douyin or Xiaohongshu login page opens, login in the browser, then press Enter in the terminal window. 4. Keep the worker terminal window open while using the system. Server API: diff --git a/dist/agent-foundation-client-yuedong/package.json b/dist/agent-foundation-client-yuedong/package.json index 97c513e..e81aec7 100644 --- a/dist/agent-foundation-client-yuedong/package.json +++ b/dist/agent-foundation-client-yuedong/package.json @@ -7,7 +7,14 @@ "douyin:worker": "node scripts/local_douyin_worker.js worker", "douyin:worker:loop": "node scripts/local_douyin_worker.js worker-loop", "douyin:check": "node scripts/local_douyin_worker.js check-login", - "douyin:metrics": "node scripts/local_douyin_worker.js metrics" + "douyin:metrics": "node scripts/local_douyin_worker.js metrics", + "xhs:login": "node scripts/local_xiaohongshu_worker.js login", + "xhs:worker": "node scripts/local_xiaohongshu_worker.js worker", + "xhs:worker:loop": "node scripts/local_xiaohongshu_worker.js worker-loop", + "xhs:check": "node scripts/local_xiaohongshu_worker.js check-login", + "xhs:metrics": "node scripts/local_xiaohongshu_worker.js metrics", + "media:worker": "node scripts/local_media_matrix_worker.js worker", + "media:worker:loop": "node scripts/local_media_matrix_worker.js worker-loop" }, "dependencies": { "playwright": "^1.61.0" diff --git a/dist/agent-foundation-client-yuedong/scripts/local_douyin_worker.js b/dist/agent-foundation-client-yuedong/scripts/local_douyin_worker.js index 9c8c3cf..c4d1a24 100644 --- a/dist/agent-foundation-client-yuedong/scripts/local_douyin_worker.js +++ b/dist/agent-foundation-client-yuedong/scripts/local_douyin_worker.js @@ -137,8 +137,9 @@ async function collectMetrics(task) { status: 'FAILED', errorMessage: status.message, result: { - action: 'sync_metrics', - loggedIn: false, + action: 'sync_metrics', + platform: 'douyin', + loggedIn: false, ...status } }; @@ -153,6 +154,7 @@ async function collectMetrics(task) { errorMessage: null, result: { action: 'sync_metrics', + platform: 'douyin', loggedIn: true, url: page.url(), title: await page.title().catch(() => ''), @@ -418,6 +420,7 @@ async function handleTask(task) { errorMessage: status.loggedIn ? null : status.message, result: { action, + platform: 'douyin', loggedIn: status.loggedIn, message: status.message, url: status.url, @@ -501,7 +504,18 @@ async function main() { throw new Error('Usage: node scripts/local_douyin_worker.js [accountId]'); } -main().catch(error => { - console.error(error.stack || error.message); - process.exit(1); -}); +module.exports = { + checkLogin, + collectMetrics, + login, + profileDir, + workerLoop, + workerOnce +}; + +if (require.main === module) { + main().catch(error => { + console.error(error.stack || error.message); + process.exit(1); + }); +} diff --git a/dist/agent-foundation-client-yuedong/scripts/local_media_matrix_worker.js b/dist/agent-foundation-client-yuedong/scripts/local_media_matrix_worker.js new file mode 100644 index 0000000..9444102 --- /dev/null +++ b/dist/agent-foundation-client-yuedong/scripts/local_media_matrix_worker.js @@ -0,0 +1,217 @@ +const douyin = require('./local_douyin_worker'); +const xiaohongshu = require('./local_xiaohongshu_worker'); + +const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880'; +const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-media-matrix-worker'; +const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token'; +const DEFAULT_DOUYIN_ACCOUNT_ID = process.env.DOUYIN_ACCOUNT_ID || 'douyin_company'; +const DEFAULT_XHS_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company'; + +const PLATFORM_HANDLERS = { + douyin: { + accountId: DEFAULT_DOUYIN_ACCOUNT_ID, + collectMetrics: douyin.collectMetrics, + checkLogin: douyin.checkLogin + }, + xiaohongshu: { + accountId: DEFAULT_XHS_ACCOUNT_ID, + collectMetrics: xiaohongshu.collectMetrics, + checkLogin: xiaohongshu.checkLogin + } +}; + +function normalizePlatform(platform) { + const value = String(platform || '').trim().toLowerCase(); + if (value === 'xhs' || value === 'rednote') { + return 'xiaohongshu'; + } + return value; +} + +function requestedPlatforms(task) { + const payload = task.payload || {}; + const raw = Array.isArray(payload.platforms) && payload.platforms.length > 0 + ? payload.platforms + : ['douyin', 'xiaohongshu']; + return raw.map(normalizePlatform).filter(platform => PLATFORM_HANDLERS[platform]); +} + +function accountIdFor(task, platform) { + const payload = task.payload || {}; + const accountIds = payload.accountIds || {}; + if (accountIds && typeof accountIds === 'object' && accountIds[platform]) { + return accountIds[platform]; + } + return PLATFORM_HANDLERS[platform].accountId; +} + +function payloadForPlatform(task, platform) { + const payload = task.payload || {}; + const platformPayloads = payload.platformPayloads || {}; + return { + ...payload, + ...(platformPayloads[platform] || {}), + platforms: undefined, + accountIds: undefined, + platformPayloads: undefined + }; +} + +async function pollTasks() { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify({ + workerId: DEFAULT_WORKER_ID, + platforms: ['multi', 'matrix', 'all'], + maxTasks: 3 + }) + }); + if (!response.ok) { + throw new Error(`Poll failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function submitResult(taskId, body) { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw new Error(`Submit result failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function handleTask(task) { + const platforms = requestedPlatforms(task); + if (platforms.length === 0) { + return submitResult(task.id, { + status: 'FAILED', + errorMessage: 'No supported platforms requested.', + result: { action: 'sync_metrics', platform: 'multi', platformResults: {} } + }); + } + + const platformResults = {}; + const videos = []; + const metrics = emptyMetrics(); + const errors = []; + + for (const platform of platforms) { + const handler = PLATFORM_HANDLERS[platform]; + const platformTask = { + ...task, + platform, + accountId: accountIdFor(task, platform), + payload: payloadForPlatform(task, platform) + }; + try { + const response = await handler.collectMetrics(platformTask); + const result = response.result || {}; + platformResults[platform] = { + ...result, + platform, + accountId: platformTask.accountId, + status: response.status + }; + addMetrics(metrics, result.metrics || {}); + for (const video of result.videos || []) { + videos.push({ ...video, platform, accountId: platformTask.accountId }); + } + if (response.status !== 'SUCCESS') { + errors.push(`${platform}: ${response.errorMessage || 'failed'}`); + } + } catch (error) { + platformResults[platform] = { + platform, + accountId: platformTask.accountId, + status: 'FAILED', + errorMessage: error.message + }; + errors.push(`${platform}: ${error.message}`); + } + } + + return submitResult(task.id, { + status: errors.length === platforms.length ? 'FAILED' : 'SUCCESS', + errorMessage: errors.length > 0 ? errors.join('; ') : null, + result: { + action: 'sync_metrics', + platform: 'multi', + platforms, + metrics, + videos, + videoCount: videos.length, + platformResults, + extractionMode: 'multi_platform' + } + }); +} + +function emptyMetrics() { + return { views: 0, likes: 0, comments: 0, collects: 0, shares: 0 }; +} + +function addMetrics(target, source) { + target.views += Number(source.views || 0); + target.likes += Number(source.likes || 0); + target.comments += Number(source.comments || 0); + target.collects += Number(source.collects || 0); + target.shares += Number(source.shares || 0); +} + +async function workerOnce() { + const tasks = await pollTasks(); + if (!Array.isArray(tasks) || tasks.length === 0) { + console.log('No pending media matrix tasks.'); + return; + } + for (const task of tasks) { + const result = await handleTask(task); + console.log(`Completed ${task.id}: ${result.status}`); + } +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function workerLoop() { + const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000); + console.log(`Media matrix worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`); + while (true) { + try { + await workerOnce(); + } catch (error) { + console.error(error.stack || error.message); + } + await sleep(intervalMs); + } +} + +async function main() { + const mode = process.argv[2]; + if (mode === 'worker') { + await workerOnce(); + return; + } + if (mode === 'worker-loop') { + await workerLoop(); + return; + } + throw new Error('Usage: node scripts/local_media_matrix_worker.js '); +} + +main().catch(error => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/dist/agent-foundation-client-yuedong/scripts/local_xiaohongshu_worker.js b/dist/agent-foundation-client-yuedong/scripts/local_xiaohongshu_worker.js new file mode 100644 index 0000000..ab1b07d --- /dev/null +++ b/dist/agent-foundation-client-yuedong/scripts/local_xiaohongshu_worker.js @@ -0,0 +1,517 @@ +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); + +const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880'; +const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-xiaohongshu-worker'; +const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token'; +const DEFAULT_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company'; +const DEFAULT_CREATOR_URL = process.env.XHS_CREATOR_URL || 'https://creator.xiaohongshu.com'; +const DEFAULT_CONTENT_URL = process.env.XHS_CONTENT_URL || 'https://creator.xiaohongshu.com'; +const DEFAULT_PROFILE_ROOT = process.env.XHS_PROFILE_ROOT || path.resolve('data/platform-profiles'); + +const LOGIN_TEXT = new RegExp([ + '\\u626b\\u7801\\u767b\\u5f55', + '\\u9a8c\\u8bc1\\u7801', + '\\u624b\\u673a\\u53f7', + '\\u767b\\u5f55', + '\\u6ce8\\u518c' +].join('|')); + +const CREATOR_TEXT = new RegExp([ + '\\u5c0f\\u7ea2\\u4e66', + '\\u521b\\u4f5c\\u670d\\u52a1', + '\\u7b14\\u8bb0', + '\\u6570\\u636e', + '\\u4f5c\\u54c1', + '\\u53d1\\u5e03' +].join('|')); + +function requirePlaywright() { + try { + return require('playwright'); + } catch (error) { + throw new Error('Playwright is not installed. Run npm.cmd install in the project root first.'); + } +} + +function profileDir(accountId = DEFAULT_ACCOUNT_ID) { + return path.resolve(DEFAULT_PROFILE_ROOT, accountId); +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function browserChannel() { + if (process.env.XHS_BROWSER_CHANNEL) { + return process.env.XHS_BROWSER_CHANNEL; + } + return process.platform === 'win32' ? 'msedge' : undefined; +} + +async function launchContext({ accountId = DEFAULT_ACCOUNT_ID, headless = false } = {}) { + const { chromium } = requirePlaywright(); + const dir = profileDir(accountId); + ensureDir(dir); + const channel = browserChannel(); + const options = { + headless, + viewport: { width: 1440, height: 1000 }, + acceptDownloads: true, + args: [ + '--disable-blink-features=AutomationControlled', + '--disable-dev-shm-usage' + ] + }; + if (channel) { + options.channel = channel; + } + return chromium.launchPersistentContext(dir, options); +} + +async function waitForEnter(message) { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + return new Promise(resolve => { + rl.question(message, () => { + rl.close(); + resolve(); + }); + }); +} + +async function login() { + const accountId = process.argv[3] || DEFAULT_ACCOUNT_ID; + const context = await launchContext({ accountId, headless: false }); + const page = context.pages()[0] || await context.newPage(); + await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); + console.log(`Xiaohongshu login profile: ${profileDir(accountId)}`); + console.log('Please login in the opened browser. Keep this window open until login succeeds.'); + await waitForEnter('Press Enter here after Xiaohongshu creator center is logged in...'); + const status = await detectLogin(page, accountId); + await context.close(); + console.log(JSON.stringify(status, null, 2)); +} + +async function checkLogin(accountId = DEFAULT_ACCOUNT_ID) { + const context = await launchContext({ accountId, headless: true }); + const page = context.pages()[0] || await context.newPage(); + try { + await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.waitForTimeout(2500); + return await detectLogin(page, accountId); + } finally { + await context.close(); + } +} + +async function detectLogin(page, accountId = DEFAULT_ACCOUNT_ID) { + const url = page.url(); + const title = await page.title().catch(() => ''); + const bodyText = await page.locator('body').innerText({ timeout: 5000 }).catch(() => ''); + const loginVisible = LOGIN_TEXT.test(bodyText); + const creatorVisible = CREATOR_TEXT.test(bodyText); + const loggedIn = creatorVisible || (!loginVisible && /xiaohongshu\.com/.test(url)); + return { + loggedIn, + status: loggedIn ? 'SUCCESS' : 'FAILED', + message: loggedIn ? 'Xiaohongshu creator profile is logged in.' : 'Xiaohongshu creator profile is not logged in.', + url, + title, + profileDir: profileDir(accountId) + }; +} + +async function collectMetrics(task) { + const payload = task.payload || {}; + const accountId = task.accountId || DEFAULT_ACCOUNT_ID; + const targetUrl = payload.metricsUrl || payload.contentUrl || DEFAULT_CONTENT_URL; + const headless = payload.headless === false ? false : true; + const context = await launchContext({ accountId, headless }); + const page = context.pages()[0] || await context.newPage(); + try { + await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.waitForTimeout(Number(payload.waitMs || 6000)); + const status = await detectLogin(page, accountId); + if (!status.loggedIn) { + return { + status: 'FAILED', + errorMessage: status.message, + result: { + action: 'sync_metrics', + platform: 'xiaohongshu', + loggedIn: false, + ...status + } + }; + } + + const text = await page.locator('body').innerText({ timeout: 15000 }).catch(() => ''); + const screenshot = await saveScreenshot(page, task.id); + const videos = await collectVideoMetrics(page, Number(payload.maxVideos || 20)); + const metrics = videos.length > 0 ? summarizeVideos(videos) : extractMetrics(text); + return { + status: 'SUCCESS', + errorMessage: null, + result: { + action: 'sync_metrics', + platform: 'xiaohongshu', + loggedIn: true, + url: page.url(), + title: await page.title().catch(() => ''), + metrics, + videos, + videoCount: videos.length, + extractionMode: videos.length > 0 ? 'video_list' : 'page_summary', + screenshot, + pageTextPreview: compactText(text).slice(0, 2000) + } + }; + } finally { + await context.close().catch(() => {}); + } +} + +async function saveScreenshot(page, taskId) { + const dir = path.resolve('target/media/xiaohongshu'); + ensureDir(dir); + const file = path.join(dir, `${taskId || Date.now()}-metrics.png`); + await page.screenshot({ path: file, fullPage: true }).catch(() => {}); + return file; +} + +function compactText(text) { + return (text || '').replace(/\s+/g, ' ').trim(); +} + +function extractMetrics(text) { + const compact = compactText(text); + return { + views: findMetric(compact, ['\\u6d4f\\u89c8', '\\u89c2\\u770b', '\\u9605\\u8bfb', '\\u66dd\\u5149']), + likes: findMetric(compact, ['\\u70b9\\u8d5e', '\\u8d5e']), + comments: findMetric(compact, ['\\u8bc4\\u8bba']), + collects: findMetric(compact, ['\\u6536\\u85cf']), + shares: findMetric(compact, ['\\u5206\\u4eab', '\\u8f6c\\u53d1']) + }; +} + +async function collectVideoMetrics(page, limit = 20) { + await autoScroll(page); + const blocks = await page.evaluate(maxBlocks => { + const selectors = [ + 'tr', + 'article', + 'li', + '[class*="note"]', + '[class*="card"]', + '[class*="item"]', + '[class*="work"]', + '[class*="video"]', + '[class*="content"]' + ]; + const seen = new Set(); + const rows = []; + for (const selector of selectors) { + for (const element of document.querySelectorAll(selector)) { + if (rows.length >= maxBlocks) { + break; + } + const rect = element.getBoundingClientRect(); + if (rect.width < 140 || rect.height < 40) { + continue; + } + const text = (element.innerText || '').replace(/\s+/g, ' ').trim(); + if (text.length < 10 || text.length > 1200) { + continue; + } + const signature = text.slice(0, 120); + if (seen.has(signature)) { + continue; + } + seen.add(signature); + const link = element.querySelector('a[href]'); + rows.push({ + text, + href: link ? link.href : '' + }); + } + } + return rows; + }, Math.max(limit * 8, 80)); + + const videos = []; + const seen = new Set(); + for (const block of blocks) { + const video = parseVideoBlock(block); + if (!video) { + continue; + } + const key = `${video.title}|${video.views}|${video.likes}|${video.comments}|${video.href}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + videos.push(video); + if (videos.length >= limit) { + break; + } + } + return videos; +} + +async function autoScroll(page) { + await page.evaluate(async () => { + await new Promise(resolve => { + let count = 0; + const timer = setInterval(() => { + window.scrollBy(0, Math.floor(window.innerHeight * 0.8)); + count += 1; + if (count >= 3) { + clearInterval(timer); + resolve(); + } + }, 500); + }); + }).catch(() => {}); + await page.waitForTimeout(800).catch(() => {}); +} + +function parseVideoBlock(block) { + const text = compactText(block.text); + const metrics = extractMetrics(text); + const score = metrics.views + metrics.likes + metrics.comments + metrics.collects + metrics.shares; + if (score <= 0) { + return null; + } + const title = extractTitle(text); + return { + title, + href: block.href || '', + views: metrics.views, + likes: metrics.likes, + comments: metrics.comments, + collects: metrics.collects, + shares: metrics.shares, + interactionCount: metrics.likes + metrics.comments + metrics.collects + metrics.shares, + interactionRate: metrics.views <= 0 + ? 0 + : (metrics.likes + metrics.comments + metrics.collects + metrics.shares) / metrics.views, + rawTextPreview: text.slice(0, 500) + }; +} + +function extractTitle(text) { + const cleaned = text + .replace(/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}.*/g, '') + .replace(/(\u6d4f\u89c8|\u89c2\u770b|\u9605\u8bfb|\u66dd\u5149|\u70b9\u8d5e|\u8d5e|\u8bc4\u8bba|\u6536\u85cf|\u5206\u4eab|\u8f6c\u53d1).*/g, '') + .trim(); + if (cleaned.length >= 2) { + return cleaned.slice(0, 120); + } + return text.slice(0, 80); +} + +function summarizeVideos(videos) { + const summary = { + views: 0, + likes: 0, + comments: 0, + collects: 0, + shares: 0 + }; + for (const video of videos) { + summary.views += Number(video.views || 0); + summary.likes += Number(video.likes || 0); + summary.comments += Number(video.comments || 0); + summary.collects += Number(video.collects || 0); + summary.shares += Number(video.shares || 0); + } + return summary; +} + +function findMetric(text, labels) { + for (const label of labels) { + const labelPattern = new RegExp(label); + const decodedLabel = label.replace(/\\u([0-9a-fA-F]{4})/g, (_, code) => String.fromCharCode(parseInt(code, 16))); + const after = new RegExp(`${label}[^0-9\\d]{0,16}([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)`); + const before = new RegExp(`([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)[^0-9\\d]{0,16}${label}`); + if (labelPattern.test(text)) { + const afterMatch = text.match(after); + if (afterMatch) { + return parseMetricNumber(afterMatch[1], afterMatch[2]); + } + const beforeMatch = text.match(before); + if (beforeMatch) { + return parseMetricNumber(beforeMatch[1], beforeMatch[2]); + } + const looseIndex = text.indexOf(decodedLabel); + if (looseIndex >= 0) { + const around = text.slice(Math.max(0, looseIndex - 30), looseIndex + 60); + const number = around.match(/([0-9][0-9,.]*)(\s*[\u4e07\u5343wWkK]?)/); + if (number) { + return parseMetricNumber(number[1], number[2]); + } + } + } + } + return 0; +} + +function parseMetricNumber(raw, unit = '') { + const value = Number(String(raw).replace(/,/g, '')); + if (!Number.isFinite(value)) { + return 0; + } + const normalizedUnit = unit.trim().toLowerCase(); + if (normalizedUnit.includes('\u4e07') || normalizedUnit.includes('w')) { + return Math.round(value * 10000); + } + if (normalizedUnit.includes('\u5343') || normalizedUnit.includes('k')) { + return Math.round(value * 1000); + } + return Math.round(value); +} + +async function pollTasks() { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify({ + workerId: DEFAULT_WORKER_ID, + platforms: ['xiaohongshu'], + maxTasks: 5 + }) + }); + if (!response.ok) { + throw new Error(`Poll failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function submitResult(taskId, body) { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw new Error(`Submit result failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function handleTask(task) { + const payload = task.payload || {}; + const action = payload.action || (task.taskType === 'SYNC_METRICS' ? 'sync_metrics' : 'check_login'); + if (action === 'check_login') { + const status = await checkLogin(task.accountId || DEFAULT_ACCOUNT_ID); + return submitResult(task.id, { + status: status.loggedIn ? 'SUCCESS' : 'FAILED', + errorMessage: status.loggedIn ? null : status.message, + result: { + action, + platform: 'xiaohongshu', + loggedIn: status.loggedIn, + message: status.message, + url: status.url, + title: status.title, + profileDir: profileDir(task.accountId || DEFAULT_ACCOUNT_ID) + } + }); + } + + if (action === 'sync_metrics') { + const result = await collectMetrics(task); + return submitResult(task.id, result); + } + + return submitResult(task.id, { + status: 'FAILED', + errorMessage: `Unsupported local Xiaohongshu worker action: ${action}`, + result: { action, platform: 'xiaohongshu' } + }); +} + +async function workerOnce() { + const tasks = await pollTasks(); + if (!Array.isArray(tasks) || tasks.length === 0) { + console.log('No pending Xiaohongshu tasks.'); + return; + } + for (const task of tasks) { + const result = await handleTask(task); + console.log(`Completed ${task.id}: ${result.status}`); + } +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function workerLoop() { + const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000); + console.log(`Xiaohongshu worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`); + while (true) { + try { + await workerOnce(); + } catch (error) { + console.error(error.stack || error.message); + } + await sleep(intervalMs); + } +} + +async function main() { + const mode = process.argv[2]; + if (mode === 'login') { + await login(); + return; + } + if (mode === 'check-login') { + const status = await checkLogin(process.argv[3] || DEFAULT_ACCOUNT_ID); + console.log(JSON.stringify(status, null, 2)); + process.exit(status.loggedIn ? 0 : 2); + } + if (mode === 'metrics') { + const task = { + id: `manual_${Date.now()}`, + taskType: 'SYNC_METRICS', + platform: 'xiaohongshu', + accountId: process.argv[3] || DEFAULT_ACCOUNT_ID, + payload: { action: 'sync_metrics', headless: false } + }; + console.log(JSON.stringify((await collectMetrics(task)).result, null, 2)); + return; + } + if (mode === 'worker') { + await workerOnce(); + return; + } + if (mode === 'worker-loop') { + await workerLoop(); + return; + } + throw new Error('Usage: node scripts/local_xiaohongshu_worker.js [accountId]'); +} + +module.exports = { + checkLogin, + collectMetrics, + login, + profileDir, + workerLoop, + workerOnce +}; + +if (require.main === module) { + main().catch(error => { + console.error(error.stack || error.message); + process.exit(1); + }); +} diff --git a/dist/agent-foundation-client-yuedong/scripts/start_local_client.ps1 b/dist/agent-foundation-client-yuedong/scripts/start_local_client.ps1 index d20221f..7c8590f 100644 --- a/dist/agent-foundation-client-yuedong/scripts/start_local_client.ps1 +++ b/dist/agent-foundation-client-yuedong/scripts/start_local_client.ps1 @@ -1,8 +1,10 @@ param( [string]$BaseUrl = "http://localhost:8880", [string]$AccountId = "douyin_company", + [string]$XhsAccountId = "xiaohongshu_company", [int]$WorkerIntervalSeconds = 30, - [switch]$RemoteBackend + [switch]$RemoteBackend, + [string]$Platforms = "douyin,xiaohongshu" ) $ErrorActionPreference = "Stop" @@ -115,25 +117,62 @@ if ($RemoteBackend -or -not (Test-LocalBaseUrl $BaseUrl)) { Write-Step "Checking Douyin login" $env:MEDIA_BASE_URL = $BaseUrl $env:DOUYIN_ACCOUNT_ID = $AccountId -$checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host $checkOutput - Write-Host "" - Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window." - & npm.cmd run douyin:login -- $AccountId +$platformList = $Platforms.Split(",") | ForEach-Object { $_.Trim().ToLower() } | Where-Object { $_ } + +if ($platformList -contains "douyin") { + $checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1 if ($LASTEXITCODE -ne 0) { - throw "Douyin login failed." + Write-Host $checkOutput + Write-Host "" + Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window." + & npm.cmd run douyin:login -- $AccountId + if ($LASTEXITCODE -ne 0) { + throw "Douyin login failed." + } + } else { + Write-Host "Douyin login profile is ready." } -} else { - Write-Host "Douyin login profile is ready." } -Write-Step "Starting Douyin worker loop" -$workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop" -Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal +if ($platformList -contains "xiaohongshu") { + Write-Step "Checking Xiaohongshu login" + $env:XHS_ACCOUNT_ID = $XhsAccountId + $xhsCheckOutput = & npm.cmd run xhs:check -- $XhsAccountId 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Host $xhsCheckOutput + Write-Host "" + Write-Host "Xiaohongshu login is required. A browser will open. Login, then press Enter in that window." + & npm.cmd run xhs:login -- $XhsAccountId + if ($LASTEXITCODE -ne 0) { + throw "Xiaohongshu login failed." + } + } else { + Write-Host "Xiaohongshu login profile is ready." + } +} + +if ($platformList -contains "douyin") { + Write-Step "Starting Douyin worker loop" + $workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop" + Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal +} + +if ($platformList -contains "xiaohongshu") { + Write-Step "Starting Xiaohongshu worker loop" + $xhsWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run xhs:worker:loop" + Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $xhsWorkerCommand) -WindowStyle Normal +} + +if (($platformList -contains "douyin") -and ($platformList -contains "xiaohongshu")) { + Write-Step "Starting media matrix worker loop" + $matrixWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run media:worker:loop" + Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $matrixWorkerCommand) -WindowStyle Normal +} Write-Host "" Write-Host "Local client is ready." Write-Host "Backend: $BaseUrl" -Write-Host "Account: $AccountId" +Write-Host "Douyin account: $AccountId" +Write-Host "Xiaohongshu account: $XhsAccountId" +Write-Host "Platforms: $($platformList -join ', ')" Write-Host "Worker polling interval: $WorkerIntervalSeconds seconds" diff --git a/dist/agent-foundation-client-yuedong/start_yuedong_client.bat b/dist/agent-foundation-client-yuedong/start_yuedong_client.bat index df335ef..863c5a0 100644 --- a/dist/agent-foundation-client-yuedong/start_yuedong_client.bat +++ b/dist/agent-foundation-client-yuedong/start_yuedong_client.bat @@ -1,5 +1,5 @@ @echo off setlocal cd /d "%~dp0" -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend -Platforms "douyin,xiaohongshu" pause diff --git a/package.json b/package.json index 97c513e..e81aec7 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,14 @@ "douyin:worker": "node scripts/local_douyin_worker.js worker", "douyin:worker:loop": "node scripts/local_douyin_worker.js worker-loop", "douyin:check": "node scripts/local_douyin_worker.js check-login", - "douyin:metrics": "node scripts/local_douyin_worker.js metrics" + "douyin:metrics": "node scripts/local_douyin_worker.js metrics", + "xhs:login": "node scripts/local_xiaohongshu_worker.js login", + "xhs:worker": "node scripts/local_xiaohongshu_worker.js worker", + "xhs:worker:loop": "node scripts/local_xiaohongshu_worker.js worker-loop", + "xhs:check": "node scripts/local_xiaohongshu_worker.js check-login", + "xhs:metrics": "node scripts/local_xiaohongshu_worker.js metrics", + "media:worker": "node scripts/local_media_matrix_worker.js worker", + "media:worker:loop": "node scripts/local_media_matrix_worker.js worker-loop" }, "dependencies": { "playwright": "^1.61.0" diff --git a/scripts/local_douyin_worker.js b/scripts/local_douyin_worker.js index 9c8c3cf..c4d1a24 100644 --- a/scripts/local_douyin_worker.js +++ b/scripts/local_douyin_worker.js @@ -137,8 +137,9 @@ async function collectMetrics(task) { status: 'FAILED', errorMessage: status.message, result: { - action: 'sync_metrics', - loggedIn: false, + action: 'sync_metrics', + platform: 'douyin', + loggedIn: false, ...status } }; @@ -153,6 +154,7 @@ async function collectMetrics(task) { errorMessage: null, result: { action: 'sync_metrics', + platform: 'douyin', loggedIn: true, url: page.url(), title: await page.title().catch(() => ''), @@ -418,6 +420,7 @@ async function handleTask(task) { errorMessage: status.loggedIn ? null : status.message, result: { action, + platform: 'douyin', loggedIn: status.loggedIn, message: status.message, url: status.url, @@ -501,7 +504,18 @@ async function main() { throw new Error('Usage: node scripts/local_douyin_worker.js [accountId]'); } -main().catch(error => { - console.error(error.stack || error.message); - process.exit(1); -}); +module.exports = { + checkLogin, + collectMetrics, + login, + profileDir, + workerLoop, + workerOnce +}; + +if (require.main === module) { + main().catch(error => { + console.error(error.stack || error.message); + process.exit(1); + }); +} diff --git a/scripts/local_media_matrix_worker.js b/scripts/local_media_matrix_worker.js new file mode 100644 index 0000000..9444102 --- /dev/null +++ b/scripts/local_media_matrix_worker.js @@ -0,0 +1,217 @@ +const douyin = require('./local_douyin_worker'); +const xiaohongshu = require('./local_xiaohongshu_worker'); + +const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880'; +const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-media-matrix-worker'; +const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token'; +const DEFAULT_DOUYIN_ACCOUNT_ID = process.env.DOUYIN_ACCOUNT_ID || 'douyin_company'; +const DEFAULT_XHS_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company'; + +const PLATFORM_HANDLERS = { + douyin: { + accountId: DEFAULT_DOUYIN_ACCOUNT_ID, + collectMetrics: douyin.collectMetrics, + checkLogin: douyin.checkLogin + }, + xiaohongshu: { + accountId: DEFAULT_XHS_ACCOUNT_ID, + collectMetrics: xiaohongshu.collectMetrics, + checkLogin: xiaohongshu.checkLogin + } +}; + +function normalizePlatform(platform) { + const value = String(platform || '').trim().toLowerCase(); + if (value === 'xhs' || value === 'rednote') { + return 'xiaohongshu'; + } + return value; +} + +function requestedPlatforms(task) { + const payload = task.payload || {}; + const raw = Array.isArray(payload.platforms) && payload.platforms.length > 0 + ? payload.platforms + : ['douyin', 'xiaohongshu']; + return raw.map(normalizePlatform).filter(platform => PLATFORM_HANDLERS[platform]); +} + +function accountIdFor(task, platform) { + const payload = task.payload || {}; + const accountIds = payload.accountIds || {}; + if (accountIds && typeof accountIds === 'object' && accountIds[platform]) { + return accountIds[platform]; + } + return PLATFORM_HANDLERS[platform].accountId; +} + +function payloadForPlatform(task, platform) { + const payload = task.payload || {}; + const platformPayloads = payload.platformPayloads || {}; + return { + ...payload, + ...(platformPayloads[platform] || {}), + platforms: undefined, + accountIds: undefined, + platformPayloads: undefined + }; +} + +async function pollTasks() { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify({ + workerId: DEFAULT_WORKER_ID, + platforms: ['multi', 'matrix', 'all'], + maxTasks: 3 + }) + }); + if (!response.ok) { + throw new Error(`Poll failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function submitResult(taskId, body) { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw new Error(`Submit result failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function handleTask(task) { + const platforms = requestedPlatforms(task); + if (platforms.length === 0) { + return submitResult(task.id, { + status: 'FAILED', + errorMessage: 'No supported platforms requested.', + result: { action: 'sync_metrics', platform: 'multi', platformResults: {} } + }); + } + + const platformResults = {}; + const videos = []; + const metrics = emptyMetrics(); + const errors = []; + + for (const platform of platforms) { + const handler = PLATFORM_HANDLERS[platform]; + const platformTask = { + ...task, + platform, + accountId: accountIdFor(task, platform), + payload: payloadForPlatform(task, platform) + }; + try { + const response = await handler.collectMetrics(platformTask); + const result = response.result || {}; + platformResults[platform] = { + ...result, + platform, + accountId: platformTask.accountId, + status: response.status + }; + addMetrics(metrics, result.metrics || {}); + for (const video of result.videos || []) { + videos.push({ ...video, platform, accountId: platformTask.accountId }); + } + if (response.status !== 'SUCCESS') { + errors.push(`${platform}: ${response.errorMessage || 'failed'}`); + } + } catch (error) { + platformResults[platform] = { + platform, + accountId: platformTask.accountId, + status: 'FAILED', + errorMessage: error.message + }; + errors.push(`${platform}: ${error.message}`); + } + } + + return submitResult(task.id, { + status: errors.length === platforms.length ? 'FAILED' : 'SUCCESS', + errorMessage: errors.length > 0 ? errors.join('; ') : null, + result: { + action: 'sync_metrics', + platform: 'multi', + platforms, + metrics, + videos, + videoCount: videos.length, + platformResults, + extractionMode: 'multi_platform' + } + }); +} + +function emptyMetrics() { + return { views: 0, likes: 0, comments: 0, collects: 0, shares: 0 }; +} + +function addMetrics(target, source) { + target.views += Number(source.views || 0); + target.likes += Number(source.likes || 0); + target.comments += Number(source.comments || 0); + target.collects += Number(source.collects || 0); + target.shares += Number(source.shares || 0); +} + +async function workerOnce() { + const tasks = await pollTasks(); + if (!Array.isArray(tasks) || tasks.length === 0) { + console.log('No pending media matrix tasks.'); + return; + } + for (const task of tasks) { + const result = await handleTask(task); + console.log(`Completed ${task.id}: ${result.status}`); + } +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function workerLoop() { + const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000); + console.log(`Media matrix worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`); + while (true) { + try { + await workerOnce(); + } catch (error) { + console.error(error.stack || error.message); + } + await sleep(intervalMs); + } +} + +async function main() { + const mode = process.argv[2]; + if (mode === 'worker') { + await workerOnce(); + return; + } + if (mode === 'worker-loop') { + await workerLoop(); + return; + } + throw new Error('Usage: node scripts/local_media_matrix_worker.js '); +} + +main().catch(error => { + console.error(error.stack || error.message); + process.exit(1); +}); diff --git a/scripts/local_xiaohongshu_worker.js b/scripts/local_xiaohongshu_worker.js new file mode 100644 index 0000000..ab1b07d --- /dev/null +++ b/scripts/local_xiaohongshu_worker.js @@ -0,0 +1,517 @@ +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); + +const DEFAULT_BASE_URL = process.env.MEDIA_BASE_URL || 'http://localhost:8880'; +const DEFAULT_WORKER_ID = process.env.MEDIA_WORKER_ID || 'local-xiaohongshu-worker'; +const DEFAULT_WORKER_TOKEN = process.env.MEDIA_WORKER_TOKEN || 'local-worker-dev-token'; +const DEFAULT_ACCOUNT_ID = process.env.XHS_ACCOUNT_ID || 'xiaohongshu_company'; +const DEFAULT_CREATOR_URL = process.env.XHS_CREATOR_URL || 'https://creator.xiaohongshu.com'; +const DEFAULT_CONTENT_URL = process.env.XHS_CONTENT_URL || 'https://creator.xiaohongshu.com'; +const DEFAULT_PROFILE_ROOT = process.env.XHS_PROFILE_ROOT || path.resolve('data/platform-profiles'); + +const LOGIN_TEXT = new RegExp([ + '\\u626b\\u7801\\u767b\\u5f55', + '\\u9a8c\\u8bc1\\u7801', + '\\u624b\\u673a\\u53f7', + '\\u767b\\u5f55', + '\\u6ce8\\u518c' +].join('|')); + +const CREATOR_TEXT = new RegExp([ + '\\u5c0f\\u7ea2\\u4e66', + '\\u521b\\u4f5c\\u670d\\u52a1', + '\\u7b14\\u8bb0', + '\\u6570\\u636e', + '\\u4f5c\\u54c1', + '\\u53d1\\u5e03' +].join('|')); + +function requirePlaywright() { + try { + return require('playwright'); + } catch (error) { + throw new Error('Playwright is not installed. Run npm.cmd install in the project root first.'); + } +} + +function profileDir(accountId = DEFAULT_ACCOUNT_ID) { + return path.resolve(DEFAULT_PROFILE_ROOT, accountId); +} + +function ensureDir(dir) { + fs.mkdirSync(dir, { recursive: true }); +} + +function browserChannel() { + if (process.env.XHS_BROWSER_CHANNEL) { + return process.env.XHS_BROWSER_CHANNEL; + } + return process.platform === 'win32' ? 'msedge' : undefined; +} + +async function launchContext({ accountId = DEFAULT_ACCOUNT_ID, headless = false } = {}) { + const { chromium } = requirePlaywright(); + const dir = profileDir(accountId); + ensureDir(dir); + const channel = browserChannel(); + const options = { + headless, + viewport: { width: 1440, height: 1000 }, + acceptDownloads: true, + args: [ + '--disable-blink-features=AutomationControlled', + '--disable-dev-shm-usage' + ] + }; + if (channel) { + options.channel = channel; + } + return chromium.launchPersistentContext(dir, options); +} + +async function waitForEnter(message) { + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + return new Promise(resolve => { + rl.question(message, () => { + rl.close(); + resolve(); + }); + }); +} + +async function login() { + const accountId = process.argv[3] || DEFAULT_ACCOUNT_ID; + const context = await launchContext({ accountId, headless: false }); + const page = context.pages()[0] || await context.newPage(); + await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); + console.log(`Xiaohongshu login profile: ${profileDir(accountId)}`); + console.log('Please login in the opened browser. Keep this window open until login succeeds.'); + await waitForEnter('Press Enter here after Xiaohongshu creator center is logged in...'); + const status = await detectLogin(page, accountId); + await context.close(); + console.log(JSON.stringify(status, null, 2)); +} + +async function checkLogin(accountId = DEFAULT_ACCOUNT_ID) { + const context = await launchContext({ accountId, headless: true }); + const page = context.pages()[0] || await context.newPage(); + try { + await page.goto(DEFAULT_CREATOR_URL, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.waitForTimeout(2500); + return await detectLogin(page, accountId); + } finally { + await context.close(); + } +} + +async function detectLogin(page, accountId = DEFAULT_ACCOUNT_ID) { + const url = page.url(); + const title = await page.title().catch(() => ''); + const bodyText = await page.locator('body').innerText({ timeout: 5000 }).catch(() => ''); + const loginVisible = LOGIN_TEXT.test(bodyText); + const creatorVisible = CREATOR_TEXT.test(bodyText); + const loggedIn = creatorVisible || (!loginVisible && /xiaohongshu\.com/.test(url)); + return { + loggedIn, + status: loggedIn ? 'SUCCESS' : 'FAILED', + message: loggedIn ? 'Xiaohongshu creator profile is logged in.' : 'Xiaohongshu creator profile is not logged in.', + url, + title, + profileDir: profileDir(accountId) + }; +} + +async function collectMetrics(task) { + const payload = task.payload || {}; + const accountId = task.accountId || DEFAULT_ACCOUNT_ID; + const targetUrl = payload.metricsUrl || payload.contentUrl || DEFAULT_CONTENT_URL; + const headless = payload.headless === false ? false : true; + const context = await launchContext({ accountId, headless }); + const page = context.pages()[0] || await context.newPage(); + try { + await page.goto(targetUrl, { waitUntil: 'domcontentloaded', timeout: 60000 }); + await page.waitForTimeout(Number(payload.waitMs || 6000)); + const status = await detectLogin(page, accountId); + if (!status.loggedIn) { + return { + status: 'FAILED', + errorMessage: status.message, + result: { + action: 'sync_metrics', + platform: 'xiaohongshu', + loggedIn: false, + ...status + } + }; + } + + const text = await page.locator('body').innerText({ timeout: 15000 }).catch(() => ''); + const screenshot = await saveScreenshot(page, task.id); + const videos = await collectVideoMetrics(page, Number(payload.maxVideos || 20)); + const metrics = videos.length > 0 ? summarizeVideos(videos) : extractMetrics(text); + return { + status: 'SUCCESS', + errorMessage: null, + result: { + action: 'sync_metrics', + platform: 'xiaohongshu', + loggedIn: true, + url: page.url(), + title: await page.title().catch(() => ''), + metrics, + videos, + videoCount: videos.length, + extractionMode: videos.length > 0 ? 'video_list' : 'page_summary', + screenshot, + pageTextPreview: compactText(text).slice(0, 2000) + } + }; + } finally { + await context.close().catch(() => {}); + } +} + +async function saveScreenshot(page, taskId) { + const dir = path.resolve('target/media/xiaohongshu'); + ensureDir(dir); + const file = path.join(dir, `${taskId || Date.now()}-metrics.png`); + await page.screenshot({ path: file, fullPage: true }).catch(() => {}); + return file; +} + +function compactText(text) { + return (text || '').replace(/\s+/g, ' ').trim(); +} + +function extractMetrics(text) { + const compact = compactText(text); + return { + views: findMetric(compact, ['\\u6d4f\\u89c8', '\\u89c2\\u770b', '\\u9605\\u8bfb', '\\u66dd\\u5149']), + likes: findMetric(compact, ['\\u70b9\\u8d5e', '\\u8d5e']), + comments: findMetric(compact, ['\\u8bc4\\u8bba']), + collects: findMetric(compact, ['\\u6536\\u85cf']), + shares: findMetric(compact, ['\\u5206\\u4eab', '\\u8f6c\\u53d1']) + }; +} + +async function collectVideoMetrics(page, limit = 20) { + await autoScroll(page); + const blocks = await page.evaluate(maxBlocks => { + const selectors = [ + 'tr', + 'article', + 'li', + '[class*="note"]', + '[class*="card"]', + '[class*="item"]', + '[class*="work"]', + '[class*="video"]', + '[class*="content"]' + ]; + const seen = new Set(); + const rows = []; + for (const selector of selectors) { + for (const element of document.querySelectorAll(selector)) { + if (rows.length >= maxBlocks) { + break; + } + const rect = element.getBoundingClientRect(); + if (rect.width < 140 || rect.height < 40) { + continue; + } + const text = (element.innerText || '').replace(/\s+/g, ' ').trim(); + if (text.length < 10 || text.length > 1200) { + continue; + } + const signature = text.slice(0, 120); + if (seen.has(signature)) { + continue; + } + seen.add(signature); + const link = element.querySelector('a[href]'); + rows.push({ + text, + href: link ? link.href : '' + }); + } + } + return rows; + }, Math.max(limit * 8, 80)); + + const videos = []; + const seen = new Set(); + for (const block of blocks) { + const video = parseVideoBlock(block); + if (!video) { + continue; + } + const key = `${video.title}|${video.views}|${video.likes}|${video.comments}|${video.href}`; + if (seen.has(key)) { + continue; + } + seen.add(key); + videos.push(video); + if (videos.length >= limit) { + break; + } + } + return videos; +} + +async function autoScroll(page) { + await page.evaluate(async () => { + await new Promise(resolve => { + let count = 0; + const timer = setInterval(() => { + window.scrollBy(0, Math.floor(window.innerHeight * 0.8)); + count += 1; + if (count >= 3) { + clearInterval(timer); + resolve(); + } + }, 500); + }); + }).catch(() => {}); + await page.waitForTimeout(800).catch(() => {}); +} + +function parseVideoBlock(block) { + const text = compactText(block.text); + const metrics = extractMetrics(text); + const score = metrics.views + metrics.likes + metrics.comments + metrics.collects + metrics.shares; + if (score <= 0) { + return null; + } + const title = extractTitle(text); + return { + title, + href: block.href || '', + views: metrics.views, + likes: metrics.likes, + comments: metrics.comments, + collects: metrics.collects, + shares: metrics.shares, + interactionCount: metrics.likes + metrics.comments + metrics.collects + metrics.shares, + interactionRate: metrics.views <= 0 + ? 0 + : (metrics.likes + metrics.comments + metrics.collects + metrics.shares) / metrics.views, + rawTextPreview: text.slice(0, 500) + }; +} + +function extractTitle(text) { + const cleaned = text + .replace(/\d{4}[-/.]\d{1,2}[-/.]\d{1,2}.*/g, '') + .replace(/(\u6d4f\u89c8|\u89c2\u770b|\u9605\u8bfb|\u66dd\u5149|\u70b9\u8d5e|\u8d5e|\u8bc4\u8bba|\u6536\u85cf|\u5206\u4eab|\u8f6c\u53d1).*/g, '') + .trim(); + if (cleaned.length >= 2) { + return cleaned.slice(0, 120); + } + return text.slice(0, 80); +} + +function summarizeVideos(videos) { + const summary = { + views: 0, + likes: 0, + comments: 0, + collects: 0, + shares: 0 + }; + for (const video of videos) { + summary.views += Number(video.views || 0); + summary.likes += Number(video.likes || 0); + summary.comments += Number(video.comments || 0); + summary.collects += Number(video.collects || 0); + summary.shares += Number(video.shares || 0); + } + return summary; +} + +function findMetric(text, labels) { + for (const label of labels) { + const labelPattern = new RegExp(label); + const decodedLabel = label.replace(/\\u([0-9a-fA-F]{4})/g, (_, code) => String.fromCharCode(parseInt(code, 16))); + const after = new RegExp(`${label}[^0-9\\d]{0,16}([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)`); + const before = new RegExp(`([0-9][0-9,.]*)(\\s*[\\u4e07\\u5343wWkK]?)[^0-9\\d]{0,16}${label}`); + if (labelPattern.test(text)) { + const afterMatch = text.match(after); + if (afterMatch) { + return parseMetricNumber(afterMatch[1], afterMatch[2]); + } + const beforeMatch = text.match(before); + if (beforeMatch) { + return parseMetricNumber(beforeMatch[1], beforeMatch[2]); + } + const looseIndex = text.indexOf(decodedLabel); + if (looseIndex >= 0) { + const around = text.slice(Math.max(0, looseIndex - 30), looseIndex + 60); + const number = around.match(/([0-9][0-9,.]*)(\s*[\u4e07\u5343wWkK]?)/); + if (number) { + return parseMetricNumber(number[1], number[2]); + } + } + } + } + return 0; +} + +function parseMetricNumber(raw, unit = '') { + const value = Number(String(raw).replace(/,/g, '')); + if (!Number.isFinite(value)) { + return 0; + } + const normalizedUnit = unit.trim().toLowerCase(); + if (normalizedUnit.includes('\u4e07') || normalizedUnit.includes('w')) { + return Math.round(value * 10000); + } + if (normalizedUnit.includes('\u5343') || normalizedUnit.includes('k')) { + return Math.round(value * 1000); + } + return Math.round(value); +} + +async function pollTasks() { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/poll`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify({ + workerId: DEFAULT_WORKER_ID, + platforms: ['xiaohongshu'], + maxTasks: 5 + }) + }); + if (!response.ok) { + throw new Error(`Poll failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function submitResult(taskId, body) { + const response = await fetch(`${DEFAULT_BASE_URL}/v1/media/workers/tasks/${taskId}/result`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Worker-Token': DEFAULT_WORKER_TOKEN + }, + body: JSON.stringify(body) + }); + if (!response.ok) { + throw new Error(`Submit result failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function handleTask(task) { + const payload = task.payload || {}; + const action = payload.action || (task.taskType === 'SYNC_METRICS' ? 'sync_metrics' : 'check_login'); + if (action === 'check_login') { + const status = await checkLogin(task.accountId || DEFAULT_ACCOUNT_ID); + return submitResult(task.id, { + status: status.loggedIn ? 'SUCCESS' : 'FAILED', + errorMessage: status.loggedIn ? null : status.message, + result: { + action, + platform: 'xiaohongshu', + loggedIn: status.loggedIn, + message: status.message, + url: status.url, + title: status.title, + profileDir: profileDir(task.accountId || DEFAULT_ACCOUNT_ID) + } + }); + } + + if (action === 'sync_metrics') { + const result = await collectMetrics(task); + return submitResult(task.id, result); + } + + return submitResult(task.id, { + status: 'FAILED', + errorMessage: `Unsupported local Xiaohongshu worker action: ${action}`, + result: { action, platform: 'xiaohongshu' } + }); +} + +async function workerOnce() { + const tasks = await pollTasks(); + if (!Array.isArray(tasks) || tasks.length === 0) { + console.log('No pending Xiaohongshu tasks.'); + return; + } + for (const task of tasks) { + const result = await handleTask(task); + console.log(`Completed ${task.id}: ${result.status}`); + } +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function workerLoop() { + const intervalMs = Number(process.env.MEDIA_WORKER_INTERVAL_MS || 30000); + console.log(`Xiaohongshu worker loop started. baseUrl=${DEFAULT_BASE_URL}, intervalMs=${intervalMs}`); + while (true) { + try { + await workerOnce(); + } catch (error) { + console.error(error.stack || error.message); + } + await sleep(intervalMs); + } +} + +async function main() { + const mode = process.argv[2]; + if (mode === 'login') { + await login(); + return; + } + if (mode === 'check-login') { + const status = await checkLogin(process.argv[3] || DEFAULT_ACCOUNT_ID); + console.log(JSON.stringify(status, null, 2)); + process.exit(status.loggedIn ? 0 : 2); + } + if (mode === 'metrics') { + const task = { + id: `manual_${Date.now()}`, + taskType: 'SYNC_METRICS', + platform: 'xiaohongshu', + accountId: process.argv[3] || DEFAULT_ACCOUNT_ID, + payload: { action: 'sync_metrics', headless: false } + }; + console.log(JSON.stringify((await collectMetrics(task)).result, null, 2)); + return; + } + if (mode === 'worker') { + await workerOnce(); + return; + } + if (mode === 'worker-loop') { + await workerLoop(); + return; + } + throw new Error('Usage: node scripts/local_xiaohongshu_worker.js [accountId]'); +} + +module.exports = { + checkLogin, + collectMetrics, + login, + profileDir, + workerLoop, + workerOnce +}; + +if (require.main === module) { + main().catch(error => { + console.error(error.stack || error.message); + process.exit(1); + }); +} diff --git a/scripts/start_local_client.ps1 b/scripts/start_local_client.ps1 index d20221f..7c8590f 100644 --- a/scripts/start_local_client.ps1 +++ b/scripts/start_local_client.ps1 @@ -1,8 +1,10 @@ param( [string]$BaseUrl = "http://localhost:8880", [string]$AccountId = "douyin_company", + [string]$XhsAccountId = "xiaohongshu_company", [int]$WorkerIntervalSeconds = 30, - [switch]$RemoteBackend + [switch]$RemoteBackend, + [string]$Platforms = "douyin,xiaohongshu" ) $ErrorActionPreference = "Stop" @@ -115,25 +117,62 @@ if ($RemoteBackend -or -not (Test-LocalBaseUrl $BaseUrl)) { Write-Step "Checking Douyin login" $env:MEDIA_BASE_URL = $BaseUrl $env:DOUYIN_ACCOUNT_ID = $AccountId -$checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host $checkOutput - Write-Host "" - Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window." - & npm.cmd run douyin:login -- $AccountId +$platformList = $Platforms.Split(",") | ForEach-Object { $_.Trim().ToLower() } | Where-Object { $_ } + +if ($platformList -contains "douyin") { + $checkOutput = & npm.cmd run douyin:check -- $AccountId 2>&1 if ($LASTEXITCODE -ne 0) { - throw "Douyin login failed." + Write-Host $checkOutput + Write-Host "" + Write-Host "Douyin login is required. A browser will open. Login, then press Enter in that window." + & npm.cmd run douyin:login -- $AccountId + if ($LASTEXITCODE -ne 0) { + throw "Douyin login failed." + } + } else { + Write-Host "Douyin login profile is ready." } -} else { - Write-Host "Douyin login profile is ready." } -Write-Step "Starting Douyin worker loop" -$workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop" -Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal +if ($platformList -contains "xiaohongshu") { + Write-Step "Checking Xiaohongshu login" + $env:XHS_ACCOUNT_ID = $XhsAccountId + $xhsCheckOutput = & npm.cmd run xhs:check -- $XhsAccountId 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Host $xhsCheckOutput + Write-Host "" + Write-Host "Xiaohongshu login is required. A browser will open. Login, then press Enter in that window." + & npm.cmd run xhs:login -- $XhsAccountId + if ($LASTEXITCODE -ne 0) { + throw "Xiaohongshu login failed." + } + } else { + Write-Host "Xiaohongshu login profile is ready." + } +} + +if ($platformList -contains "douyin") { + Write-Step "Starting Douyin worker loop" + $workerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run douyin:worker:loop" + Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) -WindowStyle Normal +} + +if ($platformList -contains "xiaohongshu") { + Write-Step "Starting Xiaohongshu worker loop" + $xhsWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run xhs:worker:loop" + Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $xhsWorkerCommand) -WindowStyle Normal +} + +if (($platformList -contains "douyin") -and ($platformList -contains "xiaohongshu")) { + Write-Step "Starting media matrix worker loop" + $matrixWorkerCommand = "Set-Location '$Root'; `$env:MEDIA_BASE_URL='$BaseUrl'; `$env:DOUYIN_ACCOUNT_ID='$AccountId'; `$env:XHS_ACCOUNT_ID='$XhsAccountId'; `$env:MEDIA_WORKER_INTERVAL_MS='$($WorkerIntervalSeconds * 1000)'; npm.cmd run media:worker:loop" + Start-Process powershell.exe -ArgumentList @("-NoExit", "-ExecutionPolicy", "Bypass", "-Command", $matrixWorkerCommand) -WindowStyle Normal +} Write-Host "" Write-Host "Local client is ready." Write-Host "Backend: $BaseUrl" -Write-Host "Account: $AccountId" +Write-Host "Douyin account: $AccountId" +Write-Host "Xiaohongshu account: $XhsAccountId" +Write-Host "Platforms: $($platformList -join ', ')" Write-Host "Worker polling interval: $WorkerIntervalSeconds seconds" diff --git a/src/main/java/com/company/agentfoundation/media/model/MediaTask.java b/src/main/java/com/company/agentfoundation/media/model/MediaTask.java index 5c7c25b..f945dad 100644 --- a/src/main/java/com/company/agentfoundation/media/model/MediaTask.java +++ b/src/main/java/com/company/agentfoundation/media/model/MediaTask.java @@ -67,48 +67,47 @@ public class MediaTask { long shares = 0; List> rows = new ArrayList<>(); List> videoRows = new ArrayList<>(); + Map byPlatform = new LinkedHashMap<>(); for (MediaTask task : tasks) { if (task.status != MediaTaskStatus.SUCCESS || task.result == null) { continue; } + Map platformResults = mapValue(task.result.get("platformResults")); + if (!platformResults.isEmpty()) { + for (Map.Entry entry : platformResults.entrySet()) { + Map platformResult = mapValue(entry.getValue()); + Map metrics = mapValue(platformResult.get("metrics")); + if (metrics.isEmpty()) { + continue; + } + MetricValues values = metricValues(metrics); + views += values.views; + likes += values.likes; + comments += values.comments; + collects += values.collects; + shares += values.shares; + String platform = entry.getKey(); + String accountId = stringValue(platformResult.get("accountId")); + byPlatform.computeIfAbsent(platform, ignored -> new MetricBucket()).add(values); + rows.add(metricRow(task.id, platform, accountId, values)); + addVideoRows(videoRows, task, platform, accountId, listOfMaps(platformResult.get("videos"))); + } + continue; + } Map metrics = mapValue(task.result.get("metrics")); if (metrics.isEmpty()) { continue; } - long taskViews = longValue(metrics.get("views")); - long taskLikes = longValue(metrics.get("likes")); - long taskComments = longValue(metrics.get("comments")); - long taskCollects = longValue(metrics.get("collects")); - long taskShares = longValue(metrics.get("shares")); - views += taskViews; - likes += taskLikes; - comments += taskComments; - collects += taskCollects; - shares += taskShares; - rows.add(Map.of( - "taskId", task.id, - "platform", task.platform, - "accountId", task.accountId, - "views", taskViews, - "likes", taskLikes, - "comments", taskComments, - "collects", taskCollects, - "shares", taskShares - )); - for (Map video : listOfMaps(task.result.get("videos"))) { - videoRows.add(Map.of( - "taskId", task.id, - "platform", task.platform, - "accountId", task.accountId, - "title", stringValue(video.get("title")), - "href", stringValue(video.get("href")), - "views", longValue(video.get("views")), - "likes", longValue(video.get("likes")), - "comments", longValue(video.get("comments")), - "collects", longValue(video.get("collects")), - "shares", longValue(video.get("shares")) - )); - } + MetricValues values = metricValues(metrics); + views += values.views; + likes += values.likes; + comments += values.comments; + collects += values.collects; + shares += values.shares; + byPlatform.computeIfAbsent(task.platform, ignored -> new MetricBucket()) + .add(values); + rows.add(metricRow(task.id, task.platform, task.accountId, values)); + addVideoRows(videoRows, task, task.platform, task.accountId, listOfMaps(task.result.get("videos"))); } Map summary = new LinkedHashMap<>(); summary.put("views", views); @@ -121,6 +120,7 @@ public class MediaTask { summary.put("rows", rows); summary.put("videoCount", videoRows.size()); summary.put("videoRows", videoRows); + summary.put("byPlatform", platformSummary(byPlatform)); return summary; } @@ -159,4 +159,96 @@ public class MediaTask { private static String stringValue(Object value) { return value == null ? "" : String.valueOf(value); } + + private static MetricValues metricValues(Map metrics) { + return new MetricValues( + longValue(metrics.get("views")), + longValue(metrics.get("likes")), + longValue(metrics.get("comments")), + longValue(metrics.get("collects")), + longValue(metrics.get("shares")) + ); + } + + private static Map metricRow(String taskId, String platform, String accountId, MetricValues values) { + return Map.of( + "taskId", taskId, + "platform", platform, + "accountId", accountId, + "views", values.views, + "likes", values.likes, + "comments", values.comments, + "collects", values.collects, + "shares", values.shares + ); + } + + private static void addVideoRows( + List> videoRows, + MediaTask task, + String platform, + String accountId, + List> videos + ) { + for (Map video : videos) { + videoRows.add(Map.of( + "taskId", task.id, + "platform", platform, + "accountId", accountId, + "title", stringValue(video.get("title")), + "href", stringValue(video.get("href")), + "views", longValue(video.get("views")), + "likes", longValue(video.get("likes")), + "comments", longValue(video.get("comments")), + "collects", longValue(video.get("collects")), + "shares", longValue(video.get("shares")) + )); + } + } + + private static Map platformSummary(Map buckets) { + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : buckets.entrySet()) { + result.put(entry.getKey(), entry.getValue().toMap()); + } + return result; + } + + private static class MetricBucket { + private long views; + private long likes; + private long comments; + private long collects; + private long shares; + + private void add(MetricValues values) { + this.views += values.views; + this.likes += values.likes; + this.comments += values.comments; + this.collects += values.collects; + this.shares += values.shares; + } + + private Map toMap() { + long interactionCount = likes + comments + collects + shares; + Map map = new LinkedHashMap<>(); + map.put("views", views); + map.put("likes", likes); + map.put("comments", comments); + map.put("collects", collects); + map.put("shares", shares); + map.put("interactionCount", interactionCount); + map.put("interactionRate", views <= 0 ? 0.0 : interactionCount * 1.0 / views); + return map; + } + } + + private record MetricValues( + long views, + long likes, + long comments, + long collects, + long shares + ) { + } } diff --git a/start_yuedong_client.bat b/start_yuedong_client.bat index df335ef..863c5a0 100644 --- a/start_yuedong_client.bat +++ b/start_yuedong_client.bat @@ -1,5 +1,5 @@ @echo off setlocal cd /d "%~dp0" -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "%~dp0scripts\start_local_client.ps1" -BaseUrl "http://agent.yuedongsports.com" -RemoteBackend -Platforms "douyin,xiaohongshu" pause