画面优化
This commit is contained in:
parent
e74900bcb0
commit
fefb6b1d3c
@ -198,10 +198,15 @@ export const createBaiduAsrMiddleware = () => async (
|
||||
|
||||
try {
|
||||
const audio = await readAudioBody(req);
|
||||
const requestStartedAt = Date.now();
|
||||
const audioDurationMs = Math.round(audio.length / 2 / 16_000 * 1000);
|
||||
console.info(`[Baidu ASR] start bytes=${audio.length} durationMs=${audioDurationMs}`);
|
||||
const text = await recognizeAudio(audio);
|
||||
console.info(`[Baidu ASR] success durationMs=${audioDurationMs} elapsedMs=${Date.now() - requestStartedAt}`);
|
||||
sendJson(res, 200, {text});
|
||||
} catch (error) {
|
||||
if (error instanceof HttpError) {
|
||||
console.warn(`[Baidu ASR] failed status=${error.statusCode} message=${error.message}`);
|
||||
sendJson(res, error.statusCode, {error: error.message});
|
||||
return;
|
||||
}
|
||||
|
||||
188
src/App.vue
188
src/App.vue
@ -2,7 +2,15 @@
|
||||
<main class="realtime-dh-wrapper">
|
||||
<section class="realtime-dh-stage" @pointerdown.capture.once="unlockAudioAndPlay">
|
||||
<div class="visual-stage">
|
||||
<div class="standby-scene" :class="{thinking: waitingForAnswer}">
|
||||
<div
|
||||
class="standby-scene"
|
||||
:class="{
|
||||
thinking: waitingForAnswer,
|
||||
ready: standbyVisualReady,
|
||||
'human-mounted': showHuman,
|
||||
'human-active': humanVisualReady
|
||||
}"
|
||||
>
|
||||
<video
|
||||
v-for="videoIndex in 2"
|
||||
:key="videoIndex"
|
||||
@ -19,6 +27,7 @@
|
||||
playsinline
|
||||
preload="auto"
|
||||
disablepictureinpicture
|
||||
@loadeddata="handleStandbyLoaded(videoIndex - 1)"
|
||||
@timeupdate="handleStandbyProgress(videoIndex - 1)"
|
||||
@ended="handleStandbyEnded(videoIndex - 1)"
|
||||
/>
|
||||
@ -104,6 +113,8 @@
|
||||
ref="voiceRenderRef"
|
||||
:speech-busy="speechBusy"
|
||||
:playback-pending="playbackPending"
|
||||
@speech-detected="handleSpeechDetected"
|
||||
@speech-aborted="handleSpeechAborted"
|
||||
@question-updated="handleQuestionUpdated"
|
||||
@question-recognized="handleQuestionRecognized"
|
||||
@answer-ready="handleAnswerReady"
|
||||
@ -131,9 +142,10 @@ import {checkPlayUnMute} from './utils/audioUtils';
|
||||
|
||||
const ReadyState = {UNINSTANTIATED: -1, CONNECTING: 0, OPEN: 1, CLOSING: 2, CLOSED: 3};
|
||||
const TOOLBAR_HIDE_DELAY_MS = 3_000;
|
||||
const POST_SPEECH_DISCONNECT_DELAY_MS = 10_000;
|
||||
const HUMAN_SCENE_FADE_OUT_MS = 920;
|
||||
const STANDBY_CROSSFADE_MS = 650;
|
||||
const HUMAN_SCENE_FADE_OUT_MS = 80;
|
||||
const HUMAN_SCENE_REVEAL_DELAY_MS = 650;
|
||||
const STANDBY_CROSSFADE_MS = 760;
|
||||
const AUDIO_RESTORE_DELAYS_MS = [0, 180, 600, 1_200];
|
||||
const dhIframe = new DHIframe('digital-human-iframe');
|
||||
const realTimeVideoReady = ref(false);
|
||||
const humanVisualReady = ref(false);
|
||||
@ -157,18 +169,20 @@ const voiceRenderRef = ref(null);
|
||||
const recognizedQuestion = ref('');
|
||||
const deepSeekAnswer = ref('');
|
||||
const waitingForAnswer = ref(false);
|
||||
const standbyVisualReady = ref(false);
|
||||
const standbyActiveVideo = ref(0);
|
||||
const standbyIncomingVideo = ref(-1);
|
||||
const standbyVideos = [null, null];
|
||||
let connectionGeneration = 0;
|
||||
let toolbarHideTimer = 0;
|
||||
let postSpeechDisconnectTimer = 0;
|
||||
let humanVisualTimer = 0;
|
||||
let humanReleaseTimer = 0;
|
||||
let standbyCrossfadeTimer = 0;
|
||||
let standbyCrossfadeRunning = false;
|
||||
let audioRestoreTimers = [];
|
||||
let videoPlaybackStartedGeneration = -1;
|
||||
|
||||
const humanReady = computed(() => realTimeVideoReady.value && humanVisualReady.value && wsConnected.value && showHuman.value);
|
||||
const humanReady = computed(() => realTimeVideoReady.value && wsConnected.value && showHuman.value);
|
||||
const connectionState = computed(() => !showHuman.value ? 'disconnected' : (humanReady.value ? 'connected' : 'connecting'));
|
||||
const connectionLabel = computed(() => ({
|
||||
disconnected: '已断开 · 不计费', connecting: '连接中', connected: '已连接'
|
||||
@ -178,6 +192,9 @@ const playbackPending = computed(() => Boolean(pendingSpeechText.value) || playb
|
||||
const setStandbyVideoRef = (element, index) => {
|
||||
standbyVideos[index] = element;
|
||||
};
|
||||
const handleStandbyLoaded = (index) => {
|
||||
if (index === standbyActiveVideo.value) standbyVisualReady.value = true;
|
||||
};
|
||||
const finishStandbyCrossfade = (outgoingIndex, incomingIndex) => {
|
||||
const outgoing = standbyVideos[outgoingIndex];
|
||||
standbyActiveVideo.value = incomingIndex;
|
||||
@ -244,6 +261,52 @@ const showToolbar = () => {
|
||||
toolbarVisible.value = true;
|
||||
scheduleToolbarHide();
|
||||
};
|
||||
const syncAudioUnlockFromBrowser = () => {
|
||||
if (soundEnabled.value && navigator.userActivation?.hasBeenActive) {
|
||||
audioUnlocked.value = true;
|
||||
soundUnlockTip.value = false;
|
||||
}
|
||||
};
|
||||
const clearAudioRestoreTimers = () => {
|
||||
audioRestoreTimers.forEach(timer => window.clearTimeout(timer));
|
||||
audioRestoreTimers = [];
|
||||
};
|
||||
const startVideoPlaybackOnce = () => {
|
||||
if (!showHuman.value || !realTimeVideoReady.value || !wsConnected.value
|
||||
|| !soundEnabled.value || !audioUnlocked.value
|
||||
|| videoPlaybackStartedGeneration === connectionGeneration) return;
|
||||
dhIframe.playVideo(true);
|
||||
videoPlaybackStartedGeneration = connectionGeneration;
|
||||
};
|
||||
const scheduleAudioRestore = () => {
|
||||
clearAudioRestoreTimers();
|
||||
if (!showHuman.value || !soundEnabled.value || !audioUnlocked.value) return;
|
||||
startVideoPlaybackOnce();
|
||||
const generation = connectionGeneration;
|
||||
audioRestoreTimers = AUDIO_RESTORE_DELAYS_MS.map(delay => window.setTimeout(() => {
|
||||
if (generation !== connectionGeneration || !showHuman.value
|
||||
|| !soundEnabled.value || !audioUnlocked.value) return;
|
||||
dhIframe.muteAudio(false);
|
||||
// 官方事件只会上报初始静音，不会上报解除静音；命令发出后以会话状态为准。
|
||||
videoIsMuted.value = false;
|
||||
if (delay === AUDIO_RESTORE_DELAYS_MS.at(-1)) void tryPlayPendingAnswer();
|
||||
}, delay));
|
||||
};
|
||||
const scheduleHumanReveal = (delay = HUMAN_SCENE_REVEAL_DELAY_MS) => {
|
||||
if (humanVisualTimer) {
|
||||
window.clearTimeout(humanVisualTimer);
|
||||
humanVisualTimer = 0;
|
||||
}
|
||||
if (!showHuman.value || !realTimeVideoReady.value || !wsConnected.value) return;
|
||||
const generation = connectionGeneration;
|
||||
humanVisualTimer = window.setTimeout(() => {
|
||||
humanVisualTimer = 0;
|
||||
if (generation === connectionGeneration && showHuman.value
|
||||
&& realTimeVideoReady.value && wsConnected.value) {
|
||||
humanVisualReady.value = true;
|
||||
}
|
||||
}, delay);
|
||||
};
|
||||
const cancelHumanRelease = () => {
|
||||
if (humanReleaseTimer) {
|
||||
window.clearTimeout(humanReleaseTimer);
|
||||
@ -251,6 +314,8 @@ const cancelHumanRelease = () => {
|
||||
}
|
||||
};
|
||||
const resetConnectionState = () => {
|
||||
clearAudioRestoreTimers();
|
||||
videoPlaybackStartedGeneration = -1;
|
||||
if (humanVisualTimer) {
|
||||
window.clearTimeout(humanVisualTimer);
|
||||
humanVisualTimer = 0;
|
||||
@ -263,8 +328,9 @@ const resetConnectionState = () => {
|
||||
activeSpeechRequestId.value = '';
|
||||
videoIsMuted.value = true;
|
||||
};
|
||||
const connectHuman = () => {
|
||||
if (showHuman.value || !pendingSpeechText.value) return;
|
||||
const connectHuman = (allowWithoutAnswer = false) => {
|
||||
if (showHuman.value || (!pendingSpeechText.value && !allowWithoutAnswer)) return;
|
||||
syncAudioUnlockFromBrowser();
|
||||
cancelHumanRelease();
|
||||
connectionGeneration += 1;
|
||||
resetConnectionState();
|
||||
@ -282,10 +348,6 @@ const finalizeHumanRelease = (preservePendingAnswer = false) => {
|
||||
showToolbar();
|
||||
};
|
||||
const releaseHuman = (preservePendingAnswer = false, smooth = false) => {
|
||||
if (postSpeechDisconnectTimer) {
|
||||
window.clearTimeout(postSpeechDisconnectTimer);
|
||||
postSpeechDisconnectTimer = 0;
|
||||
}
|
||||
cancelHumanRelease();
|
||||
if (smooth && showHuman.value && humanVisualReady.value) {
|
||||
humanVisualReady.value = false;
|
||||
@ -302,23 +364,20 @@ const disconnectHuman = () => {
|
||||
releaseHuman(false, true);
|
||||
};
|
||||
const sendSpeech = (text) => {
|
||||
if (postSpeechDisconnectTimer) {
|
||||
window.clearTimeout(postSpeechDisconnectTimer);
|
||||
postSpeechDisconnectTimer = 0;
|
||||
}
|
||||
cancelHumanRelease();
|
||||
const requestId = uuidV4();
|
||||
activeSpeechRequestId.value = requestId;
|
||||
currentSpeechText.value = text;
|
||||
speechBusy.value = true;
|
||||
dhIframe.sendMessage({action: 'TEXT_RENDER', body: text, requestId});
|
||||
scheduleAudioRestore();
|
||||
};
|
||||
const tryPlayPendingAnswer = async () => {
|
||||
if (!pendingSpeechText.value || !humanReady.value || playbackPreparing.value || speechBusy.value) return;
|
||||
playbackPreparing.value = true;
|
||||
const generation = connectionGeneration;
|
||||
try {
|
||||
if (soundEnabled.value && videoIsMuted.value) {
|
||||
if (soundEnabled.value) {
|
||||
const canPlayWithSound = audioUnlocked.value || await checkPlayUnMute();
|
||||
if (generation !== connectionGeneration || !humanReady.value) return;
|
||||
if (!canPlayWithSound) {
|
||||
@ -330,10 +389,8 @@ const tryPlayPendingAnswer = async () => {
|
||||
return;
|
||||
}
|
||||
audioUnlocked.value = true;
|
||||
dhIframe.playVideo(true);
|
||||
dhIframe.muteAudio(false);
|
||||
videoIsMuted.value = false;
|
||||
await new Promise(resolve => window.setTimeout(resolve, 240));
|
||||
scheduleAudioRestore();
|
||||
await new Promise(resolve => window.setTimeout(resolve, 140));
|
||||
} else if (!soundEnabled.value && !videoIsMuted.value) {
|
||||
dhIframe.muteAudio(true);
|
||||
videoIsMuted.value = true;
|
||||
@ -355,17 +412,25 @@ const handleAnswerReady = async (text) => {
|
||||
if (!showHuman.value) connectHuman();
|
||||
else {
|
||||
cancelHumanRelease();
|
||||
if (realTimeVideoReady.value) humanVisualReady.value = true;
|
||||
void tryPlayPendingAnswer();
|
||||
}
|
||||
};
|
||||
const handleQuestionUpdated = (text) => {
|
||||
recognizedQuestion.value = text;
|
||||
};
|
||||
const handleSpeechDetected = () => {
|
||||
cancelHumanRelease();
|
||||
// 录音期间不启动数字人 WebRTC。部分安卓浏览器在第二轮重连 WebRTC 时
|
||||
// 会切换系统音频路由，导致正在录制的麦克风数据异常并触发百度识别超时。
|
||||
// 等 DeepSeek 返回答案后再由 handleAnswerReady 建立连接，不改变画面过渡效果。
|
||||
};
|
||||
// 识别失败时保持现有连接，继续等待下一次语音输入。
|
||||
const handleSpeechAborted = () => {};
|
||||
const handleQuestionRecognized = (text) => {
|
||||
recognizedQuestion.value = text;
|
||||
deepSeekAnswer.value = '';
|
||||
waitingForAnswer.value = true;
|
||||
cancelHumanRelease();
|
||||
};
|
||||
const handleDeepSeekTimeout = () => {
|
||||
waitingForAnswer.value = false;
|
||||
@ -389,9 +454,7 @@ const unlockAudioAndPlay = () => {
|
||||
audioUnlocked.value = true;
|
||||
soundUnlockTip.value = false;
|
||||
if (showHuman.value) {
|
||||
dhIframe.playVideo(true);
|
||||
dhIframe.muteAudio(false);
|
||||
videoIsMuted.value = false;
|
||||
scheduleAudioRestore();
|
||||
window.setTimeout(() => void tryPlayPendingAnswer(), 120);
|
||||
} else if (pendingSpeechText.value) {
|
||||
connectHuman();
|
||||
@ -402,9 +465,7 @@ const enableSound = () => {
|
||||
audioUnlocked.value = true;
|
||||
soundUnlockTip.value = false;
|
||||
if (showHuman.value) {
|
||||
dhIframe.playVideo(true);
|
||||
dhIframe.muteAudio(false);
|
||||
videoIsMuted.value = false;
|
||||
scheduleAudioRestore();
|
||||
window.setTimeout(() => void tryPlayPendingAnswer(), 120);
|
||||
} else if (pendingSpeechText.value) {
|
||||
connectHuman();
|
||||
@ -421,18 +482,32 @@ const onMessage = (event) => {
|
||||
const {type, content = {}} = event.data;
|
||||
if (type === 'rtcState') {
|
||||
if (content.action === 'remoteVideoConnected') {
|
||||
// 这里只表示已获取流，画面可能仍处于白色初始化帧。
|
||||
return;
|
||||
}
|
||||
if (content.action === 'remotevideoon') {
|
||||
realTimeVideoReady.value = true;
|
||||
if (humanVisualTimer) window.clearTimeout(humanVisualTimer);
|
||||
humanVisualTimer = window.setTimeout(() => {
|
||||
scheduleAudioRestore();
|
||||
} else if (content.action === 'remotevideoloading') {
|
||||
realTimeVideoReady.value = false;
|
||||
humanVisualReady.value = false;
|
||||
if (humanVisualTimer) {
|
||||
window.clearTimeout(humanVisualTimer);
|
||||
humanVisualTimer = 0;
|
||||
if (showHuman.value && realTimeVideoReady.value) humanVisualReady.value = true;
|
||||
}, 220);
|
||||
}
|
||||
} else if (content.action === 'remoteVideoDisconnected') {
|
||||
realTimeVideoReady.value = false;
|
||||
if (humanVisualTimer) {
|
||||
window.clearTimeout(humanVisualTimer);
|
||||
humanVisualTimer = 0;
|
||||
}
|
||||
humanVisualReady.value = false;
|
||||
}
|
||||
if (content.action === 'localVideoMuted') {
|
||||
if (parseMutedState(content.body)) videoIsMuted.value = true;
|
||||
if (parseMutedState(content.body)) {
|
||||
videoIsMuted.value = true;
|
||||
if (audioUnlocked.value && soundEnabled.value) scheduleAudioRestore();
|
||||
}
|
||||
if (soundEnabled.value && videoIsMuted.value && !audioUnlocked.value
|
||||
&& (pendingSpeechText.value || speechBusy.value)) {
|
||||
soundUnlockTip.value = true;
|
||||
@ -440,17 +515,29 @@ const onMessage = (event) => {
|
||||
}
|
||||
}
|
||||
} else if (type === 'wsState') {
|
||||
if (content.readyState === ReadyState.OPEN) wsConnected.value = true;
|
||||
if (content.readyState === ReadyState.OPEN) {
|
||||
wsConnected.value = true;
|
||||
scheduleAudioRestore();
|
||||
}
|
||||
else if (content.readyState === ReadyState.CLOSED || content.readyState === ReadyState.CLOSING) {
|
||||
wsConnected.value = false;
|
||||
humanVisualReady.value = false;
|
||||
if (humanVisualTimer) {
|
||||
window.clearTimeout(humanVisualTimer);
|
||||
humanVisualTimer = 0;
|
||||
}
|
||||
speechBusy.value = false;
|
||||
activeSpeechRequestId.value = '';
|
||||
}
|
||||
} else if (type === 'msg') {
|
||||
const {action, requestId} = content;
|
||||
if (requestId === activeSpeechRequestId.value && action === 'RENDER_START') {
|
||||
scheduleAudioRestore();
|
||||
scheduleHumanReveal(HUMAN_SCENE_REVEAL_DELAY_MS);
|
||||
}
|
||||
if (requestId === activeSpeechRequestId.value && ['FINISHED', 'RENDER_ERROR', 'RENDER_INTERRUPTED'].includes(action)) {
|
||||
const replayAfterMutedPlayback = action === 'FINISHED' && soundEnabled.value
|
||||
&& videoIsMuted.value && currentSpeechText.value;
|
||||
&& !audioUnlocked.value && videoIsMuted.value && currentSpeechText.value;
|
||||
const replayText = replayAfterMutedPlayback ? currentSpeechText.value : '';
|
||||
speechBusy.value = false;
|
||||
activeSpeechRequestId.value = '';
|
||||
@ -458,33 +545,30 @@ const onMessage = (event) => {
|
||||
if (replayAfterMutedPlayback) {
|
||||
pendingSpeechText.value = replayText;
|
||||
soundUnlockTip.value = true;
|
||||
releaseHuman(true, true);
|
||||
showToolbar();
|
||||
return;
|
||||
}
|
||||
postSpeechDisconnectTimer = window.setTimeout(() => {
|
||||
postSpeechDisconnectTimer = 0;
|
||||
if (!pendingSpeechText.value && !speechBusy.value) releaseHuman(false, true);
|
||||
}, POST_SPEECH_DISCONNECT_DELAY_MS);
|
||||
}
|
||||
if (action === 'DISCONNECT_ALERT') {
|
||||
showTimeoutTip.value = true;
|
||||
showToolbar();
|
||||
// cp-inactiveDisconnectSec=0 后通常不会触发；收到旧会话提醒也保持连接。
|
||||
showTimeoutTip.value = false;
|
||||
} else if (action === 'TIMEOUT_EXIT') disconnectHuman();
|
||||
}
|
||||
};
|
||||
|
||||
watch([humanReady, videoIsMuted, soundEnabled, pendingSpeechText], () => void tryPlayPendingAnswer(), {flush: 'post'});
|
||||
onMounted(() => {
|
||||
syncAudioUnlockFromBrowser();
|
||||
dhIframe.registerMessageReceived(onMessage);
|
||||
scheduleToolbarHide();
|
||||
});
|
||||
onUnmounted(() => {
|
||||
connectionGeneration += 1;
|
||||
clearToolbarTimer();
|
||||
if (postSpeechDisconnectTimer) window.clearTimeout(postSpeechDisconnectTimer);
|
||||
if (humanVisualTimer) window.clearTimeout(humanVisualTimer);
|
||||
if (humanReleaseTimer) window.clearTimeout(humanReleaseTimer);
|
||||
if (standbyCrossfadeTimer) window.clearTimeout(standbyCrossfadeTimer);
|
||||
clearAudioRestoreTimers();
|
||||
standbyVideos.forEach(video => video?.pause());
|
||||
dhIframe.removeMessageReceived(onMessage);
|
||||
});
|
||||
@ -493,18 +577,22 @@ onUnmounted(() => {
|
||||
<style scoped>
|
||||
#digital-human-iframe { width: 100%; height: 100%; border: none; display: block; }
|
||||
.realtime-dh-wrapper, .realtime-dh-stage { width: 100%; min-height: 100dvh; }
|
||||
.realtime-dh-wrapper { background: #f3f4fb; }
|
||||
.realtime-dh-stage { position: relative; overflow: hidden; display: flex; align-items: center; justify-content: center; background: #f3f4fb; }
|
||||
.realtime-dh-wrapper { background: #5a2f1b; }
|
||||
.realtime-dh-stage { position: relative; overflow: hidden; display: flex; align-items: center; justify-content: center; background: #5a2f1b; }
|
||||
.visual-stage { position: absolute; z-index: 0; left: 50%; top: 50%; width: min(100%, 177.7778dvh); aspect-ratio: 16 / 9; overflow: hidden; transform: translate(-50%, -50%); background: #5a2f1b; }
|
||||
.standby-scene, .human-frame { position: absolute; inset: 0; }
|
||||
.standby-scene { z-index: 0; overflow: hidden; background: #5a2f1b; }
|
||||
.standby-video { position: absolute; inset: 0; width: 100%; height: 100%; display: block; object-fit: cover; object-position: center; opacity: 0; background: #5a2f1b; transition: opacity 650ms ease-in-out, filter 500ms ease; }
|
||||
.standby-scene { z-index: 2; overflow: hidden; opacity: 0; background: #5a2f1b; transform: translateZ(0) scale(1.008); filter: brightness(.92); backface-visibility: hidden; will-change: opacity, transform, filter; transition: opacity 420ms ease-out, transform 1150ms cubic-bezier(.22, 1, .36, 1), filter 900ms ease; }
|
||||
.standby-scene.ready { opacity: 1; transform: translateZ(0) scale(1); filter: brightness(1); }
|
||||
.standby-scene.ready.human-mounted { transition: none; }
|
||||
.standby-scene.ready.human-mounted.human-active { opacity: 0; pointer-events: none; transition: opacity 180ms ease-out; }
|
||||
.standby-video { position: absolute; inset: 0; width: 100%; height: 100%; display: block; object-fit: cover; object-position: center; opacity: 0; background: #5a2f1b; transform: translateZ(0); backface-visibility: hidden; will-change: opacity, filter; transition: opacity 760ms cubic-bezier(.4, 0, .2, 1), filter 650ms ease; }
|
||||
.standby-video.visible, .standby-video.incoming { opacity: 1; }
|
||||
.standby-video.visible { z-index: 1; }.standby-video.incoming { z-index: 2; }
|
||||
.standby-scene.thinking .standby-video { filter: saturate(.92) brightness(.86); }
|
||||
.standby-shade { position: absolute; inset: 0; background: linear-gradient(180deg, rgba(18, 8, 4, .03) 42%, rgba(17, 8, 4, .58) 100%); }
|
||||
.human-frame { z-index: 1; opacity: 0; overflow: hidden; background: #5a2f1b; pointer-events: none; transform: translateZ(0) scale(1.004); filter: blur(1.5px); backface-visibility: hidden; will-change: opacity, transform, filter; transition: opacity 900ms cubic-bezier(.4, 0, .2, 1), transform 1100ms cubic-bezier(.22, 1, .36, 1), filter 700ms ease-out; }
|
||||
.human-frame.ready { opacity: 1; pointer-events: auto; transform: translateZ(0) scale(1); filter: blur(0); }
|
||||
.human-frame { z-index: 1; overflow: hidden; background: #5a2f1b; pointer-events: none; transform: translateZ(0); backface-visibility: hidden; }
|
||||
.human-frame.ready { pointer-events: auto; }
|
||||
.human-frame :deep(#digital-human-iframe) { position: relative; z-index: 1; width: 100%; height: 100%; border: 0; display: block; }
|
||||
.thinking-panel { position: absolute; z-index: 2; left: 50%; bottom: clamp(120px, 17vh, 190px); display: flex; align-items: center; gap: 13px; min-width: min(360px, calc(100% - 36px)); padding: 13px 17px; transform: translateX(-50%); border: 1px solid rgba(255, 238, 220, .35); border-radius: 18px; color: #fff7ed; background: rgba(37, 18, 10, .7); box-shadow: 0 18px 48px rgba(20, 7, 2, .3); backdrop-filter: blur(13px); }
|
||||
.thinking-orbit { position: relative; flex: 0 0 36px; width: 36px; height: 36px; }
|
||||
.thinking-orbit span { position: absolute; inset: 0; border: 1px solid rgba(255, 221, 181, .44); border-radius: 50%; }
|
||||
@ -551,5 +639,5 @@ onUnmounted(() => {
|
||||
.realtime-dh-right-tip { bottom: calc(38dvh + 22px); }
|
||||
}
|
||||
@media (max-width: 520px) { .connection-toolbar { left: 8px; right: 8px; gap: 6px; padding: 6px; }.connection-badge { margin-right: auto; max-width: 102px; overflow: hidden; text-overflow: ellipsis; font-size: 11px; }.connection-toolbar :deep(.el-button) { padding: 7px 10px; }.thinking-orbit { flex-basis: 30px; width: 30px; height: 30px; }.thinking-orbit span:nth-child(3) { inset: 11px; } }
|
||||
@media (prefers-reduced-motion: reduce) { .standby-video, .human-frame { transition-duration: 0s; }.thinking-panel.active .thinking-orbit span, .thinking-dots i { animation: none; } }
|
||||
@media (prefers-reduced-motion: reduce) { .standby-scene, .standby-video, .human-frame { transition-duration: 0s; }.thinking-panel.active .thinking-orbit span, .thinking-dots i { animation: none; } }
|
||||
</style>
|
||||
|
||||
Binary file not shown.
@ -19,13 +19,15 @@ export default {
|
||||
ttsPerson: null,
|
||||
'cp-ttsSample': '16000',
|
||||
initMode: 'noAudio',
|
||||
videoBg: '#F3F4FB',
|
||||
videoBg: '#5A2F1B',
|
||||
resolutionWidth: 1920,
|
||||
resolutionHeight: 1080,
|
||||
showDebugger: true,
|
||||
ttsPer: '20008742',
|
||||
backgroundImageUrl: 'https://meta-human-editor-test.cdn.bcebos.com/17f8e526-1530-48ab-b02a-1c6138e1da1e/ffda0bfd-a0f5-49cc-bff4-1501ac1aa155/defaultBg.png',
|
||||
'cp-preAlertSec': 120,
|
||||
// 连接由外层页面按 DeepSeek 30 秒超时规则管理，关闭百度组件自身的闲置退出提醒。
|
||||
'cp-inactiveDisconnectSec': 0,
|
||||
'cp-preAlertSec': 0,
|
||||
'cp-positionV2': '{"location":{"top":0,"left":0,"width":1920,"height":1080}}'
|
||||
};
|
||||
const QUERY_OVERRIDE_KEYS = new Set([
|
||||
@ -36,6 +38,7 @@ export default {
|
||||
'resolutionHeight',
|
||||
'showDebugger',
|
||||
'backgroundImageUrl',
|
||||
'cp-inactiveDisconnectSec',
|
||||
'cp-preAlertSec',
|
||||
'cp-positionV2'
|
||||
]);
|
||||
|
||||
@ -11,11 +11,12 @@ const OUTPUT_SAMPLE_RATE = 16_000;
|
||||
const SILENCE_AFTER_SPEECH_MS = 1_200;
|
||||
const TRAILING_SILENCE_PADDING_MS = 180;
|
||||
const MIN_VOICED_AUDIO_MS = 260;
|
||||
const MAX_UTTERANCE_MS = 55_000;
|
||||
const MAX_UTTERANCE_MS = 20_000;
|
||||
const PRE_ROLL_MS = 350;
|
||||
const PROCESSOR_BUFFER_SIZE = 1_024;
|
||||
const MIN_VOLUME_THRESHOLD = 0.012;
|
||||
const VOICE_TRIGGER_FRAMES = 2;
|
||||
const SPEECH_PEAK_HANGOVER_RATIO = 0.15;
|
||||
|
||||
const props = defineProps({
|
||||
speechBusy: {type: Boolean, default: false},
|
||||
@ -24,6 +25,8 @@ const props = defineProps({
|
||||
const emit = defineEmits([
|
||||
'questionUpdated',
|
||||
'questionRecognized',
|
||||
'speechDetected',
|
||||
'speechAborted',
|
||||
'answerReady',
|
||||
'requestTimeout',
|
||||
'requestError',
|
||||
@ -58,11 +61,13 @@ let recording = false;
|
||||
let speechStartedAt = 0;
|
||||
let lastVoiceAt = 0;
|
||||
let voicedAudioMs = 0;
|
||||
let peakVoiceRms = 0;
|
||||
let audioFrames = [];
|
||||
let audioSampleCount = 0;
|
||||
let preRollFrames = [];
|
||||
let preRollSampleCount = 0;
|
||||
let microphonePermission = null;
|
||||
let captureLifecyclePromise = Promise.resolve();
|
||||
|
||||
const canCapture = () => !disposed && !recognizing.value && !loading.value && !props.speechBusy && !props.playbackPending;
|
||||
|
||||
@ -78,6 +83,7 @@ const resetUtterance = () => {
|
||||
speechStartedAt = 0;
|
||||
lastVoiceAt = 0;
|
||||
voicedAudioMs = 0;
|
||||
peakVoiceRms = 0;
|
||||
audioFrames = [];
|
||||
audioSampleCount = 0;
|
||||
preRollFrames = [];
|
||||
@ -188,7 +194,10 @@ const recognizePcm = async (pcm) => {
|
||||
try {
|
||||
const response = await fetch('/api/baidu/asr', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'audio/pcm;rate=16000'},
|
||||
headers: {
|
||||
'Content-Type': 'audio/pcm;rate=16000',
|
||||
'X-Audio-Duration-Ms': String(Math.round(pcm.byteLength / 2 / OUTPUT_SAMPLE_RATE * 1000))
|
||||
},
|
||||
body: pcm,
|
||||
signal: controller.signal
|
||||
});
|
||||
@ -203,6 +212,7 @@ const recognizePcm = async (pcm) => {
|
||||
recognizing.value = false;
|
||||
await askDeepSeek(question);
|
||||
} catch (error) {
|
||||
emit('speechAborted');
|
||||
if (error?.name !== 'AbortError') {
|
||||
const message = error?.message || '语音识别失败，请重新说一遍';
|
||||
emit('voiceStatus', `${message}，正在继续监听`);
|
||||
@ -229,6 +239,7 @@ const finishUtterance = () => {
|
||||
resetUtterance();
|
||||
|
||||
if (!enoughVoice || sampleCount === 0) {
|
||||
emit('speechAborted');
|
||||
emit('voiceStatus', '声音太短，正在继续监听');
|
||||
return;
|
||||
}
|
||||
@ -267,7 +278,10 @@ const processAudioFrame = (input) => {
|
||||
let squareSum = 0;
|
||||
for (let index = 0; index < frame.length; index += 1) squareSum += frame[index] * frame[index];
|
||||
const rms = Math.sqrt(squareSum / frame.length);
|
||||
const threshold = Math.min(0.08, Math.max(MIN_VOLUME_THRESHOLD, noiseFloor * 3.2));
|
||||
const baseThreshold = Math.min(0.08, Math.max(MIN_VOLUME_THRESHOLD, noiseFloor * 3.2));
|
||||
const threshold = recording && peakVoiceRms > 0
|
||||
? Math.max(baseThreshold, peakVoiceRms * SPEECH_PEAK_HANGOVER_RATIO)
|
||||
: baseThreshold;
|
||||
const voiceDetected = rms >= threshold;
|
||||
const now = performance.now();
|
||||
const frameDurationMs = frame.length / inputSampleRate * 1000;
|
||||
@ -282,6 +296,8 @@ const processAudioFrame = (input) => {
|
||||
|
||||
if (consecutiveVoiceFrames >= VOICE_TRIGGER_FRAMES) {
|
||||
recording = true;
|
||||
peakVoiceRms = rms;
|
||||
emit('speechDetected');
|
||||
speechStartedAt = now;
|
||||
lastVoiceAt = now;
|
||||
voicedAudioMs = frameDurationMs * consecutiveVoiceFrames;
|
||||
@ -297,6 +313,7 @@ const processAudioFrame = (input) => {
|
||||
audioFrames.push(frame);
|
||||
audioSampleCount += frame.length;
|
||||
if (voiceDetected) {
|
||||
peakVoiceRms = Math.max(peakVoiceRms, rms);
|
||||
lastVoiceAt = now;
|
||||
voicedAudioMs += frameDurationMs;
|
||||
}
|
||||
@ -354,6 +371,7 @@ const initializeVoice = async () => {
|
||||
if (!AudioContextConstructor) throw new Error('当前浏览器不支持网页录音处理');
|
||||
audioContext = new AudioContextConstructor({latencyHint: 'interactive'});
|
||||
inputSampleRate = audioContext.sampleRate;
|
||||
noiseFloor = 0.003;
|
||||
sourceNode = audioContext.createMediaStreamSource(mediaStream);
|
||||
processorNode = audioContext.createScriptProcessor(PROCESSOR_BUFFER_SIZE, 1, 1);
|
||||
silentGainNode = audioContext.createGain();
|
||||
@ -400,7 +418,7 @@ const handleVisibilityChange = () => {
|
||||
};
|
||||
|
||||
const resumeAutomaticListening = async () => {
|
||||
if (disposed || document.hidden || initializing) return;
|
||||
if (disposed || document.hidden || initializing || props.speechBusy || props.playbackPending) return;
|
||||
if (mediaStream?.active && audioContext) {
|
||||
await audioContext.resume().catch(() => undefined);
|
||||
if (audioContext.state === 'running') {
|
||||
@ -412,6 +430,16 @@ const resumeAutomaticListening = async () => {
|
||||
if (microphonePermission?.state !== 'denied') await initializeVoice();
|
||||
};
|
||||
|
||||
const queueCaptureLifecycle = (operation) => {
|
||||
captureLifecyclePromise = captureLifecyclePromise
|
||||
.catch(() => undefined)
|
||||
.then(operation)
|
||||
.catch(error => {
|
||||
if (!disposed) reportVoiceError(error?.message || '麦克风恢复失败，请检查浏览器权限');
|
||||
});
|
||||
return captureLifecyclePromise;
|
||||
};
|
||||
|
||||
const handleMicrophonePermissionChange = () => {
|
||||
if (microphonePermission?.state === 'granted') {
|
||||
emit('voiceError', '');
|
||||
@ -440,11 +468,21 @@ const handleDeviceChange = () => {
|
||||
if (!mediaStream?.active) void resumeAutomaticListening();
|
||||
};
|
||||
|
||||
watch(() => [props.speechBusy, props.playbackPending], ([speechBusy, playbackPending]) => {
|
||||
watch(() => [props.speechBusy, props.playbackPending], ([speechBusy, playbackPending], previous = [false, false]) => {
|
||||
const answering = speechBusy || playbackPending;
|
||||
const wasAnswering = previous[0] || previous[1];
|
||||
resetUtterance();
|
||||
if (speechBusy || playbackPending) {
|
||||
if (answering) {
|
||||
listening.value = false;
|
||||
emit('voiceStatus', '数字人正在回答');
|
||||
if (!wasAnswering) {
|
||||
// 口播开始前彻底释放麦克风，避免数字人扬声器声音和 WebRTC 音频路由
|
||||
// 污染下一轮录音。浏览器已授权后，重新获取不会再次弹权限窗口。
|
||||
void queueCaptureLifecycle(() => stopAudioCapture());
|
||||
}
|
||||
} else if (wasAnswering && !recognizing.value && !loading.value) {
|
||||
// 口播结束后创建全新的 MediaStream 与 AudioContext；不能只修改 listening 状态。
|
||||
void queueCaptureLifecycle(() => initializeVoice());
|
||||
} else if (!recognizing.value && !loading.value && mediaStream?.active) {
|
||||
listening.value = true;
|
||||
emit('voiceStatus', '麦克风已开启，请直接说话');
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
line-height: 1.5;
|
||||
font-weight: 400;
|
||||
color: rgba(255, 255, 255, 0.87);
|
||||
background-color: #242424;
|
||||
background-color: #5a2f1b;
|
||||
font-synthesis: none;
|
||||
text-rendering: optimizeLegibility;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
@ -20,6 +20,7 @@ body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overflow-x: hidden;
|
||||
background-color: #5a2f1b;
|
||||
}
|
||||
|
||||
#root {
|
||||
@ -29,6 +30,7 @@ body {
|
||||
height: 100%;
|
||||
margin: 0 auto;
|
||||
text-align: center;
|
||||
background-color: #5a2f1b;
|
||||
}
|
||||
/* a {
|
||||
font-weight: 500;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user