489 lines
17 KiB
Vue
489 lines
17 KiB
Vue
<template>
|
||
<span class="sr-only" aria-live="polite">{{ assistiveStatus }}</span>
|
||
</template>
|
||
|
||
<script setup>
|
||
import {computed, onMounted, onUnmounted, ref, watch} from 'vue';
|
||
import {ElMessage} from 'element-plus';
|
||
|
||
const DEEPSEEK_TIMEOUT_MS = 30_000;
|
||
const OUTPUT_SAMPLE_RATE = 16_000;
|
||
const SILENCE_AFTER_SPEECH_MS = 1_200;
|
||
const TRAILING_SILENCE_PADDING_MS = 180;
|
||
const MIN_VOICED_AUDIO_MS = 260;
|
||
const MAX_UTTERANCE_MS = 55_000;
|
||
const PRE_ROLL_MS = 350;
|
||
const PROCESSOR_BUFFER_SIZE = 1_024;
|
||
const MIN_VOLUME_THRESHOLD = 0.012;
|
||
const VOICE_TRIGGER_FRAMES = 2;
|
||
|
||
const props = defineProps({
|
||
speechBusy: {type: Boolean, default: false},
|
||
playbackPending: {type: Boolean, default: false}
|
||
});
|
||
const emit = defineEmits([
|
||
'questionUpdated',
|
||
'questionRecognized',
|
||
'answerReady',
|
||
'requestTimeout',
|
||
'requestError',
|
||
'voiceError',
|
||
'voiceStatus'
|
||
]);
|
||
|
||
const listening = ref(false);
|
||
const recognizing = ref(false);
|
||
const loading = ref(false);
|
||
const assistiveStatus = computed(() => {
|
||
if (recognizing.value) return '正在将语音转换成文字';
|
||
if (loading.value) return '正在向 DeepSeek 提问';
|
||
if (props.speechBusy || props.playbackPending) return '数字人正在回答';
|
||
if (listening.value) return '麦克风已开启，正在聆听';
|
||
return '语音助手正在准备';
|
||
});
|
||
|
||
let mediaStream = null;
|
||
let audioContext = null;
|
||
let sourceNode = null;
|
||
let processorNode = null;
|
||
let silentGainNode = null;
|
||
let requestController = null;
|
||
let asrController = null;
|
||
let disposed = false;
|
||
let initializing = false;
|
||
let inputSampleRate = 48_000;
|
||
let noiseFloor = 0.003;
|
||
let consecutiveVoiceFrames = 0;
|
||
let recording = false;
|
||
let speechStartedAt = 0;
|
||
let lastVoiceAt = 0;
|
||
let voicedAudioMs = 0;
|
||
let audioFrames = [];
|
||
let audioSampleCount = 0;
|
||
let preRollFrames = [];
|
||
let preRollSampleCount = 0;
|
||
let microphonePermission = null;
|
||
|
||
const canCapture = () => !disposed && !recognizing.value && !loading.value && !props.speechBusy && !props.playbackPending;
|
||
|
||
const reportVoiceError = (message) => {
|
||
listening.value = false;
|
||
emit('voiceStatus', message);
|
||
emit('voiceError', message);
|
||
};
|
||
|
||
const resetUtterance = () => {
|
||
consecutiveVoiceFrames = 0;
|
||
recording = false;
|
||
speechStartedAt = 0;
|
||
lastVoiceAt = 0;
|
||
voicedAudioMs = 0;
|
||
audioFrames = [];
|
||
audioSampleCount = 0;
|
||
preRollFrames = [];
|
||
preRollSampleCount = 0;
|
||
};
|
||
|
||
const mergeFrames = (frames, sampleCount) => {
|
||
const merged = new Float32Array(sampleCount);
|
||
let offset = 0;
|
||
frames.forEach(frame => {
|
||
merged.set(frame, offset);
|
||
offset += frame.length;
|
||
});
|
||
return merged;
|
||
};
|
||
|
||
const resampleTo16k = (samples, sourceRate) => {
|
||
if (sourceRate === OUTPUT_SAMPLE_RATE) return samples;
|
||
const ratio = sourceRate / OUTPUT_SAMPLE_RATE;
|
||
const outputLength = Math.max(1, Math.floor(samples.length / ratio));
|
||
const output = new Float32Array(outputLength);
|
||
|
||
if (ratio < 1) {
|
||
for (let index = 0; index < outputLength; index += 1) {
|
||
const sourcePosition = index * ratio;
|
||
const leftIndex = Math.floor(sourcePosition);
|
||
const rightIndex = Math.min(samples.length - 1, leftIndex + 1);
|
||
const fraction = sourcePosition - leftIndex;
|
||
output[index] = samples[leftIndex] * (1 - fraction) + samples[rightIndex] * fraction;
|
||
}
|
||
return output;
|
||
}
|
||
|
||
for (let index = 0; index < outputLength; index += 1) {
|
||
const start = Math.floor(index * ratio);
|
||
const end = Math.min(samples.length, Math.max(start + 1, Math.floor((index + 1) * ratio)));
|
||
let sum = 0;
|
||
for (let sourceIndex = start; sourceIndex < end; sourceIndex += 1) sum += samples[sourceIndex];
|
||
output[index] = sum / (end - start);
|
||
}
|
||
return output;
|
||
};
|
||
|
||
const encodePcm16 = (samples) => {
|
||
const pcm = new Int16Array(samples.length);
|
||
for (let index = 0; index < samples.length; index += 1) {
|
||
const value = Math.max(-1, Math.min(1, samples[index]));
|
||
pcm[index] = value < 0 ? value * 0x8000 : value * 0x7fff;
|
||
}
|
||
return pcm;
|
||
};
|
||
|
||
const askDeepSeek = async (question) => {
|
||
loading.value = true;
|
||
listening.value = false;
|
||
emit('voiceStatus', '已识别问题，正在等待 DeepSeek 回答…');
|
||
const controller = new AbortController();
|
||
requestController = controller;
|
||
let timedOut = false;
|
||
const timeoutId = window.setTimeout(() => {
|
||
timedOut = true;
|
||
emit('requestTimeout');
|
||
controller.abort();
|
||
}, DEEPSEEK_TIMEOUT_MS);
|
||
|
||
try {
|
||
const response = await fetch('/api/deepseek/chat', {
|
||
method: 'POST',
|
||
headers: {'Content-Type': 'application/json'},
|
||
body: JSON.stringify({question}),
|
||
signal: controller.signal
|
||
});
|
||
const data = await response.json().catch(() => ({}));
|
||
if (!response.ok) throw new Error(data.error || 'DeepSeek 请求失败，请稍后重试');
|
||
const answer = typeof data.answer === 'string' ? data.answer.trim() : '';
|
||
if (!answer) throw new Error('DeepSeek 没有返回可播报的答案');
|
||
emit('answerReady', answer);
|
||
emit('voiceStatus', 'DeepSeek 已返回，数字人正在准备口播');
|
||
} catch (error) {
|
||
if (timedOut || error?.name === 'AbortError') {
|
||
ElMessage.warning('DeepSeek 30 秒未返回，数字人已自动断开');
|
||
} else {
|
||
const message = error?.message || 'DeepSeek 请求失败，请稍后重试';
|
||
emit('requestError', message);
|
||
emit('voiceStatus', '回答失败，已继续监听');
|
||
ElMessage.error(message);
|
||
}
|
||
} finally {
|
||
window.clearTimeout(timeoutId);
|
||
if (requestController === controller) {
|
||
requestController = null;
|
||
loading.value = false;
|
||
if (!props.speechBusy && !props.playbackPending) {
|
||
listening.value = true;
|
||
emit('voiceStatus', '麦克风已开启，请直接说话');
|
||
}
|
||
}
|
||
}
|
||
};
|
||
|
||
const recognizePcm = async (pcm) => {
|
||
recognizing.value = true;
|
||
listening.value = false;
|
||
emit('voiceStatus', '已检测到停顿，正在识别语音…');
|
||
const controller = new AbortController();
|
||
asrController = controller;
|
||
|
||
try {
|
||
const response = await fetch('/api/baidu/asr', {
|
||
method: 'POST',
|
||
headers: {'Content-Type': 'audio/pcm;rate=16000'},
|
||
body: pcm,
|
||
signal: controller.signal
|
||
});
|
||
const data = await response.json().catch(() => ({}));
|
||
if (!response.ok) throw new Error(data.error || '百度语音识别失败，请重新说一遍');
|
||
const question = typeof data.text === 'string' ? data.text.trim() : '';
|
||
if (!question) throw new Error('没有识别到清晰语音，请重新说一遍');
|
||
|
||
emit('voiceError', '');
|
||
emit('questionUpdated', question);
|
||
emit('questionRecognized', question);
|
||
recognizing.value = false;
|
||
await askDeepSeek(question);
|
||
} catch (error) {
|
||
if (error?.name !== 'AbortError') {
|
||
const message = error?.message || '语音识别失败，请重新说一遍';
|
||
emit('voiceStatus', `${message}，正在继续监听`);
|
||
if (!message.includes('没有识别') && !message.includes('录音时间太短')) {
|
||
emit('voiceError', message);
|
||
ElMessage.error(message);
|
||
}
|
||
}
|
||
} finally {
|
||
if (asrController === controller) asrController = null;
|
||
recognizing.value = false;
|
||
if (!loading.value && !props.speechBusy && !props.playbackPending) {
|
||
listening.value = true;
|
||
}
|
||
}
|
||
};
|
||
|
||
const finishUtterance = () => {
|
||
if (!recording || recognizing.value || loading.value) return;
|
||
const frames = audioFrames;
|
||
const sampleCount = audioSampleCount;
|
||
const enoughVoice = voicedAudioMs >= MIN_VOICED_AUDIO_MS;
|
||
const trailingSilenceMs = Math.max(0, performance.now() - lastVoiceAt);
|
||
resetUtterance();
|
||
|
||
if (!enoughVoice || sampleCount === 0) {
|
||
emit('voiceStatus', '声音太短，正在继续监听');
|
||
return;
|
||
}
|
||
|
||
const merged = mergeFrames(frames, sampleCount);
|
||
const removableSilenceMs = Math.max(0, trailingSilenceMs - TRAILING_SILENCE_PADDING_MS);
|
||
const removableSamples = Math.min(
|
||
Math.max(0, merged.length - 1),
|
||
Math.floor(inputSampleRate * removableSilenceMs / 1000)
|
||
);
|
||
const trimmed = removableSamples > 0
|
||
? merged.subarray(0, merged.length - removableSamples)
|
||
: merged;
|
||
const pcm = encodePcm16(resampleTo16k(trimmed, inputSampleRate));
|
||
void recognizePcm(pcm);
|
||
};
|
||
|
||
const pushPreRollFrame = (frame) => {
|
||
preRollFrames.push(frame);
|
||
preRollSampleCount += frame.length;
|
||
const maxSamples = Math.ceil(inputSampleRate * PRE_ROLL_MS / 1000);
|
||
while (preRollSampleCount > maxSamples && preRollFrames.length > 1) {
|
||
const removed = preRollFrames.shift();
|
||
preRollSampleCount -= removed.length;
|
||
}
|
||
};
|
||
|
||
const processAudioFrame = (input) => {
|
||
if (!canCapture()) {
|
||
if (recording || preRollFrames.length) resetUtterance();
|
||
return;
|
||
}
|
||
|
||
const frame = new Float32Array(input.length);
|
||
frame.set(input);
|
||
let squareSum = 0;
|
||
for (let index = 0; index < frame.length; index += 1) squareSum += frame[index] * frame[index];
|
||
const rms = Math.sqrt(squareSum / frame.length);
|
||
const threshold = Math.min(0.08, Math.max(MIN_VOLUME_THRESHOLD, noiseFloor * 3.2));
|
||
const voiceDetected = rms >= threshold;
|
||
const now = performance.now();
|
||
const frameDurationMs = frame.length / inputSampleRate * 1000;
|
||
|
||
if (!recording) {
|
||
pushPreRollFrame(frame);
|
||
if (voiceDetected) consecutiveVoiceFrames += 1;
|
||
else {
|
||
consecutiveVoiceFrames = 0;
|
||
noiseFloor = Math.min(0.02, noiseFloor * 0.96 + rms * 0.04);
|
||
}
|
||
|
||
if (consecutiveVoiceFrames >= VOICE_TRIGGER_FRAMES) {
|
||
recording = true;
|
||
speechStartedAt = now;
|
||
lastVoiceAt = now;
|
||
voicedAudioMs = frameDurationMs * consecutiveVoiceFrames;
|
||
audioFrames = preRollFrames;
|
||
audioSampleCount = preRollSampleCount;
|
||
preRollFrames = [];
|
||
preRollSampleCount = 0;
|
||
emit('voiceStatus', '正在听你说话，说完后稍作停顿即可');
|
||
}
|
||
return;
|
||
}
|
||
|
||
audioFrames.push(frame);
|
||
audioSampleCount += frame.length;
|
||
if (voiceDetected) {
|
||
lastVoiceAt = now;
|
||
voicedAudioMs += frameDurationMs;
|
||
}
|
||
|
||
if (now - lastVoiceAt >= SILENCE_AFTER_SPEECH_MS || now - speechStartedAt >= MAX_UTTERANCE_MS) {
|
||
finishUtterance();
|
||
}
|
||
};
|
||
|
||
const stopAudioCapture = async () => {
|
||
processorNode && (processorNode.onaudioprocess = null);
|
||
processorNode?.disconnect();
|
||
sourceNode?.disconnect();
|
||
silentGainNode?.disconnect();
|
||
mediaStream?.getTracks().forEach(track => track.stop());
|
||
const context = audioContext;
|
||
processorNode = null;
|
||
sourceNode = null;
|
||
silentGainNode = null;
|
||
mediaStream = null;
|
||
audioContext = null;
|
||
resetUtterance();
|
||
if (context && context.state !== 'closed') await context.close().catch(() => undefined);
|
||
};
|
||
|
||
const initializeVoice = async () => {
|
||
if (disposed || initializing) return;
|
||
if (!window.isSecureContext) {
|
||
reportVoiceError('语音交互必须通过 HTTPS 访问；HTTP 页面不能使用手机麦克风');
|
||
return;
|
||
}
|
||
if (!navigator.mediaDevices?.getUserMedia) {
|
||
reportVoiceError('当前浏览器不支持麦克风录音，请使用最新版 Chrome、Edge 或 Safari');
|
||
return;
|
||
}
|
||
|
||
initializing = true;
|
||
emit('voiceStatus', '正在申请麦克风权限…');
|
||
try {
|
||
await stopAudioCapture();
|
||
mediaStream = await navigator.mediaDevices.getUserMedia({
|
||
audio: {
|
||
channelCount: 1,
|
||
echoCancellation: true,
|
||
noiseSuppression: true,
|
||
autoGainControl: true
|
||
}
|
||
});
|
||
if (disposed) {
|
||
mediaStream.getTracks().forEach(track => track.stop());
|
||
return;
|
||
}
|
||
|
||
const AudioContextConstructor = window.AudioContext || window.webkitAudioContext;
|
||
if (!AudioContextConstructor) throw new Error('当前浏览器不支持网页录音处理');
|
||
audioContext = new AudioContextConstructor({latencyHint: 'interactive'});
|
||
inputSampleRate = audioContext.sampleRate;
|
||
sourceNode = audioContext.createMediaStreamSource(mediaStream);
|
||
processorNode = audioContext.createScriptProcessor(PROCESSOR_BUFFER_SIZE, 1, 1);
|
||
silentGainNode = audioContext.createGain();
|
||
silentGainNode.gain.value = 0;
|
||
processorNode.onaudioprocess = event => processAudioFrame(event.inputBuffer.getChannelData(0));
|
||
sourceNode.connect(processorNode);
|
||
processorNode.connect(silentGainNode);
|
||
silentGainNode.connect(audioContext.destination);
|
||
await audioContext.resume();
|
||
|
||
emit('voiceError', '');
|
||
listening.value = true;
|
||
emit('voiceStatus', '麦克风已开启，请直接说话');
|
||
} catch (error) {
|
||
const messages = {
|
||
NotAllowedError: '未获得麦克风权限，请在网站设置中允许麦克风；授权后页面会自动开始监听',
|
||
SecurityError: '浏览器安全策略阻止了麦克风，请允许权限；授权后页面会自动开始监听',
|
||
NotFoundError: '没有检测到可用麦克风，请检查手机或耳机麦克风',
|
||
NotReadableError: '麦克风无法读取，可能正在被其他应用占用',
|
||
AbortError: '麦克风启动失败，页面将在恢复后自动重试'
|
||
};
|
||
reportVoiceError(messages[error?.name] || error?.message || '无法打开麦克风，请检查设备和浏览器权限');
|
||
await stopAudioCapture();
|
||
} finally {
|
||
initializing = false;
|
||
}
|
||
};
|
||
|
||
const activateVoice = async () => {
|
||
if (disposed) return;
|
||
emit('voiceError', '');
|
||
resetUtterance();
|
||
if (!audioContext || !mediaStream?.active) {
|
||
await initializeVoice();
|
||
return;
|
||
}
|
||
await audioContext.resume().catch(() => undefined);
|
||
listening.value = true;
|
||
emit('voiceStatus', '麦克风已开启，请直接说话');
|
||
};
|
||
|
||
const handleVisibilityChange = () => {
|
||
if (!document.hidden) void resumeAutomaticListening();
|
||
};
|
||
|
||
const resumeAutomaticListening = async () => {
|
||
if (disposed || document.hidden || initializing) return;
|
||
if (mediaStream?.active && audioContext) {
|
||
await audioContext.resume().catch(() => undefined);
|
||
if (audioContext.state === 'running') {
|
||
listening.value = !recognizing.value && !loading.value && !props.speechBusy && !props.playbackPending;
|
||
if (listening.value) emit('voiceStatus', '麦克风已开启，请直接说话');
|
||
}
|
||
return;
|
||
}
|
||
if (microphonePermission?.state !== 'denied') await initializeVoice();
|
||
};
|
||
|
||
const handleMicrophonePermissionChange = () => {
|
||
if (microphonePermission?.state === 'granted') {
|
||
emit('voiceError', '');
|
||
void resumeAutomaticListening();
|
||
} else if (microphonePermission?.state === 'denied') {
|
||
reportVoiceError('麦克风权限已被禁止，请在浏览器网站设置中改为允许；授权后会自动开始监听');
|
||
}
|
||
};
|
||
|
||
const monitorMicrophonePermission = async () => {
|
||
if (!navigator.permissions?.query) return;
|
||
try {
|
||
microphonePermission = await navigator.permissions.query({name: 'microphone'});
|
||
microphonePermission.addEventListener('change', handleMicrophonePermissionChange);
|
||
} catch {
|
||
microphonePermission = null;
|
||
}
|
||
};
|
||
|
||
const startAutomaticListening = async () => {
|
||
await monitorMicrophonePermission();
|
||
await resumeAutomaticListening();
|
||
};
|
||
|
||
const handleDeviceChange = () => {
|
||
if (!mediaStream?.active) void resumeAutomaticListening();
|
||
};
|
||
|
||
watch(() => [props.speechBusy, props.playbackPending], ([speechBusy, playbackPending]) => {
|
||
resetUtterance();
|
||
if (speechBusy || playbackPending) {
|
||
listening.value = false;
|
||
emit('voiceStatus', '数字人正在回答');
|
||
} else if (!recognizing.value && !loading.value && mediaStream?.active) {
|
||
listening.value = true;
|
||
emit('voiceStatus', '麦克风已开启，请直接说话');
|
||
}
|
||
});
|
||
|
||
defineExpose({activateVoice});
|
||
|
||
onMounted(() => {
|
||
document.addEventListener('visibilitychange', handleVisibilityChange);
|
||
window.addEventListener('focus', resumeAutomaticListening);
|
||
window.addEventListener('pageshow', resumeAutomaticListening);
|
||
navigator.mediaDevices?.addEventListener?.('devicechange', handleDeviceChange);
|
||
void startAutomaticListening();
|
||
});
|
||
onUnmounted(() => {
|
||
disposed = true;
|
||
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
||
window.removeEventListener('focus', resumeAutomaticListening);
|
||
window.removeEventListener('pageshow', resumeAutomaticListening);
|
||
navigator.mediaDevices?.removeEventListener?.('devicechange', handleDeviceChange);
|
||
microphonePermission?.removeEventListener('change', handleMicrophonePermissionChange);
|
||
requestController?.abort();
|
||
asrController?.abort();
|
||
void stopAudioCapture();
|
||
});
|
||
</script>
|
||
|
||
<style scoped>
|
||
.sr-only {
|
||
position: absolute;
|
||
width: 1px;
|
||
height: 1px;
|
||
padding: 0;
|
||
margin: -1px;
|
||
overflow: hidden;
|
||
clip: rect(0, 0, 0, 0);
|
||
white-space: nowrap;
|
||
border: 0;
|
||
}
|
||
</style>
|