shuziren/src/components/Render/VoiceRender.vue

<template>
    <span class="sr-only" aria-live="polite">{{ assistiveStatus }}</span>
</template>

<script setup>
import {computed, onMounted, onUnmounted, ref, watch} from 'vue';
import {ElMessage} from 'element-plus';

const DEEPSEEK_TIMEOUT_MS = 30_000;
const OUTPUT_SAMPLE_RATE = 16_000;
const SILENCE_AFTER_SPEECH_MS = 1_200;
const TRAILING_SILENCE_PADDING_MS = 180;
const MIN_VOICED_AUDIO_MS = 260;
const MAX_UTTERANCE_MS = 55_000;
const PRE_ROLL_MS = 350;
const PROCESSOR_BUFFER_SIZE = 1_024;
const MIN_VOLUME_THRESHOLD = 0.012;
const VOICE_TRIGGER_FRAMES = 2;

const props = defineProps({
    speechBusy: {type: Boolean, default: false},
    playbackPending: {type: Boolean, default: false}
});
const emit = defineEmits([
    'questionUpdated',
    'questionRecognized',
    'answerReady',
    'requestTimeout',
    'requestError',
    'voiceError',
    'voiceStatus'
]);

const listening = ref(false);
const recognizing = ref(false);
const loading = ref(false);
const assistiveStatus = computed(() => {
    if (recognizing.value) return '正在将语音转换成文字';
    if (loading.value) return '正在向 DeepSeek 提问';
    if (props.speechBusy || props.playbackPending) return '数字人正在回答';
    if (listening.value) return '麦克风已开启，正在聆听';
    return '语音助手正在准备';
});

let mediaStream = null;
let audioContext = null;
let sourceNode = null;
let processorNode = null;
let silentGainNode = null;
let requestController = null;
let asrController = null;
let disposed = false;
let initializing = false;
let inputSampleRate = 48_000;
let noiseFloor = 0.003;
let consecutiveVoiceFrames = 0;
let recording = false;
let speechStartedAt = 0;
let lastVoiceAt = 0;
let voicedAudioMs = 0;
let audioFrames = [];
let audioSampleCount = 0;
let preRollFrames = [];
let preRollSampleCount = 0;
let microphonePermission = null;

const canCapture = () => !disposed && !recognizing.value && !loading.value && !props.speechBusy && !props.playbackPending;

const reportVoiceError = (message) => {
    listening.value = false;
    emit('voiceStatus', message);
    emit('voiceError', message);
};

const resetUtterance = () => {
    consecutiveVoiceFrames = 0;
    recording = false;
    speechStartedAt = 0;
    lastVoiceAt = 0;
    voicedAudioMs = 0;
    audioFrames = [];
    audioSampleCount = 0;
    preRollFrames = [];
    preRollSampleCount = 0;
};

const mergeFrames = (frames, sampleCount) => {
    const merged = new Float32Array(sampleCount);
    let offset = 0;
    frames.forEach(frame => {
        merged.set(frame, offset);
        offset += frame.length;
    });
    return merged;
};

const resampleTo16k = (samples, sourceRate) => {
    if (sourceRate === OUTPUT_SAMPLE_RATE) return samples;
    const ratio = sourceRate / OUTPUT_SAMPLE_RATE;
    const outputLength = Math.max(1, Math.floor(samples.length / ratio));
    const output = new Float32Array(outputLength);

    if (ratio < 1) {
        for (let index = 0; index < outputLength; index += 1) {
            const sourcePosition = index * ratio;
            const leftIndex = Math.floor(sourcePosition);
            const rightIndex = Math.min(samples.length - 1, leftIndex + 1);
            const fraction = sourcePosition - leftIndex;
            output[index] = samples[leftIndex] * (1 - fraction) + samples[rightIndex] * fraction;
        }
        return output;
    }

    for (let index = 0; index < outputLength; index += 1) {
        const start = Math.floor(index * ratio);
        const end = Math.min(samples.length, Math.max(start + 1, Math.floor((index + 1) * ratio)));
        let sum = 0;
        for (let sourceIndex = start; sourceIndex < end; sourceIndex += 1) sum += samples[sourceIndex];
        output[index] = sum / (end - start);
    }
    return output;
};

const encodePcm16 = (samples) => {
    const pcm = new Int16Array(samples.length);
    for (let index = 0; index < samples.length; index += 1) {
        const value = Math.max(-1, Math.min(1, samples[index]));
        pcm[index] = value < 0 ? value * 0x8000 : value * 0x7fff;
    }
    return pcm;
};

const askDeepSeek = async (question) => {
    loading.value = true;
    listening.value = false;
    emit('voiceStatus', '已识别问题，正在等待 DeepSeek 回答…');
    const controller = new AbortController();
    requestController = controller;
    let timedOut = false;
    const timeoutId = window.setTimeout(() => {
        timedOut = true;
        emit('requestTimeout');
        controller.abort();
    }, DEEPSEEK_TIMEOUT_MS);

    try {
        const response = await fetch('/api/deepseek/chat', {
            method: 'POST',
            headers: {'Content-Type': 'application/json'},
            body: JSON.stringify({question}),
            signal: controller.signal
        });
        const data = await response.json().catch(() => ({}));
        if (!response.ok) throw new Error(data.error || 'DeepSeek 请求失败，请稍后重试');
        const answer = typeof data.answer === 'string' ? data.answer.trim() : '';
        if (!answer) throw new Error('DeepSeek 没有返回可播报的答案');
        emit('answerReady', answer);
        emit('voiceStatus', 'DeepSeek 已返回，数字人正在准备口播');
    } catch (error) {
        if (timedOut || error?.name === 'AbortError') {
            ElMessage.warning('DeepSeek 30 秒未返回，数字人已自动断开');
        } else {
            const message = error?.message || 'DeepSeek 请求失败，请稍后重试';
            emit('requestError', message);
            emit('voiceStatus', '回答失败，已继续监听');
            ElMessage.error(message);
        }
    } finally {
        window.clearTimeout(timeoutId);
        if (requestController === controller) {
            requestController = null;
            loading.value = false;
            if (!props.speechBusy && !props.playbackPending) {
                listening.value = true;
                emit('voiceStatus', '麦克风已开启，请直接说话');
            }
        }
    }
};

const recognizePcm = async (pcm) => {
    recognizing.value = true;
    listening.value = false;
    emit('voiceStatus', '已检测到停顿，正在识别语音…');
    const controller = new AbortController();
    asrController = controller;

    try {
        const response = await fetch('/api/baidu/asr', {
            method: 'POST',
            headers: {'Content-Type': 'audio/pcm;rate=16000'},
            body: pcm,
            signal: controller.signal
        });
        const data = await response.json().catch(() => ({}));
        if (!response.ok) throw new Error(data.error || '百度语音识别失败，请重新说一遍');
        const question = typeof data.text === 'string' ? data.text.trim() : '';
        if (!question) throw new Error('没有识别到清晰语音，请重新说一遍');

        emit('voiceError', '');
        emit('questionUpdated', question);
        emit('questionRecognized', question);
        recognizing.value = false;
        await askDeepSeek(question);
    } catch (error) {
        if (error?.name !== 'AbortError') {
            const message = error?.message || '语音识别失败，请重新说一遍';
            emit('voiceStatus', `${message}，正在继续监听`);
            if (!message.includes('没有识别') && !message.includes('录音时间太短')) {
                emit('voiceError', message);
                ElMessage.error(message);
            }
        }
    } finally {
        if (asrController === controller) asrController = null;
        recognizing.value = false;
        if (!loading.value && !props.speechBusy && !props.playbackPending) {
            listening.value = true;
        }
    }
};

const finishUtterance = () => {
    if (!recording || recognizing.value || loading.value) return;
    const frames = audioFrames;
    const sampleCount = audioSampleCount;
    const enoughVoice = voicedAudioMs >= MIN_VOICED_AUDIO_MS;
    const trailingSilenceMs = Math.max(0, performance.now() - lastVoiceAt);
    resetUtterance();

    if (!enoughVoice || sampleCount === 0) {
        emit('voiceStatus', '声音太短，正在继续监听');
        return;
    }

    const merged = mergeFrames(frames, sampleCount);
    const removableSilenceMs = Math.max(0, trailingSilenceMs - TRAILING_SILENCE_PADDING_MS);
    const removableSamples = Math.min(
        Math.max(0, merged.length - 1),
        Math.floor(inputSampleRate * removableSilenceMs / 1000)
    );
    const trimmed = removableSamples > 0
        ? merged.subarray(0, merged.length - removableSamples)
        : merged;
    const pcm = encodePcm16(resampleTo16k(trimmed, inputSampleRate));
    void recognizePcm(pcm);
};

const pushPreRollFrame = (frame) => {
    preRollFrames.push(frame);
    preRollSampleCount += frame.length;
    const maxSamples = Math.ceil(inputSampleRate * PRE_ROLL_MS / 1000);
    while (preRollSampleCount > maxSamples && preRollFrames.length > 1) {
        const removed = preRollFrames.shift();
        preRollSampleCount -= removed.length;
    }
};

const processAudioFrame = (input) => {
    if (!canCapture()) {
        if (recording || preRollFrames.length) resetUtterance();
        return;
    }

    const frame = new Float32Array(input.length);
    frame.set(input);
    let squareSum = 0;
    for (let index = 0; index < frame.length; index += 1) squareSum += frame[index] * frame[index];
    const rms = Math.sqrt(squareSum / frame.length);
    const threshold = Math.min(0.08, Math.max(MIN_VOLUME_THRESHOLD, noiseFloor * 3.2));
    const voiceDetected = rms >= threshold;
    const now = performance.now();
    const frameDurationMs = frame.length / inputSampleRate * 1000;

    if (!recording) {
        pushPreRollFrame(frame);
        if (voiceDetected) consecutiveVoiceFrames += 1;
        else {
            consecutiveVoiceFrames = 0;
            noiseFloor = Math.min(0.02, noiseFloor * 0.96 + rms * 0.04);
        }

        if (consecutiveVoiceFrames >= VOICE_TRIGGER_FRAMES) {
            recording = true;
            speechStartedAt = now;
            lastVoiceAt = now;
            voicedAudioMs = frameDurationMs * consecutiveVoiceFrames;
            audioFrames = preRollFrames;
            audioSampleCount = preRollSampleCount;
            preRollFrames = [];
            preRollSampleCount = 0;
            emit('voiceStatus', '正在听你说话，说完后稍作停顿即可');
        }
        return;
    }

    audioFrames.push(frame);
    audioSampleCount += frame.length;
    if (voiceDetected) {
        lastVoiceAt = now;
        voicedAudioMs += frameDurationMs;
    }

    if (now - lastVoiceAt >= SILENCE_AFTER_SPEECH_MS || now - speechStartedAt >= MAX_UTTERANCE_MS) {
        finishUtterance();
    }
};

const stopAudioCapture = async () => {
    processorNode && (processorNode.onaudioprocess = null);
    processorNode?.disconnect();
    sourceNode?.disconnect();
    silentGainNode?.disconnect();
    mediaStream?.getTracks().forEach(track => track.stop());
    const context = audioContext;
    processorNode = null;
    sourceNode = null;
    silentGainNode = null;
    mediaStream = null;
    audioContext = null;
    resetUtterance();
    if (context && context.state !== 'closed') await context.close().catch(() => undefined);
};

const initializeVoice = async () => {
    if (disposed || initializing) return;
    if (!window.isSecureContext) {
        reportVoiceError('语音交互必须通过 HTTPS 访问；HTTP 页面不能使用手机麦克风');
        return;
    }
    if (!navigator.mediaDevices?.getUserMedia) {
        reportVoiceError('当前浏览器不支持麦克风录音，请使用最新版 Chrome、Edge 或 Safari');
        return;
    }

    initializing = true;
    emit('voiceStatus', '正在申请麦克风权限…');
    try {
        await stopAudioCapture();
        mediaStream = await navigator.mediaDevices.getUserMedia({
            audio: {
                channelCount: 1,
                echoCancellation: true,
                noiseSuppression: true,
                autoGainControl: true
            }
        });
        if (disposed) {
            mediaStream.getTracks().forEach(track => track.stop());
            return;
        }

        const AudioContextConstructor = window.AudioContext || window.webkitAudioContext;
        if (!AudioContextConstructor) throw new Error('当前浏览器不支持网页录音处理');
        audioContext = new AudioContextConstructor({latencyHint: 'interactive'});
        inputSampleRate = audioContext.sampleRate;
        sourceNode = audioContext.createMediaStreamSource(mediaStream);
        processorNode = audioContext.createScriptProcessor(PROCESSOR_BUFFER_SIZE, 1, 1);
        silentGainNode = audioContext.createGain();
        silentGainNode.gain.value = 0;
        processorNode.onaudioprocess = event => processAudioFrame(event.inputBuffer.getChannelData(0));
        sourceNode.connect(processorNode);
        processorNode.connect(silentGainNode);
        silentGainNode.connect(audioContext.destination);
        await audioContext.resume();

        emit('voiceError', '');
        listening.value = true;
        emit('voiceStatus', '麦克风已开启，请直接说话');
    } catch (error) {
        const messages = {
            NotAllowedError: '未获得麦克风权限，请在网站设置中允许麦克风；授权后页面会自动开始监听',
            SecurityError: '浏览器安全策略阻止了麦克风，请允许权限；授权后页面会自动开始监听',
            NotFoundError: '没有检测到可用麦克风，请检查手机或耳机麦克风',
            NotReadableError: '麦克风无法读取，可能正在被其他应用占用',
            AbortError: '麦克风启动失败，页面将在恢复后自动重试'
        };
        reportVoiceError(messages[error?.name] || error?.message || '无法打开麦克风，请检查设备和浏览器权限');
        await stopAudioCapture();
    } finally {
        initializing = false;
    }
};

const activateVoice = async () => {
    if (disposed) return;
    emit('voiceError', '');
    resetUtterance();
    if (!audioContext || !mediaStream?.active) {
        await initializeVoice();
        return;
    }
    await audioContext.resume().catch(() => undefined);
    listening.value = true;
    emit('voiceStatus', '麦克风已开启，请直接说话');
};

const handleVisibilityChange = () => {
    if (!document.hidden) void resumeAutomaticListening();
};

const resumeAutomaticListening = async () => {
    if (disposed || document.hidden || initializing) return;
    if (mediaStream?.active && audioContext) {
        await audioContext.resume().catch(() => undefined);
        if (audioContext.state === 'running') {
            listening.value = !recognizing.value && !loading.value && !props.speechBusy && !props.playbackPending;
            if (listening.value) emit('voiceStatus', '麦克风已开启，请直接说话');
        }
        return;
    }
    if (microphonePermission?.state !== 'denied') await initializeVoice();
};

const handleMicrophonePermissionChange = () => {
    if (microphonePermission?.state === 'granted') {
        emit('voiceError', '');
        void resumeAutomaticListening();
    } else if (microphonePermission?.state === 'denied') {
        reportVoiceError('麦克风权限已被禁止，请在浏览器网站设置中改为允许；授权后会自动开始监听');
    }
};

const monitorMicrophonePermission = async () => {
    if (!navigator.permissions?.query) return;
    try {
        microphonePermission = await navigator.permissions.query({name: 'microphone'});
        microphonePermission.addEventListener('change', handleMicrophonePermissionChange);
    } catch {
        microphonePermission = null;
    }
};

const startAutomaticListening = async () => {
    await monitorMicrophonePermission();
    await resumeAutomaticListening();
};

const handleDeviceChange = () => {
    if (!mediaStream?.active) void resumeAutomaticListening();
};

watch(() => [props.speechBusy, props.playbackPending], ([speechBusy, playbackPending]) => {
    resetUtterance();
    if (speechBusy || playbackPending) {
        listening.value = false;
        emit('voiceStatus', '数字人正在回答');
    } else if (!recognizing.value && !loading.value && mediaStream?.active) {
        listening.value = true;
        emit('voiceStatus', '麦克风已开启，请直接说话');
    }
});

defineExpose({activateVoice});

onMounted(() => {
    document.addEventListener('visibilitychange', handleVisibilityChange);
    window.addEventListener('focus', resumeAutomaticListening);
    window.addEventListener('pageshow', resumeAutomaticListening);
    navigator.mediaDevices?.addEventListener?.('devicechange', handleDeviceChange);
    void startAutomaticListening();
});
onUnmounted(() => {
    disposed = true;
    document.removeEventListener('visibilitychange', handleVisibilityChange);
    window.removeEventListener('focus', resumeAutomaticListening);
    window.removeEventListener('pageshow', resumeAutomaticListening);
    navigator.mediaDevices?.removeEventListener?.('devicechange', handleDeviceChange);
    microphonePermission?.removeEventListener('change', handleMicrophonePermissionChange);
    requestController?.abort();
    asrController?.abort();
    void stopAudioCapture();
});
</script>

<style scoped>
.sr-only {
    position: absolute;
    width: 1px;
    height: 1px;
    padding: 0;
    margin: -1px;
    overflow: hidden;
    clip: rect(0, 0, 0, 0);
    white-space: nowrap;
    border: 0;
}
</style>