flat:4.7暂存

hook/useRealtimeRecorder.js

@@ -3,72 +3,121 @@ import {
     onUnmounted
 } from 'vue';
 
-export function useAudioRecorder(wsUrl) {
+function mergeText(prevText, newText) {
+    if (newText.startsWith(prevText)) {
+        return newText; // 直接替换，避免重复拼接
+    }
+    return prevText + newText; // 兼容意外情况
+}
 
+export function useAudioRecorder(wsUrl) {
     // 状态变量
     const isRecording = ref(false);
     const isStopping = ref(false);
     const isSocketConnected = ref(false);
     const recordingDuration = ref(0);
-    const bufferPressure = ref(0); // 缓冲区压力 (0-100)
-    const currentInterval = ref(0); // 当前发送间隔
+    const audioDataForDisplay = ref(new Array(16).fill(0.01));
+    const volumeLevel = ref(0);
 
     // 音频相关
     const audioContext = ref(null);
     const mediaStream = ref(null);
     const workletNode = ref(null);
+    const analyser = ref(null);
 
     // 网络相关
     const socket = ref(null);
-    const audioBuffer = ref([]);
-    const bufferInterval = ref(null);
 
     // 配置常量
     const SAMPLE_RATE = 16000;
-    const BASE_INTERVAL_MS = 300; // 基础发送间隔
-    const MIN_INTERVAL_MS = 100; // 最小发送间隔
-    const MAX_BUFFER_SIZE = 20; // 最大缓冲区块数
-    const PRESSURE_THRESHOLD = 0.7; // 加快发送的阈值 (70%)
+    const SILENCE_THRESHOLD = 0.02; // 静音阈值 (0-1)
+    const SILENCE_DURATION = 400; // 静音持续时间(ms)后切片
+    const MIN_SOUND_DURATION = 300; // 最小有效声音持续时间(ms)
+
+    // 音频处理变量
+    const lastSoundTime = ref(0);
+    const audioChunks = ref([]);
+    const currentChunkStartTime = ref(0);
+    const silenceStartTime = ref(0);
+
+    // 语音识别结果
+    const recognizedText = ref('');
+    const lastFinalText = ref(''); // 保存最终确认的文本
 
     // AudioWorklet处理器代码
     const workletProcessorCode = `
-  class AudioProcessor extends AudioWorkletProcessor {
-    constructor(options) {
-      super();
-      this.sampleRate = options.processorOptions.sampleRate;
-      this.samplesPerChunk = Math.floor(this.sampleRate * 0.1); // 每100ms的样本数
-      this.buffer = new Int16Array(this.samplesPerChunk);
-      this.index = 0;
-    }
-    
-    process(inputs) {
-      const input = inputs[0];
-      if (input.length > 0) {
-        const inputChannel = input[0];
-        
-        for (let i = 0; i < inputChannel.length; i++) {
-          // 转换为16位PCM
-          this.buffer[this.index++] = Math.max(-32768, Math.min(32767, inputChannel[i] * 32767));
-          
-          // 当缓冲区满时发送
-          if (this.index >= this.samplesPerChunk) {
-            this.port.postMessage({
-              audioData: this.buffer.buffer,
-              timestamp: Date.now()
-            }, [this.buffer.buffer]);
-            
-            // 创建新缓冲区
+    class AudioProcessor extends AudioWorkletProcessor {
+        constructor(options) {
+            super();
+            this.silenceThreshold = options.processorOptions.silenceThreshold;
+            this.sampleRate = options.processorOptions.sampleRate;
+            this.samplesPerChunk = Math.floor(this.sampleRate * 0.05); // 50ms的块
             this.buffer = new Int16Array(this.samplesPerChunk);
             this.index = 0;
-          }
+            this.lastUpdate = 0;
+        }
+        
+        calculateVolume(inputs) {
+            const input = inputs[0];
+            if (!input || input.length === 0) return 0;
+            
+            let sum = 0;
+            const inputChannel = input[0];
+            for (let i = 0; i < inputChannel.length; i++) {
+                sum += inputChannel[i] * inputChannel[i];
+            }
+            return Math.sqrt(sum / inputChannel.length);
+        }
+        
+        process(inputs) {
+            const now = currentTime;
+            const volume = this.calculateVolume(inputs);
+            
+            // 每50ms发送一次分析数据
+            if (now - this.lastUpdate > 0.05) {
+                this.lastUpdate = now;
+                
+                // 简单的频率分析 (模拟16个频段)
+                const simulatedFreqData = [];
+                for (let i = 0; i < 16; i++) {
+                    simulatedFreqData.push(
+                        Math.min(1, volume * 10 + (Math.random() * 0.2 - 0.1))
+                    );
+                }
+                
+                this.port.postMessage({
+                    type: 'analysis',
+                    volume: volume,
+                    frequencyData: simulatedFreqData,
+                    isSilent: volume < this.silenceThreshold,
+                    timestamp: now
+                });
+            }
+            
+            // 原始音频处理
+            const input = inputs[0];
+            if (input && input.length > 0) {
+                const inputChannel = input[0];
+                for (let i = 0; i < inputChannel.length; i++) {
+                    this.buffer[this.index++] = Math.max(-32768, Math.min(32767, inputChannel[i] * 32767));
+                    
+                    if (this.index >= this.samplesPerChunk) {
+                        this.port.postMessage({
+                            type: 'audio',
+                            audioData: this.buffer.buffer,
+                            timestamp: now
+                        }, [this.buffer.buffer]);
+                        
+                        this.buffer = new Int16Array(this.samplesPerChunk);
+                        this.index = 0;
+                    }
+                }
+            }
+            return true;
         }
-      }
-      return true;
     }
-  }
-  
-  registerProcessor('audio-processor', AudioProcessor);
-  `;
+    registerProcessor('audio-processor', AudioProcessor);
+    `;
 
     // 初始化WebSocket连接
     const initSocket = (wsUrl) => {
@@ -77,77 +126,91 @@ export function useAudioRecorder(wsUrl) {
 
             socket.value.onopen = () => {
                 isSocketConnected.value = true;
-                console.log('WebSocket连接已建立');
                 resolve();
             };
 
             socket.value.onerror = (error) => {
-                console.error('WebSocket连接错误:', error);
                 reject(error);
             };
 
-            socket.value.onclose = (event) => {
-                console.log(`WebSocket连接关闭，代码: ${event.code}, 原因: ${event.reason}`);
+            socket.value.onclose = () => {
                 isSocketConnected.value = false;
-                console.log('WebSocket连接已关闭');
             };
+
+            socket.value.onmessage = handleMessage;
         });
     };
 
-    // 计算动态发送间隔
-    const calculateDynamicInterval = () => {
-        const pressureFactor = bufferPressure.value / 100;
-        // 压力越大，间隔越小（发送越快）
-        return Math.max(
-            MIN_INTERVAL_MS,
-            BASE_INTERVAL_MS - (pressureFactor * (BASE_INTERVAL_MS - MIN_INTERVAL_MS))
-        );
+    const handleMessage = (values) => {
+        try {
+            const data = JSON.parse(event.data);
+            if (data.text) {
+                const {
+                    asrEnd,
+                    text
+                } = data
+                if (asrEnd === 'true') {
+                    recognizedText.value += data.text;
+                } else {
+                    lastFinalText.value = '';
+                }
+            }
+        } catch (error) {
+            console.error('解析识别结果失败:', error);
+        }
+    }
+
+    // 处理音频切片
+    const processAudioChunk = (isSilent) => {
+        const now = Date.now();
+
+        if (!isSilent) {
+            // 检测到声音
+            lastSoundTime.value = now;
+
+            if (silenceStartTime.value > 0) {
+                // 从静音恢复到有声音
+                silenceStartTime.value = 0;
+            }
+        } else {
+            // 静音状态
+            if (silenceStartTime.value === 0) {
+                silenceStartTime.value = now;
+            }
+
+            // 检查是否达到静音切片条件
+            if (now - silenceStartTime.value >= SILENCE_DURATION &&
+                now - currentChunkStartTime.value >= MIN_SOUND_DURATION) {
+                sendCurrentChunk();
+            }
+        }
     };
 
-    // 发送缓冲的音频数据
-    const sendBufferedAudio = () => {
-        if (audioBuffer.value.length === 0 || !socket.value || socket.value.readyState !== WebSocket.OPEN) {
+    // 发送当前音频块
+    const sendCurrentChunk = () => {
+        if (audioChunks.value.length === 0 || !socket.value || socket.value.readyState !== WebSocket.OPEN) {
             return;
         }
 
         try {
-            // 将缓冲区大小限制为8000字节 (小于8192)
-            const MAX_CHUNK_SIZE = 8000 / 2; // 16位 = 2字节，所以4000个样本
+            // 合并所有块
+            const totalBytes = audioChunks.value.reduce((total, chunk) => total + chunk.byteLength, 0);
+            const combined = new Int16Array(totalBytes / 2);
+            let offset = 0;
 
-            let samplesToSend = [];
-            let totalSamples = 0;
+            audioChunks.value.forEach(chunk => {
+                const samples = new Int16Array(chunk);
+                combined.set(samples, offset);
+                offset += samples.length;
+            });
 
-            // 收集不超过限制的样本
-            while (audioBuffer.value.length > 0 && totalSamples < MAX_CHUNK_SIZE) {
-                const buffer = audioBuffer.value[0];
-                const samples = new Int16Array(buffer);
-                const remainingSpace = MAX_CHUNK_SIZE - totalSamples;
-
-                if (samples.length <= remainingSpace) {
-                    samplesToSend.push(samples);
-                    totalSamples += samples.length;
-                    audioBuffer.value.shift();
-                } else {
-                    // 只取部分样本
-                    samplesToSend.push(samples.slice(0, remainingSpace));
-                    audioBuffer.value[0] = samples.slice(remainingSpace).buffer;
-                    totalSamples = MAX_CHUNK_SIZE;
-                }
-            }
-
-            // 合并样本并发送
-            if (totalSamples > 0) {
-                const combined = new Int16Array(totalSamples);
-                let offset = 0;
-
-                samplesToSend.forEach(chunk => {
-                    combined.set(chunk, offset);
-                    offset += chunk.length;
-                });
-
-                socket.value.send(combined.buffer);
-            }
+            // 发送合并后的数据
+            socket.value.send(combined.buffer);
+            audioChunks.value = [];
 
+            // 记录新块的开始时间
+            currentChunkStartTime.value = Date.now();
+            silenceStartTime.value = 0;
         } catch (error) {
             console.error('发送音频数据时出错:', error);
         }
@@ -158,31 +221,34 @@ export function useAudioRecorder(wsUrl) {
         if (isRecording.value) return;
 
         try {
+            // 重置状态
+            recognizedText.value = '';
+            lastFinalText.value = '';
             // 重置状态
             recordingDuration.value = 0;
-            audioBuffer.value = [];
-            bufferPressure.value = 0;
-            currentInterval.value = BASE_INTERVAL_MS;
-            console.log('正在初始化WebSocket连接...');
+            audioChunks.value = [];
+            lastSoundTime.value = 0;
+            currentChunkStartTime.value = Date.now();
+            silenceStartTime.value = 0;
+
             // 初始化WebSocket
             await initSocket(wsUrl);
-            console.log('正在获取音频设备权限...');
+
             // 获取音频流
             mediaStream.value = await navigator.mediaDevices.getUserMedia({
                 audio: {
                     sampleRate: SAMPLE_RATE,
                     channelCount: 1,
-                    echoCancellation: false,
-                    noiseSuppression: false,
+                    echoCancellation: true,
+                    noiseSuppression: true,
                     autoGainControl: false
                 },
                 video: false
             });
-            console.log('正在初始化音频上下文...');
+
             // 创建音频上下文
             audioContext.value = new(window.AudioContext || window.webkitAudioContext)({
-                sampleRate: SAMPLE_RATE,
-                latencyHint: 'interactive'
+                sampleRate: SAMPLE_RATE
             });
 
             // 注册AudioWorklet
@@ -195,23 +261,20 @@ export function useAudioRecorder(wsUrl) {
 
             // 创建AudioWorkletNode
             workletNode.value = new AudioWorkletNode(audioContext.value, 'audio-processor', {
-                numberOfInputs: 1,
-                numberOfOutputs: 1,
-                outputChannelCount: [1],
                 processorOptions: {
+                    silenceThreshold: SILENCE_THRESHOLD,
                     sampleRate: SAMPLE_RATE
                 }
             });
 
             // 处理音频数据
             workletNode.value.port.onmessage = (e) => {
-                if (e.data.audioData instanceof ArrayBuffer) {
-                    audioBuffer.value.push(e.data.audioData);
-
-                    // 当缓冲区压力超过阈值时立即尝试发送
-                    if (audioBuffer.value.length / MAX_BUFFER_SIZE > PRESSURE_THRESHOLD) {
-                        sendBufferedAudio();
-                    }
+                if (e.data.type === 'audio') {
+                    audioChunks.value.push(e.data.audioData);
+                } else if (e.data.type === 'analysis') {
+                    audioDataForDisplay.value = e.data.frequencyData;
+                    volumeLevel.value = e.data.volume;
+                    processAudioChunk(e.data.isSilent);
                 }
             };
 
@@ -220,12 +283,7 @@ export function useAudioRecorder(wsUrl) {
             source.connect(workletNode.value);
             workletNode.value.connect(audioContext.value.destination);
 
-            // 启动定时发送
-            bufferInterval.value = setInterval(sendBufferedAudio, currentInterval.value);
-            console.log('录音初始化完成，开始录制');
-            // 更新状态
             isRecording.value = true;
-            console.log(`开始录音，采样率: ${audioContext.value.sampleRate}Hz`);
 
         } catch (error) {
             console.error('启动录音失败:', error);
@@ -234,39 +292,27 @@ export function useAudioRecorder(wsUrl) {
         }
     };
 
-    // 停止录音并保存
+    // 停止录音
     const stopRecording = async () => {
         if (!isRecording.value || isStopping.value) return;
 
         isStopping.value = true;
 
         try {
-            // 停止定时器
-            if (bufferInterval.value) {
-                clearInterval(bufferInterval.value);
-                bufferInterval.value = null;
-            }
-
-            // 发送剩余音频数据
-            if (audioBuffer.value.length > 0) {
-                console.log(`正在发送剩余 ${audioBuffer.value.length} 个音频块...`);
-                sendBufferedAudio();
-            }
+            // 发送最后一个音频块（无论是否静音）
+            sendCurrentChunk();
 
             // 发送结束标记
             if (socket.value?.readyState === WebSocket.OPEN) {
-                console.log('发送结束标记...');
                 socket.value.send(JSON.stringify({
                     action: 'end',
                     duration: recordingDuration.value
                 }));
 
-                // 等待数据发送完成
-                await new Promise((resolve) => {
+                await new Promise(resolve => {
                     if (socket.value.bufferedAmount === 0) {
                         resolve();
                     } else {
-                        console.log(`等待 ${socket.value.bufferedAmount} 字节数据发送...`);
                         const timer = setInterval(() => {
                             if (socket.value.bufferedAmount === 0) {
                                 clearInterval(timer);
@@ -275,14 +321,10 @@ export function useAudioRecorder(wsUrl) {
                         }, 50);
                     }
                 });
-
-                // 关闭连接
-                console.log('正在关闭WebSocket连接...');
                 socket.value.close();
             }
 
             cleanup();
-            console.log('录音已停止并保存');
 
         } catch (error) {
             console.error('停止录音时出错:', error);
@@ -292,20 +334,33 @@ export function useAudioRecorder(wsUrl) {
         }
     };
 
-    // 取消录音
+    // 清理资源
+    const cleanup = () => {
+        if (mediaStream.value) {
+            mediaStream.value.getTracks().forEach(track => track.stop());
+            mediaStream.value = null;
+        }
+
+        if (workletNode.value) {
+            workletNode.value.disconnect();
+            workletNode.value = null;
+        }
+
+        if (audioContext.value && audioContext.value.state !== 'closed') {
+            audioContext.value.close();
+            audioContext.value = null;
+        }
+
+        audioChunks.value = [];
+        isRecording.value = false;
+        isSocketConnected.value = false;
+    };
+
+    /// 取消录音
     const cancelRecording = async () => {
         if (!isRecording.value || isStopping.value) return;
-
         isStopping.value = true;
-
         try {
-            // 停止定时器
-            if (bufferInterval.value) {
-                clearInterval(bufferInterval.value);
-                bufferInterval.value = null;
-            }
-
-            // 发送取消标记
             if (socket.value?.readyState === WebSocket.OPEN) {
                 console.log('发送结束标记...');
                 socket.value.send(JSON.stringify({
@@ -313,11 +368,7 @@ export function useAudioRecorder(wsUrl) {
                 }));
                 socket.value.close();
             }
-            console.log('清理资源...');
-            cleanup();
-            console.log('录音已成功停止');
-            console.log('录音已取消');
-
+            cleanup()
         } catch (error) {
             console.error('取消录音时出错:', error);
             throw error;
@@ -326,51 +377,9 @@ export function useAudioRecorder(wsUrl) {
         }
     };
 
-    // 清理资源
-    const cleanup = () => {
-        // 清除定时器
-        if (bufferInterval.value) {
-            clearInterval(bufferInterval.value);
-            bufferInterval.value = null;
-        }
-
-        // 关闭音频流
-        if (mediaStream.value) {
-            console.log('正在停止媒体流...');
-            mediaStream.value.getTracks().forEach(track => track.stop());
-            mediaStream.value = null;
-        }
-
-        // 断开音频节点
-        if (workletNode.value) {
-            workletNode.value.disconnect();
-            workletNode.value.port.onmessage = null;
-            workletNode.value = null;
-        }
-
-        // 关闭音频上下文
-        if (audioContext.value) {
-            if (audioContext.value.state !== 'closed') {
-                audioContext.value.close().catch(e => {
-                    console.warn('关闭AudioContext时出错:', e);
-                });
-            }
-            audioContext.value = null;
-        }
-
-        // 清空缓冲区
-        audioBuffer.value = [];
-        bufferPressure.value = 0;
-
-        // 重置状态
-        isRecording.value = false;
-        isSocketConnected.value = false;
-    };
-
-    // 组件卸载时自动清理
     onUnmounted(() => {
         if (isRecording.value) {
-            cancelRecording();
+            stopRecording();
         }
     });
 
@@ -379,97 +388,12 @@ export function useAudioRecorder(wsUrl) {
         isStopping,
         isSocketConnected,
         recordingDuration,
-        bufferPressure,
-        currentInterval,
+        audioDataForDisplay,
+        volumeLevel,
         startRecording,
         stopRecording,
+        recognizedText,
+        lastFinalText,
         cancelRecording
     };
-}
-
-
-// import {
-//     ref
-// } from 'vue'
-// export function useRealtimeRecorder(wsUrl) {
-//     const isRecording = ref(false)
-//     const mediaRecorder = ref(null)
-//     const socket = ref(null)
-//     const recognizedText = ref('')
-
-//     const startRecording = async () => {
-//         if (!navigator.mediaDevices?.getUserMedia) {
-//             uni.showToast({
-//                 title: '当前环境不支持录音',
-//                 icon: 'none'
-//             })
-//             return
-//         }
-//         recognizedText.value = ''
-//         const stream = await navigator.mediaDevices.getUserMedia({
-//             audio: {
-//                 sampleRate: 16000,
-//                 channelCount: 1,
-//                 echoCancellation: false,
-//                 noiseSuppression: false,
-//                 autoGainControl: false
-//             },
-//             video: false
-//         })
-//         socket.value = new WebSocket(wsUrl)
-
-//         socket.value.onopen = () => {
-//             console.log('[WebSocket] 连接已建立')
-//         }
-
-//         socket.value.onmessage = (event) => {
-//             recognizedText.value = JSON.parse(event.data).text
-//         }
-
-//         const recorder = new MediaRecorder(stream, {
-//             mimeType: 'audio/webm;codecs=opus',
-//             audioBitsPerSecond: 16000,
-//         })
-
-//         recorder.ondataavailable = (e) => {
-//             if (e.data.size > 0 && socket.value?.readyState === WebSocket.OPEN) {
-//                 socket.value.send(e.data)
-//             }
-//         }
-
-//         recorder.start(300) // 每 300ms 发送一段数据
-//         mediaRecorder.value = recorder
-//         isRecording.value = true
-//     }
-
-//     const stopRecording = () => {
-//         mediaRecorder.value?.stop()
-//         mediaRecorder.value = null
-//         isRecording.value = false
-
-//         if (socket.value?.readyState === WebSocket.OPEN) {
-//             socket.value.send('[end]')
-//             socket.value.close()
-//         }
-//     }
-
-//     const cancelRecording = () => {
-//         mediaRecorder.value?.stop()
-//         mediaRecorder.value = null
-//         isRecording.value = false
-//         recognizedText.value = ''
-
-//         if (socket.value?.readyState === WebSocket.OPEN) {
-//             socket.value.send('[cancel]')
-//             socket.value.close()
-//         }
-//     }
-
-//     return {
-//         isRecording,
-//         recognizedText,
-//         startRecording,
-//         stopRecording,
-//         cancelRecording
-//     }
-// }
+}

hook/useSpeechReader.js

@@ -0,0 +1,109 @@
+import {
+    ref,
+    onBeforeUnmount,
+    onMounted
+} from 'vue'
+import {
+    onHide,
+    onUnload
+} from '@dcloudio/uni-app'
+
+function formatTextForSpeech(rawText) {
+    return rawText
+        // 去除链接 markdown 格式 [xxx](url)
+        .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
+        // 去除 Markdown 语法符号
+        .replace(/[*_`>#\-]/g, '')
+        // 将换行转换为句号
+        .replace(/\n+/g, '。')
+        // 多个标点统一转为句号（表示停顿）
+        .replace(/[。，,.、!！?？；;：:~～…··\-\/\\]{1,}/g, '。')
+        // 合并多个句号
+        .replace(/([。]{2,})/g, '。')
+        // 去除多余空格
+        .replace(/\s+/g, ' ')
+        // 去除开头结尾的句号
+        .replace(/^[。]+|[。]+$/g, '')
+        .trim()
+}
+
+export function useSpeechReader() {
+    const isSpeaking = ref(false)
+    const isPaused = ref(false)
+    let utterance = null
+
+    const cleanMarkdown = (text) => {
+        return formatTextForSpeech(text)
+    }
+
+    const speak = (text, options = {
+        lang: 'zh-CN',
+        rate: 0.9,
+        pitch: 1.2
+    }) => {
+        cancel() // 重置之前的
+        const voices = speechSynthesis.getVoices()
+        const chineseVoices = voices.filter(v => v.lang.includes('zh'))
+        // console.log(chineseVoices.map((item) => item.name))
+        const cleanText = cleanMarkdown(text)
+        utterance = new SpeechSynthesisUtterance(cleanText)
+        // utterance.voice = chineseVoices.find(v => v.name === 'Shelley')
+        utterance.lang = options.lang || 'zh-CN'
+        utterance.rate = options.rate || 1
+        utterance.pitch = options.pitch || 1.1 // 音调（0 - 2，偏高比较柔和）
+
+        utterance.onend = () => {
+            isSpeaking.value = false
+            isPaused.value = false
+        }
+
+        speechSynthesis.speak(utterance)
+        isSpeaking.value = true
+        isPaused.value = false
+    }
+
+    const pause = () => {
+        if (isSpeaking.value && !isPaused.value) {
+            speechSynthesis.pause()
+            isPaused.value = true
+        }
+    }
+
+    const resume = () => {
+        if (isSpeaking.value && isPaused.value) {
+            speechSynthesis.resume()
+            isPaused.value = false
+        }
+    }
+
+    const cancel = () => {
+        speechSynthesis.cancel()
+        isSpeaking.value = false
+        isPaused.value = false
+    }
+    // 页面刷新/关闭时
+    onMounted(() => {
+        if (typeof window !== 'undefined') {
+            window.addEventListener('beforeunload', cancel)
+        }
+    })
+
+    onBeforeUnmount(() => {
+        cancel()
+        if (typeof window !== 'undefined') {
+            window.removeEventListener('beforeunload', cancel)
+        }
+    })
+
+    onHide(cancel)
+    onUnload(cancel)
+
+    return {
+        speak,
+        pause,
+        resume,
+        cancel,
+        isSpeaking,
+        isPaused,
+    }
+}