Agent Zero AI framework
STT continued
frdel1a91334a420f
diff --git a/webui/index.css b/webui/index.cssindex 7f9b8c7b8..fde8a6c4e 100644--- a/webui/index.css+++ b/webui/index.css@@ -907,35 +907,6 @@ pre {background-color: var(--color-primary-light);}-/* MIC BUTTON */-#microphone-button {- background-color: #3270e2;-}--#microphone-button:hover {- background-color: #4382e8;-}-#microphone-button.recording {- background-color: #ff4136; /* Red color for recording */- transition: background-color 0.3s ease;-}--.mic-pulse {- animation: pulse 1.5s infinite;-}--@keyframes pulse {- 0% {- transform: scale(1);- }- 50% {- transform: scale(1.1);- }- 100% {- transform: scale(1);- }-}-.chat-button svg {width: 1.5rem;height: 1.5rem;diff --git a/webui/index.html b/webui/index.htmlindex 65b4d127a..309670d60 100644--- a/webui/index.html+++ b/webui/index.html@@ -8,6 +8,7 @@<link rel="stylesheet" href="index.css"><link rel="stylesheet" href="toast.css"><link rel="stylesheet" href="settings.css">+ <link rel="stylesheet" href="speech.css"><script>window.safeCall = function (name, ...args) {@@ -236,7 +237,7 @@</button><!-- Microphone button -->- <button class="chat-button" id="microphone-button" aria-label="Start/Stop recording">+ <button class="chat-button mic-inactive" id="microphone-button" aria-label="Start/Stop recording"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24" height="24"><path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z"/></svg>diff --git a/webui/speech copy.js b/webui/speech copy.jsnew file mode 100644index 000000000..eb2050075--- /dev/null+++ b/webui/speech copy.js@@ -0,0 +1,337 @@+import { pipeline, read_audio } from './[email protected]';+import { updateChatInput, sendMessage } from './index.js';++const microphoneButton = document.getElementById('microphone-button');+let microphoneInput = null;+let isProcessingClick = false;++class MicrophoneInput {+ /**+ * Voice Input Handler with Whisper Transcription+ *+ * Whisper Model Size Configuration:+ * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)+ * - Best for: Quick prototyping, low-resource environments+ * - Pros: Very fast, low memory usage+ * - Cons: Lowest transcription accuracy+ *+ * - 'base': Small model, good balance of speed and accuracy (~74MB)+ * - Best for: General-purpose voice input+ * - Pros: Reasonable accuracy, moderate resource usage+ * - Cons: Less accurate than larger models+ *+ * - 'small': Medium-sized model, better accuracy (~244MB)+ * - Best for: More precise transcription needs+ * - Pros: Improved accuracy over base model+ * - Cons: Slower, more memory-intensive+ *+ * - 'medium': Large model with high accuracy (~769MB)+ * - Best for: Professional transcription, multi-language support+ * - Pros: Very high accuracy+ * - Cons: Significant computational resources required+ *+ * - 'large': Largest model, highest accuracy (~1.5GB)+ * - Best for: Professional, multi-language transcription+ * - Pros: Highest possible accuracy+ * - Cons: Slowest, most resource-intensive+ *+ * Recommended Default: 'base' for most web applications+ */+ constructor(updateCallback, options = {}) {+ this.mediaRecorder = null;+ this.audioChunks = [];+ this.isRecording = false;+ this.updateCallback = updateCallback;+ this.isFinalizing = false;+ this.messageSent = false; // move messageSent into class++ // New properties for silence detection+ this.audioContext = null;+ this.mediaStreamSource = null;+ this.analyserNode = null;+ this.silenceTimer = null;+ this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed+ this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds++ this.options = {+ modelSize: 'tiny',+ language: 'en',+ chunkDuration: 3000,+ ...options+ };+ }++ async initialize() {+ try {++ this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);++ const stream = await navigator.mediaDevices.getUserMedia({+ audio: {+ echoCancellation: true,+ noiseSuppression: true,+ channelCount: 1+ }+ });++ // Configure MediaRecorder+ this.mediaRecorder = new MediaRecorder(stream);++ // Handle audio data availability+ this.mediaRecorder.ondataavailable = async (event) => {+ if (event.data.size > 0) {+ this.audioChunks.push(event.data);+ // await this.processAudioChunk(event.data);+ }+ };++ // Handle recording stop+ this.mediaRecorder.onstop = async () => {+ await this.finalizeRecording();+ };++ // Set up AudioContext and AnalyserNode for silence detection+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);+ this.analyserNode = this.audioContext.createAnalyser();+ this.analyserNode.minDecibels = -90;+ this.analyserNode.maxDecibels = -10;+ this.analyserNode.smoothingTimeConstant = 0.85;++ this.mediaStreamSource.connect(this.analyserNode);+ } catch (error) {+ console.error('Microphone initialization error:', error);+ toast('Failed to access microphone. Please check permissions.', 'error');+ }+ }++ startSilenceDetection() {+ const dataArray = new Uint8Array(this.analyserNode.fftSize);+ const checkSilence = () => {+ this.analyserNode.getByteTimeDomainData(dataArray);++ let sum = 0;+ for (let i = 0; i < dataArray.length; i++) {+ const amplitude = (dataArray[i] - 128) / 128;+ sum += amplitude * amplitude;+ }+ const rms = Math.sqrt(sum / dataArray.length);++ if (rms < this.silenceThreshold) {+ if (!this.silenceTimer) {+ this.silenceTimer = setTimeout(() => {+ if (this.isRecording) {+ console.log('Silence detected. Stopping recording.');+ this.stopRecording();+ microphoneButton.classList.remove('recording');+ microphoneButton.classList.remove('mic-pulse');+ }+ }, this.silenceDuration);+ }+ } else {+ if (this.silenceTimer) {+ clearTimeout(this.silenceTimer);+ this.silenceTimer = null;+ }+ }++ if (this.isRecording) {+ requestAnimationFrame(checkSilence);+ }+ };++ if (this.isRecording) {+ requestAnimationFrame(checkSilence);+ }+ }++ startRecording() {+ if (this.mediaRecorder && this.audioContext) {+ this.isRecording = true;+ this.audioChunks = [];+ this.messageSent = false;+ this.mediaRecorder.start(this.options.chunkDuration);+ this.audioContext.resume();+ this.startSilenceDetection();+ }+ }++ stopRecording() {+ if (this.mediaRecorder && this.isRecording) {+ this.isRecording = false;+ if (!this.isFinalizing) {+ this.isFinalizing = true;+ this.mediaRecorder.stop();+ this.audioContext.suspend();+ if (this.silenceTimer) {+ clearTimeout(this.silenceTimer);+ this.silenceTimer = null;+ }+ }+ }+ }+++ async finalizeRecording() {+ if (this.isFinalizing) {+ this.isFinalizing = false;++ if (this.audioChunks.length > 0) {++ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });+ const audioUrl = URL.createObjectURL(audioBlob);+ const samplingRate = 16000; // Adjust as needed for the model+ const audioData = await read_audio(audioUrl, samplingRate);+ URL.revokeObjectURL(audioUrl);++ // Transcribe the audio+ const result = await this.transcriber(audioData);++ if (result.text) {+ console.log('Final transcription received:', result.text);+ await this.updateCallback(result.text, true);+ } else {+ console.warn('Final transcription returned empty text.');+ }+++ // Release the object URL after use++ // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });+ // this.audioChunks = []; // Clear for next recording++ // const reader = new FileReader();+ // reader.onloadend = async () => {+ // const base64Data = reader.result.split(',')[1];++ // try {+ // const response = await fetch('/transcribe', {+ // method: 'POST',+ // headers: {+ // 'Content-Type': 'application/json'+ // },+ // body: JSON.stringify({+ // audio_data: base64Data,+ // model_size: this.options.modelSize,+ // language: this.options.language,+ // is_final: true+ // })+ // });++ // const result = await response.json();++ // if (result.text) {+ // console.log('Final transcription received:', result.text);+ // await this.updateCallback(result.text, true);+ // } else {+ // console.warn('Final transcription returned empty text.');+ // }+ // } catch (transcribeError) {+ // console.error('Final transcription error:', transcribeError);+ // toast('Final transcription failed.', 'error');+ // } finally {+ // // Reset the microphone button state+ // microphoneButton.classList.remove('recording');+ // microphoneButton.classList.remove('mic-pulse');+ // microphoneButton.style.backgroundColor = '';+ // }+ // };+ // reader.readAsDataURL(audioBlob);+ }+ }+ }+}++export default MicrophoneInput;++async function initializeMicrophoneInput() {+ console.log('Initializing microphone input');++ microphoneInput = new MicrophoneInput(+ async (text, isFinal) => {+ if (isFinal) {+ console.log('Final transcription callback received:', text);+ updateChatInput(text)+ // chatInput.value = text;+ // adjustTextareaHeight();++ if (!microphoneInput.messageSent) {+ microphoneInput.messageSent = true;+ console.log('Sending message');+ await sendMessage();+ }+ }+ },+ {+ modelSize: 'tiny',+ language: 'en',+ silenceThreshold: 0.07, // Adjust as needed+ silenceDuration: 2000, // Adjust as needed+ onError: (error) => {+ console.error('Microphone input error:', error);+ toast('Microphone error: ' + error.message, 'error');+ // Reset recording state+ if (microphoneButton.classList.contains('recording')) {+ microphoneButton.classList.remove('recording');+ }+ }+ }+ );++ await microphoneInput.initialize();+}+++function toggleRecording() {+ console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);++ if (microphoneInput.isRecording) {+ microphoneInput.stopRecording();+ microphoneButton.classList.remove('recording');+ // Add pulsing animation class+ microphoneButton.classList.remove('mic-pulse');+ } else {+ microphoneInput.startRecording();+ microphoneButton.classList.add('recording');+ // Add pulsing animation class+ microphoneButton.classList.add('mic-pulse');+ }++ // Add visual feedback+ microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';+ console.log('New recording state:', microphoneInput.isRecording);+}++// Some error handling for microphone input+async function requestMicrophonePermission() {+ try {+ await navigator.mediaDevices.getUserMedia({ audio: true });+ return true;+ } catch (err) {+ console.error('Error accessing microphone:', err);+ toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');+ return false;+ }+}+// microphoneButton click event listener modifier+microphoneButton.addEventListener('click', async () => {+ console.log('Microphone button clicked');+ if (isProcessingClick) {+ console.log('Click already being processed, ignoring');+ return;+ }+ isProcessingClick = true;++ const hasPermission = await requestMicrophonePermission();+ if (!hasPermission) return;++ if (!microphoneInput) {+ await initializeMicrophoneInput();+ }++ await toggleRecording();++ setTimeout(() => {+ isProcessingClick = false;+ }, 300); // Add a 300ms delay before allowing another click+});diff --git a/webui/speech.css b/webui/speech.cssnew file mode 100644index 000000000..07c66d2a0--- /dev/null+++ b/webui/speech.css@@ -0,0 +1,55 @@+/* MIC BUTTON */+#microphone-button {++}++#microphone-button:hover {+}++#microphone-button.recording {+ background-color: #ff4136; /* Red color for recording */+ transition: background-color 0.3s ease;+}++@keyframes pulse {+ 0% {+ transform: scale(1);+ }+ 50% {+ transform: scale(1.1);+ }+ 100% {+ transform: scale(1);+ }+ }++.mic-pulse {+ animation: pulse 1.5s infinite;+}+++.mic-inactive{+ background-color: grey;+}++.mic-activating{+ background-color: silver;+ animation: pulse 0.8s infinite;+}++.mic-listening {+ background-color: red;+}++.mic-recording {+ background-color: green;+}++.mic-waiting {+ background-color: teal;+}++.mic-processing {+ background-color: darkcyan;+ animation: pulse 0.8s infinite;+}\ No newline at end of filediff --git a/webui/speech.js b/webui/speech.jsindex eb2050075..f7dd0055d 100644--- a/webui/speech.js+++ b/webui/speech.js@@ -5,66 +5,149 @@ const microphoneButton = document.getElementById('microphone-button');let microphoneInput = null;let isProcessingClick = false;+const Status = {+ INACTIVE: 'inactive',+ ACTIVATING: 'activating',+ LISTENING: 'listening',+ RECORDING: 'recording',+ WAITING: 'waiting',+ PROCESSING: 'processing'+};+class MicrophoneInput {- /**- * Voice Input Handler with Whisper Transcription- *- * Whisper Model Size Configuration:- * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)- * - Best for: Quick prototyping, low-resource environments- * - Pros: Very fast, low memory usage- * - Cons: Lowest transcription accuracy- *- * - 'base': Small model, good balance of speed and accuracy (~74MB)- * - Best for: General-purpose voice input- * - Pros: Reasonable accuracy, moderate resource usage- * - Cons: Less accurate than larger models- *- * - 'small': Medium-sized model, better accuracy (~244MB)- * - Best for: More precise transcription needs- * - Pros: Improved accuracy over base model- * - Cons: Slower, more memory-intensive- *- * - 'medium': Large model with high accuracy (~769MB)- * - Best for: Professional transcription, multi-language support- * - Pros: Very high accuracy- * - Cons: Significant computational resources required- *- * - 'large': Largest model, highest accuracy (~1.5GB)- * - Best for: Professional, multi-language transcription- * - Pros: Highest possible accuracy- * - Cons: Slowest, most resource-intensive- *- * Recommended Default: 'base' for most web applications- */constructor(updateCallback, options = {}) {this.mediaRecorder = null;this.audioChunks = [];- this.isRecording = false;+ this.lastChunk = [];this.updateCallback = updateCallback;- this.isFinalizing = false;- this.messageSent = false; // move messageSent into class+ this.messageSent = false;- // New properties for silence detection+ // Audio analysis propertiesthis.audioContext = null;this.mediaStreamSource = null;this.analyserNode = null;- this.silenceTimer = null;- this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed- this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds+ this._status = Status.INACTIVE;++ // Timing properties+ this.lastAudioTime = null;+ this.waitingTimer = null;+ this.silenceStartTime = null;+ this.hasStartedRecording = false;+ this.analysisFrame = null;this.options = {modelSize: 'tiny',language: 'en',- chunkDuration: 3000,+ silenceThreshold: 0.15,+ silenceDuration: 1000,+ waitingTimeout: 2000,+ minSpeechDuration: 500,...options};}+ get status() {+ return this._status;+ }++ set status(newStatus) {+ if (this._status === newStatus) return;++ const oldStatus = this._status;+ this._status = newStatus;+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);++ // Update UI+ microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);+ microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);+ microphoneButton.setAttribute('data-status', newStatus);++ // Handle state-specific behaviors+ this.handleStatusChange(oldStatus, newStatus);+ }++ handleStatusChange(oldStatus, newStatus) {++ //last chunk kept only for transition to recording status+ if (newStatus != Status.RECORDING) { this.lastChunk = null; }++ switch (newStatus) {+ case Status.INACTIVE:+ this.handleInactiveState();+ break;+ case Status.LISTENING:+ this.handleListeningState();+ break;+ case Status.RECORDING:+ this.handleRecordingState();+ break;+ case Status.WAITING:+ this.handleWaitingState();+ break;+ case Status.PROCESSING:+ this.handleProcessingState();+ break;+ }+ }++ handleInactiveState() {+ this.stopRecording();+ this.stopAudioAnalysis();+ if (this.waitingTimer) {+ clearTimeout(this.waitingTimer);+ this.waitingTimer = null;+ }+ }++ handleListeningState() {+ this.stopRecording();+ this.audioChunks = [];+ this.hasStartedRecording = false;+ this.silenceStartTime = null;+ this.lastAudioTime = null;+ this.messageSent = false;+ this.startAudioAnalysis();+ }++ handleRecordingState() {+ if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {+ this.hasStartedRecording = true;+ this.mediaRecorder.start(1000);+ console.log('Speech started');+ }+ if (this.waitingTimer) {+ clearTimeout(this.waitingTimer);+ this.waitingTimer = null;+ }+ }++ handleWaitingState() {+ // Don't stop recording during waiting state+ this.waitingTimer = setTimeout(() => {+ if (this.status === Status.WAITING) {+ this.status = Status.PROCESSING;+ }+ }, this.options.waitingTimeout);+ }++ handleProcessingState() {+ this.stopRecording();+ this.process();+ }++ stopRecording() {+ if (this.mediaRecorder?.state === 'recording') {+ this.mediaRecorder.stop();+ this.hasStartedRecording = false;+ }+ }+async initialize() {try {-- this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);+ this.transcriber = await pipeline(+ 'automatic-speech-recognition',+ `Xenova/whisper-${this.options.modelSize}.${this.options.language}`+ );const stream = await navigator.mediaDevices.getUserMedia({audio: {@@ -74,42 +157,51 @@ class MicrophoneInput {}});- // Configure MediaRecorderthis.mediaRecorder = new MediaRecorder(stream);-- // Handle audio data availability- this.mediaRecorder.ondataavailable = async (event) => {- if (event.data.size > 0) {+ this.mediaRecorder.ondataavailable = (event) => {+ if (event.data.size > 0 &&+ (this.status === Status.RECORDING || this.status === Status.WAITING)) {+ if (this.lastChunk) {+ this.audioChunks.push(this.lastChunk);+ this.lastChunk = null;+ }this.audioChunks.push(event.data);- // await this.processAudioChunk(event.data);+ console.log('Audio chunk received, total chunks:', this.audioChunks.length);+ }+ else if (this.status === Status.LISTENING) {+ this.lastChunk = event.data;}};- // Handle recording stop- this.mediaRecorder.onstop = async () => {- await this.finalizeRecording();- };-- // Set up AudioContext and AnalyserNode for silence detection- this.audioContext = new (window.AudioContext || window.webkitAudioContext)();- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);- this.analyserNode = this.audioContext.createAnalyser();- this.analyserNode.minDecibels = -90;- this.analyserNode.maxDecibels = -10;- this.analyserNode.smoothingTimeConstant = 0.85;-- this.mediaStreamSource.connect(this.analyserNode);+ this.setupAudioAnalysis(stream);+ return true;} catch (error) {+console.error('Microphone initialization error:', error);toast('Failed to access microphone. Please check permissions.', 'error');+ return false;}}- startSilenceDetection() {- const dataArray = new Uint8Array(this.analyserNode.fftSize);- const checkSilence = () => {+ setupAudioAnalysis(stream) {+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);+ this.analyserNode = this.audioContext.createAnalyser();+ this.analyserNode.fftSize = 2048;+ this.analyserNode.minDecibels = -90;+ this.analyserNode.maxDecibels = -10;+ this.analyserNode.smoothingTimeConstant = 0.85;+ this.mediaStreamSource.connect(this.analyserNode);+ }++ startAudioAnalysis() {+ const analyzeFrame = () => {+ if (this.status === Status.INACTIVE) return;++ const dataArray = new Uint8Array(this.analyserNode.fftSize);this.analyserNode.getByteTimeDomainData(dataArray);+ // Calculate RMS volumelet sum = 0;for (let i = 0; i < dataArray.length; i++) {const amplitude = (dataArray[i] - 128) / 128;@@ -117,147 +209,94 @@ class MicrophoneInput {}const rms = Math.sqrt(sum / dataArray.length);- if (rms < this.silenceThreshold) {- if (!this.silenceTimer) {- this.silenceTimer = setTimeout(() => {- if (this.isRecording) {- console.log('Silence detected. Stopping recording.');- this.stopRecording();- microphoneButton.classList.remove('recording');- microphoneButton.classList.remove('mic-pulse');- }- }, this.silenceDuration);+ const now = Date.now();++ // Update status based on audio level+ if (rms > this.options.silenceThreshold) {+ this.lastAudioTime = now;+ this.silenceStartTime = null;++ if (this.status === Status.LISTENING || this.status === Status.WAITING) {+ this.status = Status.RECORDING;}- } else {- if (this.silenceTimer) {- clearTimeout(this.silenceTimer);- this.silenceTimer = null;+ } else if (this.status === Status.RECORDING) {+ if (!this.silenceStartTime) {+ this.silenceStartTime = now;}- }- if (this.isRecording) {- requestAnimationFrame(checkSilence);+ const silenceDuration = now - this.silenceStartTime;+ if (silenceDuration >= this.options.silenceDuration) {+ this.status = Status.WAITING;+ }}++ this.analysisFrame = requestAnimationFrame(analyzeFrame);};- if (this.isRecording) {- requestAnimationFrame(checkSilence);- }+ this.analysisFrame = requestAnimationFrame(analyzeFrame);}- startRecording() {- if (this.mediaRecorder && this.audioContext) {- this.isRecording = true;- this.audioChunks = [];- this.messageSent = false;- this.mediaRecorder.start(this.options.chunkDuration);- this.audioContext.resume();- this.startSilenceDetection();+ stopAudioAnalysis() {+ if (this.analysisFrame) {+ cancelAnimationFrame(this.analysisFrame);+ this.analysisFrame = null;}}- stopRecording() {- if (this.mediaRecorder && this.isRecording) {- this.isRecording = false;- if (!this.isFinalizing) {- this.isFinalizing = true;- this.mediaRecorder.stop();- this.audioContext.suspend();- if (this.silenceTimer) {- clearTimeout(this.silenceTimer);- this.silenceTimer = null;- }- }+ async process() {+ if (this.audioChunks.length === 0) {+ this.status = Status.LISTENING;+ return;}- }+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });+ const audioUrl = URL.createObjectURL(audioBlob);- async finalizeRecording() {- if (this.isFinalizing) {- this.isFinalizing = false;-- if (this.audioChunks.length > 0) {-- const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });- const audioUrl = URL.createObjectURL(audioBlob);- const samplingRate = 16000; // Adjust as needed for the model- const audioData = await read_audio(audioUrl, samplingRate);- URL.revokeObjectURL(audioUrl);-- // Transcribe the audio- const result = await this.transcriber(audioData);-- if (result.text) {- console.log('Final transcription received:', result.text);- await this.updateCallback(result.text, true);- } else {- console.warn('Final transcription returned empty text.');- }--- // Release the object URL after use-- // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });- // this.audioChunks = []; // Clear for next recording-- // const reader = new FileReader();- // reader.onloadend = async () => {- // const base64Data = reader.result.split(',')[1];-- // try {- // const response = await fetch('/transcribe', {- // method: 'POST',- // headers: {- // 'Content-Type': 'application/json'- // },- // body: JSON.stringify({- // audio_data: base64Data,- // model_size: this.options.modelSize,- // language: this.options.language,- // is_final: true- // })- // });-- // const result = await response.json();-- // if (result.text) {- // console.log('Final transcription received:', result.text);- // await this.updateCallback(result.text, true);- // } else {- // console.warn('Final transcription returned empty text.');- // }- // } catch (transcribeError) {- // console.error('Final transcription error:', transcribeError);- // toast('Final transcription failed.', 'error');- // } finally {- // // Reset the microphone button state- // microphoneButton.classList.remove('recording');- // microphoneButton.classList.remove('mic-pulse');- // microphoneButton.style.backgroundColor = '';- // }- // };- // reader.readAsDataURL(audioBlob);+ try {+ const samplingRate = 16000;+ const audioData = await read_audio(audioUrl, samplingRate);+ const result = await this.transcriber(audioData);+ const text = this.filterResult(result.text || "")++ if (text) {+ console.log('Transcription:', result.text);+ await this.updateCallback(result.text, true);}+ } catch (error) {+ console.error('Transcription error:', error);+ toast('Transcription failed.', 'error');+ } finally {+ URL.revokeObjectURL(audioUrl);+ this.audioChunks = [];+ this.status = Status.LISTENING;}}++ filterResult(text) {+ text = text.trim()+ let ok = false+ while (!ok) {+ if (!text) break+ if (text[0] === '{' && text[text.length - 1] === '}') break+ if (text[0] === '(' && text[text.length - 1] === ')') break+ if (text[0] === '[' && text[text.length - 1] === ']') break+ ok = true+ }+ if (ok) return text+ else console.log(`Discarding transcription: ${text}`)+ }}-export default MicrophoneInput;-async function initializeMicrophoneInput() {- console.log('Initializing microphone input');+// Initialize and handle click events+async function initializeMicrophoneInput() {microphoneInput = new MicrophoneInput(async (text, isFinal) => {if (isFinal) {- console.log('Final transcription callback received:', text);- updateChatInput(text)- // chatInput.value = text;- // adjustTextareaHeight();-+ updateChatInput(text);if (!microphoneInput.messageSent) {microphoneInput.messageSent = true;- console.log('Sending message');await sendMessage();}}@@ -265,42 +304,37 @@ async function initializeMicrophoneInput() {{modelSize: 'tiny',language: 'en',- silenceThreshold: 0.07, // Adjust as needed- silenceDuration: 2000, // Adjust as needed- onError: (error) => {- console.error('Microphone input error:', error);- toast('Microphone error: ' + error.message, 'error');- // Reset recording state- if (microphoneButton.classList.contains('recording')) {- microphoneButton.classList.remove('recording');- }- }+ silenceThreshold: 0.07,+ silenceDuration: 1000,+ waitingTimeout: 1500});+ microphoneInput.status = Status.ACTIVATING;- await microphoneInput.initialize();+ return await microphoneInput.initialize();}+microphoneButton.addEventListener('click', async () => {+ if (isProcessingClick) return;+ isProcessingClick = true;-function toggleRecording() {- console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);+ const hasPermission = await requestMicrophonePermission();+ if (!hasPermission) return;- if (microphoneInput.isRecording) {- microphoneInput.stopRecording();- microphoneButton.classList.remove('recording');- // Add pulsing animation class- microphoneButton.classList.remove('mic-pulse');- } else {- microphoneInput.startRecording();- microphoneButton.classList.add('recording');- // Add pulsing animation class- microphoneButton.classList.add('mic-pulse');- }+ try {+ if (!microphoneInput && !await initializeMicrophoneInput()) {+ return;+ }- // Add visual feedback- microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';- console.log('New recording state:', microphoneInput.isRecording);-}+ // Simply toggle between INACTIVE and LISTENING states+ microphoneInput.status =+ (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;+ } finally {+ setTimeout(() => {+ isProcessingClick = false;+ }, 300);+ }+});// Some error handling for microphone inputasync function requestMicrophonePermission() {@@ -313,25 +347,3 @@ async function requestMicrophonePermission() {return false;}}-// microphoneButton click event listener modifier-microphoneButton.addEventListener('click', async () => {- console.log('Microphone button clicked');- if (isProcessingClick) {- console.log('Click already being processed, ignoring');- return;- }- isProcessingClick = true;-- const hasPermission = await requestMicrophonePermission();- if (!hasPermission) return;-- if (!microphoneInput) {- await initializeMicrophoneInput();- }-- await toggleRecording();-- setTimeout(() => {- isProcessingClick = false;- }, 300); // Add a 300ms delay before allowing another click-});diff --git a/webui/test2.html b/webui/test2.htmlnew file mode 100644index 000000000..02e431458--- /dev/null+++ b/webui/test2.html@@ -0,0 +1,122 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="UTF-8">+ <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">+ <title>Agent Zero</title>++ <script type="module">+ import { pipeline, read_audio } from './[email protected]';++ let transcriber;+ let mediaRecorder;+ let isRecording = false;+ let audioChunks = [];+ let chunks_to_process = [{ tokens: [], finalized: false }];++ async function initTranscriber() {+ try {+ transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');+ } catch (error) {+ console.error("Failed to initialize transcriber:", error);+ }+ }++ async function toggleRecording() {+ if (isRecording) {+ stopRecording();+ } else {+ startRecording();+ }+ }++ async function startRecording() {+ isRecording = true;+ document.getElementById("micButton").innerText = "Stop Recording";++ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });+ mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });++ // Handle data as it’s available+ mediaRecorder.ondataavailable = (event) => {+ audioChunks.push(event.data);++ // Process chunks every 3 seconds+ if (audioChunks.length >= 3) {+ processAudioChunks();+ }+ };++ // Capture small chunks every second+ mediaRecorder.start(1000);+ }++ async function processAudioChunks() {+ const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });+ const audioUrl = URL.createObjectURL(audioBlob);++ try {+ const audioData = await read_audio(audioUrl, 16000);++ if (transcriber) {+ await transcriber(audioData, {+ callback_function: mergeChunksCallback,+ chunk_callback: processChunkCallback,+ return_timestamps: true,+ force_full_sequences: false,+ });+ } else {+ console.warn("Transcriber is not ready yet.");+ }+ } catch (error) {+ console.error("Error during transcription:", error);+ }++ // Cleanup and reset buffer+ URL.revokeObjectURL(audioUrl);+ audioChunks = [];+ }++ function processChunkCallback(chunk) {+ let lastChunk = chunks_to_process[chunks_to_process.length - 1];+ Object.assign(lastChunk, chunk);+ lastChunk.finalized = true;++ if (!chunk.is_last) {+ chunks_to_process.push({ tokens: [], finalized: false });+ }+ }++ function mergeChunksCallback(item) {+ let lastChunk = chunks_to_process[chunks_to_process.length - 1];+ lastChunk.tokens = [...item[0].output_token_ids];++ // Merge text chunks and update transcript+ const mergedText = transcriber.tokenizer._decode_asr(chunks_to_process, {+ time_precision: 1.0 / transcriber.processor.feature_extractor.config.chunk_length,+ return_timestamps: true,+ force_full_sequences: false,+ });++ document.getElementById("transcript").innerText += mergedText + ' ';+ }++ function stopRecording() {+ isRecording = false;+ document.getElementById("micButton").innerText = "Start Recording";+ if (mediaRecorder && mediaRecorder.state !== 'inactive') {+ mediaRecorder.stop();+ }+ }++ window.toggleRecording = toggleRecording;+ window.onload = initTranscriber;+ </script>+</head>++<body>+ <h1>Agent Zero Speech Transcription</h1>+ <button id="micButton" onclick="toggleRecording()">Start Recording</button>+ <p id="transcript">Transcript will appear here...</p>+</body>+</html>