Skip to content

Agent Zero AI framework

STT continued

frdel1a91334a420f
diff --git a/webui/index.css b/webui/index.css
index 7f9b8c7b8..fde8a6c4e 100644
--- a/webui/index.css
+++ b/webui/index.css
@@ -907,35 +907,6 @@ pre {
background-color: var(--color-primary-light);
}
-/* MIC BUTTON */
-#microphone-button {
- background-color: #3270e2;
-}
-
-#microphone-button:hover {
- background-color: #4382e8;
-}
-#microphone-button.recording {
- background-color: #ff4136; /* Red color for recording */
- transition: background-color 0.3s ease;
-}
-
-.mic-pulse {
- animation: pulse 1.5s infinite;
-}
-
-@keyframes pulse {
- 0% {
- transform: scale(1);
- }
- 50% {
- transform: scale(1.1);
- }
- 100% {
- transform: scale(1);
- }
-}
-
.chat-button svg {
width: 1.5rem;
height: 1.5rem;
diff --git a/webui/index.html b/webui/index.html
index 65b4d127a..309670d60 100644
--- a/webui/index.html
+++ b/webui/index.html
@@ -8,6 +8,7 @@
<link rel="stylesheet" href="index.css">
<link rel="stylesheet" href="toast.css">
<link rel="stylesheet" href="settings.css">
+ <link rel="stylesheet" href="speech.css">
<script>
window.safeCall = function (name, ...args) {
@@ -236,7 +237,7 @@
</button>
<!-- Microphone button -->
- <button class="chat-button" id="microphone-button" aria-label="Start/Stop recording">
+ <button class="chat-button mic-inactive" id="microphone-button" aria-label="Start/Stop recording">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24" height="24">
<path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z"/>
</svg>
diff --git a/webui/speech copy.js b/webui/speech copy.js
new file mode 100644
index 000000000..eb2050075
--- /dev/null
+++ b/webui/speech copy.js
@@ -0,0 +1,337 @@
+import { pipeline, read_audio } from './[email protected]';
+import { updateChatInput, sendMessage } from './index.js';
+
+const microphoneButton = document.getElementById('microphone-button');
+let microphoneInput = null;
+let isProcessingClick = false;
+
+class MicrophoneInput {
+ /**
+ * Voice Input Handler with Whisper Transcription
+ *
+ * Whisper Model Size Configuration:
+ * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
+ * - Best for: Quick prototyping, low-resource environments
+ * - Pros: Very fast, low memory usage
+ * - Cons: Lowest transcription accuracy
+ *
+ * - 'base': Small model, good balance of speed and accuracy (~74MB)
+ * - Best for: General-purpose voice input
+ * - Pros: Reasonable accuracy, moderate resource usage
+ * - Cons: Less accurate than larger models
+ *
+ * - 'small': Medium-sized model, better accuracy (~244MB)
+ * - Best for: More precise transcription needs
+ * - Pros: Improved accuracy over base model
+ * - Cons: Slower, more memory-intensive
+ *
+ * - 'medium': Large model with high accuracy (~769MB)
+ * - Best for: Professional transcription, multi-language support
+ * - Pros: Very high accuracy
+ * - Cons: Significant computational resources required
+ *
+ * - 'large': Largest model, highest accuracy (~1.5GB)
+ * - Best for: Professional, multi-language transcription
+ * - Pros: Highest possible accuracy
+ * - Cons: Slowest, most resource-intensive
+ *
+ * Recommended Default: 'base' for most web applications
+ */
+ constructor(updateCallback, options = {}) {
+ this.mediaRecorder = null;
+ this.audioChunks = [];
+ this.isRecording = false;
+ this.updateCallback = updateCallback;
+ this.isFinalizing = false;
+ this.messageSent = false; // move messageSent into class
+
+ // New properties for silence detection
+ this.audioContext = null;
+ this.mediaStreamSource = null;
+ this.analyserNode = null;
+ this.silenceTimer = null;
+ this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
+ this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
+
+ this.options = {
+ modelSize: 'tiny',
+ language: 'en',
+ chunkDuration: 3000,
+ ...options
+ };
+ }
+
+ async initialize() {
+ try {
+
+ this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);
+
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ channelCount: 1
+ }
+ });
+
+ // Configure MediaRecorder
+ this.mediaRecorder = new MediaRecorder(stream);
+
+ // Handle audio data availability
+ this.mediaRecorder.ondataavailable = async (event) => {
+ if (event.data.size > 0) {
+ this.audioChunks.push(event.data);
+ // await this.processAudioChunk(event.data);
+ }
+ };
+
+ // Handle recording stop
+ this.mediaRecorder.onstop = async () => {
+ await this.finalizeRecording();
+ };
+
+ // Set up AudioContext and AnalyserNode for silence detection
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
+ this.analyserNode = this.audioContext.createAnalyser();
+ this.analyserNode.minDecibels = -90;
+ this.analyserNode.maxDecibels = -10;
+ this.analyserNode.smoothingTimeConstant = 0.85;
+
+ this.mediaStreamSource.connect(this.analyserNode);
+ } catch (error) {
+ console.error('Microphone initialization error:', error);
+ toast('Failed to access microphone. Please check permissions.', 'error');
+ }
+ }
+
+ startSilenceDetection() {
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
+ const checkSilence = () => {
+ this.analyserNode.getByteTimeDomainData(dataArray);
+
+ let sum = 0;
+ for (let i = 0; i < dataArray.length; i++) {
+ const amplitude = (dataArray[i] - 128) / 128;
+ sum += amplitude * amplitude;
+ }
+ const rms = Math.sqrt(sum / dataArray.length);
+
+ if (rms < this.silenceThreshold) {
+ if (!this.silenceTimer) {
+ this.silenceTimer = setTimeout(() => {
+ if (this.isRecording) {
+ console.log('Silence detected. Stopping recording.');
+ this.stopRecording();
+ microphoneButton.classList.remove('recording');
+ microphoneButton.classList.remove('mic-pulse');
+ }
+ }, this.silenceDuration);
+ }
+ } else {
+ if (this.silenceTimer) {
+ clearTimeout(this.silenceTimer);
+ this.silenceTimer = null;
+ }
+ }
+
+ if (this.isRecording) {
+ requestAnimationFrame(checkSilence);
+ }
+ };
+
+ if (this.isRecording) {
+ requestAnimationFrame(checkSilence);
+ }
+ }
+
+ startRecording() {
+ if (this.mediaRecorder && this.audioContext) {
+ this.isRecording = true;
+ this.audioChunks = [];
+ this.messageSent = false;
+ this.mediaRecorder.start(this.options.chunkDuration);
+ this.audioContext.resume();
+ this.startSilenceDetection();
+ }
+ }
+
+ stopRecording() {
+ if (this.mediaRecorder && this.isRecording) {
+ this.isRecording = false;
+ if (!this.isFinalizing) {
+ this.isFinalizing = true;
+ this.mediaRecorder.stop();
+ this.audioContext.suspend();
+ if (this.silenceTimer) {
+ clearTimeout(this.silenceTimer);
+ this.silenceTimer = null;
+ }
+ }
+ }
+ }
+
+
+ async finalizeRecording() {
+ if (this.isFinalizing) {
+ this.isFinalizing = false;
+
+ if (this.audioChunks.length > 0) {
+
+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
+ const audioUrl = URL.createObjectURL(audioBlob);
+ const samplingRate = 16000; // Adjust as needed for the model
+ const audioData = await read_audio(audioUrl, samplingRate);
+ URL.revokeObjectURL(audioUrl);
+
+ // Transcribe the audio
+ const result = await this.transcriber(audioData);
+
+ if (result.text) {
+ console.log('Final transcription received:', result.text);
+ await this.updateCallback(result.text, true);
+ } else {
+ console.warn('Final transcription returned empty text.');
+ }
+
+
+ // Release the object URL after use
+
+ // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
+ // this.audioChunks = []; // Clear for next recording
+
+ // const reader = new FileReader();
+ // reader.onloadend = async () => {
+ // const base64Data = reader.result.split(',')[1];
+
+ // try {
+ // const response = await fetch('/transcribe', {
+ // method: 'POST',
+ // headers: {
+ // 'Content-Type': 'application/json'
+ // },
+ // body: JSON.stringify({
+ // audio_data: base64Data,
+ // model_size: this.options.modelSize,
+ // language: this.options.language,
+ // is_final: true
+ // })
+ // });
+
+ // const result = await response.json();
+
+ // if (result.text) {
+ // console.log('Final transcription received:', result.text);
+ // await this.updateCallback(result.text, true);
+ // } else {
+ // console.warn('Final transcription returned empty text.');
+ // }
+ // } catch (transcribeError) {
+ // console.error('Final transcription error:', transcribeError);
+ // toast('Final transcription failed.', 'error');
+ // } finally {
+ // // Reset the microphone button state
+ // microphoneButton.classList.remove('recording');
+ // microphoneButton.classList.remove('mic-pulse');
+ // microphoneButton.style.backgroundColor = '';
+ // }
+ // };
+ // reader.readAsDataURL(audioBlob);
+ }
+ }
+ }
+}
+
+export default MicrophoneInput;
+
+async function initializeMicrophoneInput() {
+ console.log('Initializing microphone input');
+
+ microphoneInput = new MicrophoneInput(
+ async (text, isFinal) => {
+ if (isFinal) {
+ console.log('Final transcription callback received:', text);
+ updateChatInput(text)
+ // chatInput.value = text;
+ // adjustTextareaHeight();
+
+ if (!microphoneInput.messageSent) {
+ microphoneInput.messageSent = true;
+ console.log('Sending message');
+ await sendMessage();
+ }
+ }
+ },
+ {
+ modelSize: 'tiny',
+ language: 'en',
+ silenceThreshold: 0.07, // Adjust as needed
+ silenceDuration: 2000, // Adjust as needed
+ onError: (error) => {
+ console.error('Microphone input error:', error);
+ toast('Microphone error: ' + error.message, 'error');
+ // Reset recording state
+ if (microphoneButton.classList.contains('recording')) {
+ microphoneButton.classList.remove('recording');
+ }
+ }
+ }
+ );
+
+ await microphoneInput.initialize();
+}
+
+
+function toggleRecording() {
+ console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
+
+ if (microphoneInput.isRecording) {
+ microphoneInput.stopRecording();
+ microphoneButton.classList.remove('recording');
+ // Add pulsing animation class
+ microphoneButton.classList.remove('mic-pulse');
+ } else {
+ microphoneInput.startRecording();
+ microphoneButton.classList.add('recording');
+ // Add pulsing animation class
+ microphoneButton.classList.add('mic-pulse');
+ }
+
+ // Add visual feedback
+ microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
+ console.log('New recording state:', microphoneInput.isRecording);
+}
+
+// Some error handling for microphone input
+async function requestMicrophonePermission() {
+ try {
+ await navigator.mediaDevices.getUserMedia({ audio: true });
+ return true;
+ } catch (err) {
+ console.error('Error accessing microphone:', err);
+ toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');
+ return false;
+ }
+}
+// microphoneButton click event listener modifier
+microphoneButton.addEventListener('click', async () => {
+ console.log('Microphone button clicked');
+ if (isProcessingClick) {
+ console.log('Click already being processed, ignoring');
+ return;
+ }
+ isProcessingClick = true;
+
+ const hasPermission = await requestMicrophonePermission();
+ if (!hasPermission) return;
+
+ if (!microphoneInput) {
+ await initializeMicrophoneInput();
+ }
+
+ await toggleRecording();
+
+ setTimeout(() => {
+ isProcessingClick = false;
+ }, 300); // Add a 300ms delay before allowing another click
+});
diff --git a/webui/speech.css b/webui/speech.css
new file mode 100644
index 000000000..07c66d2a0
--- /dev/null
+++ b/webui/speech.css
@@ -0,0 +1,55 @@
+/* MIC BUTTON */
+#microphone-button {
+
+}
+
+#microphone-button:hover {
+}
+
+#microphone-button.recording {
+ background-color: #ff4136; /* Red color for recording */
+ transition: background-color 0.3s ease;
+}
+
+@keyframes pulse {
+ 0% {
+ transform: scale(1);
+ }
+ 50% {
+ transform: scale(1.1);
+ }
+ 100% {
+ transform: scale(1);
+ }
+ }
+
+.mic-pulse {
+ animation: pulse 1.5s infinite;
+}
+
+
+.mic-inactive{
+ background-color: grey;
+}
+
+.mic-activating{
+ background-color: silver;
+ animation: pulse 0.8s infinite;
+}
+
+.mic-listening {
+ background-color: red;
+}
+
+.mic-recording {
+ background-color: green;
+}
+
+.mic-waiting {
+ background-color: teal;
+}
+
+.mic-processing {
+ background-color: darkcyan;
+ animation: pulse 0.8s infinite;
+}
\ No newline at end of file
diff --git a/webui/speech.js b/webui/speech.js
index eb2050075..f7dd0055d 100644
--- a/webui/speech.js
+++ b/webui/speech.js
@@ -5,66 +5,149 @@ const microphoneButton = document.getElementById('microphone-button');
let microphoneInput = null;
let isProcessingClick = false;
+const Status = {
+ INACTIVE: 'inactive',
+ ACTIVATING: 'activating',
+ LISTENING: 'listening',
+ RECORDING: 'recording',
+ WAITING: 'waiting',
+ PROCESSING: 'processing'
+};
+
class MicrophoneInput {
- /**
- * Voice Input Handler with Whisper Transcription
- *
- * Whisper Model Size Configuration:
- * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
- * - Best for: Quick prototyping, low-resource environments
- * - Pros: Very fast, low memory usage
- * - Cons: Lowest transcription accuracy
- *
- * - 'base': Small model, good balance of speed and accuracy (~74MB)
- * - Best for: General-purpose voice input
- * - Pros: Reasonable accuracy, moderate resource usage
- * - Cons: Less accurate than larger models
- *
- * - 'small': Medium-sized model, better accuracy (~244MB)
- * - Best for: More precise transcription needs
- * - Pros: Improved accuracy over base model
- * - Cons: Slower, more memory-intensive
- *
- * - 'medium': Large model with high accuracy (~769MB)
- * - Best for: Professional transcription, multi-language support
- * - Pros: Very high accuracy
- * - Cons: Significant computational resources required
- *
- * - 'large': Largest model, highest accuracy (~1.5GB)
- * - Best for: Professional, multi-language transcription
- * - Pros: Highest possible accuracy
- * - Cons: Slowest, most resource-intensive
- *
- * Recommended Default: 'base' for most web applications
- */
constructor(updateCallback, options = {}) {
this.mediaRecorder = null;
this.audioChunks = [];
- this.isRecording = false;
+ this.lastChunk = [];
this.updateCallback = updateCallback;
- this.isFinalizing = false;
- this.messageSent = false; // move messageSent into class
+ this.messageSent = false;
- // New properties for silence detection
+ // Audio analysis properties
this.audioContext = null;
this.mediaStreamSource = null;
this.analyserNode = null;
- this.silenceTimer = null;
- this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
- this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
+ this._status = Status.INACTIVE;
+
+ // Timing properties
+ this.lastAudioTime = null;
+ this.waitingTimer = null;
+ this.silenceStartTime = null;
+ this.hasStartedRecording = false;
+ this.analysisFrame = null;
this.options = {
modelSize: 'tiny',
language: 'en',
- chunkDuration: 3000,
+ silenceThreshold: 0.15,
+ silenceDuration: 1000,
+ waitingTimeout: 2000,
+ minSpeechDuration: 500,
...options
};
}
+ get status() {
+ return this._status;
+ }
+
+ set status(newStatus) {
+ if (this._status === newStatus) return;
+
+ const oldStatus = this._status;
+ this._status = newStatus;
+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
+
+ // Update UI
+ microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);
+ microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);
+ microphoneButton.setAttribute('data-status', newStatus);
+
+ // Handle state-specific behaviors
+ this.handleStatusChange(oldStatus, newStatus);
+ }
+
+ handleStatusChange(oldStatus, newStatus) {
+
+ //last chunk kept only for transition to recording status
+ if (newStatus != Status.RECORDING) { this.lastChunk = null; }
+
+ switch (newStatus) {
+ case Status.INACTIVE:
+ this.handleInactiveState();
+ break;
+ case Status.LISTENING:
+ this.handleListeningState();
+ break;
+ case Status.RECORDING:
+ this.handleRecordingState();
+ break;
+ case Status.WAITING:
+ this.handleWaitingState();
+ break;
+ case Status.PROCESSING:
+ this.handleProcessingState();
+ break;
+ }
+ }
+
+ handleInactiveState() {
+ this.stopRecording();
+ this.stopAudioAnalysis();
+ if (this.waitingTimer) {
+ clearTimeout(this.waitingTimer);
+ this.waitingTimer = null;
+ }
+ }
+
+ handleListeningState() {
+ this.stopRecording();
+ this.audioChunks = [];
+ this.hasStartedRecording = false;
+ this.silenceStartTime = null;
+ this.lastAudioTime = null;
+ this.messageSent = false;
+ this.startAudioAnalysis();
+ }
+
+ handleRecordingState() {
+ if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {
+ this.hasStartedRecording = true;
+ this.mediaRecorder.start(1000);
+ console.log('Speech started');
+ }
+ if (this.waitingTimer) {
+ clearTimeout(this.waitingTimer);
+ this.waitingTimer = null;
+ }
+ }
+
+ handleWaitingState() {
+ // Don't stop recording during waiting state
+ this.waitingTimer = setTimeout(() => {
+ if (this.status === Status.WAITING) {
+ this.status = Status.PROCESSING;
+ }
+ }, this.options.waitingTimeout);
+ }
+
+ handleProcessingState() {
+ this.stopRecording();
+ this.process();
+ }
+
+ stopRecording() {
+ if (this.mediaRecorder?.state === 'recording') {
+ this.mediaRecorder.stop();
+ this.hasStartedRecording = false;
+ }
+ }
+
async initialize() {
try {
-
- this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);
+ this.transcriber = await pipeline(
+ 'automatic-speech-recognition',
+ `Xenova/whisper-${this.options.modelSize}.${this.options.language}`
+ );
const stream = await navigator.mediaDevices.getUserMedia({
audio: {
@@ -74,42 +157,51 @@ class MicrophoneInput {
}
});
- // Configure MediaRecorder
this.mediaRecorder = new MediaRecorder(stream);
-
- // Handle audio data availability
- this.mediaRecorder.ondataavailable = async (event) => {
- if (event.data.size > 0) {
+ this.mediaRecorder.ondataavailable = (event) => {
+ if (event.data.size > 0 &&
+ (this.status === Status.RECORDING || this.status === Status.WAITING)) {
+ if (this.lastChunk) {
+ this.audioChunks.push(this.lastChunk);
+ this.lastChunk = null;
+ }
this.audioChunks.push(event.data);
- // await this.processAudioChunk(event.data);
+ console.log('Audio chunk received, total chunks:', this.audioChunks.length);
+ }
+ else if (this.status === Status.LISTENING) {
+ this.lastChunk = event.data;
}
};
- // Handle recording stop
- this.mediaRecorder.onstop = async () => {
- await this.finalizeRecording();
- };
-
- // Set up AudioContext and AnalyserNode for silence detection
- this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
- this.analyserNode = this.audioContext.createAnalyser();
- this.analyserNode.minDecibels = -90;
- this.analyserNode.maxDecibels = -10;
- this.analyserNode.smoothingTimeConstant = 0.85;
-
- this.mediaStreamSource.connect(this.analyserNode);
+ this.setupAudioAnalysis(stream);
+ return true;
} catch (error) {
+
console.error('Microphone initialization error:', error);
toast('Failed to access microphone. Please check permissions.', 'error');
+ return false;
}
}
- startSilenceDetection() {
- const dataArray = new Uint8Array(this.analyserNode.fftSize);
- const checkSilence = () => {
+ setupAudioAnalysis(stream) {
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
+ this.analyserNode = this.audioContext.createAnalyser();
+ this.analyserNode.fftSize = 2048;
+ this.analyserNode.minDecibels = -90;
+ this.analyserNode.maxDecibels = -10;
+ this.analyserNode.smoothingTimeConstant = 0.85;
+ this.mediaStreamSource.connect(this.analyserNode);
+ }
+
+ startAudioAnalysis() {
+ const analyzeFrame = () => {
+ if (this.status === Status.INACTIVE) return;
+
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
this.analyserNode.getByteTimeDomainData(dataArray);
+ // Calculate RMS volume
let sum = 0;
for (let i = 0; i < dataArray.length; i++) {
const amplitude = (dataArray[i] - 128) / 128;
@@ -117,147 +209,94 @@ class MicrophoneInput {
}
const rms = Math.sqrt(sum / dataArray.length);
- if (rms < this.silenceThreshold) {
- if (!this.silenceTimer) {
- this.silenceTimer = setTimeout(() => {
- if (this.isRecording) {
- console.log('Silence detected. Stopping recording.');
- this.stopRecording();
- microphoneButton.classList.remove('recording');
- microphoneButton.classList.remove('mic-pulse');
- }
- }, this.silenceDuration);
+ const now = Date.now();
+
+ // Update status based on audio level
+ if (rms > this.options.silenceThreshold) {
+ this.lastAudioTime = now;
+ this.silenceStartTime = null;
+
+ if (this.status === Status.LISTENING || this.status === Status.WAITING) {
+ this.status = Status.RECORDING;
}
- } else {
- if (this.silenceTimer) {
- clearTimeout(this.silenceTimer);
- this.silenceTimer = null;
+ } else if (this.status === Status.RECORDING) {
+ if (!this.silenceStartTime) {
+ this.silenceStartTime = now;
}
- }
- if (this.isRecording) {
- requestAnimationFrame(checkSilence);
+ const silenceDuration = now - this.silenceStartTime;
+ if (silenceDuration >= this.options.silenceDuration) {
+ this.status = Status.WAITING;
+ }
}
+
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
};
- if (this.isRecording) {
- requestAnimationFrame(checkSilence);
- }
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
}
- startRecording() {
- if (this.mediaRecorder && this.audioContext) {
- this.isRecording = true;
- this.audioChunks = [];
- this.messageSent = false;
- this.mediaRecorder.start(this.options.chunkDuration);
- this.audioContext.resume();
- this.startSilenceDetection();
+ stopAudioAnalysis() {
+ if (this.analysisFrame) {
+ cancelAnimationFrame(this.analysisFrame);
+ this.analysisFrame = null;
}
}
- stopRecording() {
- if (this.mediaRecorder && this.isRecording) {
- this.isRecording = false;
- if (!this.isFinalizing) {
- this.isFinalizing = true;
- this.mediaRecorder.stop();
- this.audioContext.suspend();
- if (this.silenceTimer) {
- clearTimeout(this.silenceTimer);
- this.silenceTimer = null;
- }
- }
+ async process() {
+ if (this.audioChunks.length === 0) {
+ this.status = Status.LISTENING;
+ return;
}
- }
+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
+ const audioUrl = URL.createObjectURL(audioBlob);
- async finalizeRecording() {
- if (this.isFinalizing) {
- this.isFinalizing = false;
-
- if (this.audioChunks.length > 0) {
-
- const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
- const audioUrl = URL.createObjectURL(audioBlob);
- const samplingRate = 16000; // Adjust as needed for the model
- const audioData = await read_audio(audioUrl, samplingRate);
- URL.revokeObjectURL(audioUrl);
-
- // Transcribe the audio
- const result = await this.transcriber(audioData);
-
- if (result.text) {
- console.log('Final transcription received:', result.text);
- await this.updateCallback(result.text, true);
- } else {
- console.warn('Final transcription returned empty text.');
- }
-
-
- // Release the object URL after use
-
- // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
- // this.audioChunks = []; // Clear for next recording
-
- // const reader = new FileReader();
- // reader.onloadend = async () => {
- // const base64Data = reader.result.split(',')[1];
-
- // try {
- // const response = await fetch('/transcribe', {
- // method: 'POST',
- // headers: {
- // 'Content-Type': 'application/json'
- // },
- // body: JSON.stringify({
- // audio_data: base64Data,
- // model_size: this.options.modelSize,
- // language: this.options.language,
- // is_final: true
- // })
- // });
-
- // const result = await response.json();
-
- // if (result.text) {
- // console.log('Final transcription received:', result.text);
- // await this.updateCallback(result.text, true);
- // } else {
- // console.warn('Final transcription returned empty text.');
- // }
- // } catch (transcribeError) {
- // console.error('Final transcription error:', transcribeError);
- // toast('Final transcription failed.', 'error');
- // } finally {
- // // Reset the microphone button state
- // microphoneButton.classList.remove('recording');
- // microphoneButton.classList.remove('mic-pulse');
- // microphoneButton.style.backgroundColor = '';
- // }
- // };
- // reader.readAsDataURL(audioBlob);
+ try {
+ const samplingRate = 16000;
+ const audioData = await read_audio(audioUrl, samplingRate);
+ const result = await this.transcriber(audioData);
+ const text = this.filterResult(result.text || "")
+
+ if (text) {
+ console.log('Transcription:', result.text);
+ await this.updateCallback(result.text, true);
}
+ } catch (error) {
+ console.error('Transcription error:', error);
+ toast('Transcription failed.', 'error');
+ } finally {
+ URL.revokeObjectURL(audioUrl);
+ this.audioChunks = [];
+ this.status = Status.LISTENING;
}
}
+
+ filterResult(text) {
+ text = text.trim()
+ let ok = false
+ while (!ok) {
+ if (!text) break
+ if (text[0] === '{' && text[text.length - 1] === '}') break
+ if (text[0] === '(' && text[text.length - 1] === ')') break
+ if (text[0] === '[' && text[text.length - 1] === ']') break
+ ok = true
+ }
+ if (ok) return text
+ else console.log(`Discarding transcription: ${text}`)
+ }
}
-export default MicrophoneInput;
-async function initializeMicrophoneInput() {
- console.log('Initializing microphone input');
+// Initialize and handle click events
+async function initializeMicrophoneInput() {
microphoneInput = new MicrophoneInput(
async (text, isFinal) => {
if (isFinal) {
- console.log('Final transcription callback received:', text);
- updateChatInput(text)
- // chatInput.value = text;
- // adjustTextareaHeight();
-
+ updateChatInput(text);
if (!microphoneInput.messageSent) {
microphoneInput.messageSent = true;
- console.log('Sending message');
await sendMessage();
}
}
@@ -265,42 +304,37 @@ async function initializeMicrophoneInput() {
{
modelSize: 'tiny',
language: 'en',
- silenceThreshold: 0.07, // Adjust as needed
- silenceDuration: 2000, // Adjust as needed
- onError: (error) => {
- console.error('Microphone input error:', error);
- toast('Microphone error: ' + error.message, 'error');
- // Reset recording state
- if (microphoneButton.classList.contains('recording')) {
- microphoneButton.classList.remove('recording');
- }
- }
+ silenceThreshold: 0.07,
+ silenceDuration: 1000,
+ waitingTimeout: 1500
}
);
+ microphoneInput.status = Status.ACTIVATING;
- await microphoneInput.initialize();
+ return await microphoneInput.initialize();
}
+microphoneButton.addEventListener('click', async () => {
+ if (isProcessingClick) return;
+ isProcessingClick = true;
-function toggleRecording() {
- console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
+ const hasPermission = await requestMicrophonePermission();
+ if (!hasPermission) return;
- if (microphoneInput.isRecording) {
- microphoneInput.stopRecording();
- microphoneButton.classList.remove('recording');
- // Add pulsing animation class
- microphoneButton.classList.remove('mic-pulse');
- } else {
- microphoneInput.startRecording();
- microphoneButton.classList.add('recording');
- // Add pulsing animation class
- microphoneButton.classList.add('mic-pulse');
- }
+ try {
+ if (!microphoneInput && !await initializeMicrophoneInput()) {
+ return;
+ }
- // Add visual feedback
- microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
- console.log('New recording state:', microphoneInput.isRecording);
-}
+ // Simply toggle between INACTIVE and LISTENING states
+ microphoneInput.status =
+ (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;
+ } finally {
+ setTimeout(() => {
+ isProcessingClick = false;
+ }, 300);
+ }
+});
// Some error handling for microphone input
async function requestMicrophonePermission() {
@@ -313,25 +347,3 @@ async function requestMicrophonePermission() {
return false;
}
}
-// microphoneButton click event listener modifier
-microphoneButton.addEventListener('click', async () => {
- console.log('Microphone button clicked');
- if (isProcessingClick) {
- console.log('Click already being processed, ignoring');
- return;
- }
- isProcessingClick = true;
-
- const hasPermission = await requestMicrophonePermission();
- if (!hasPermission) return;
-
- if (!microphoneInput) {
- await initializeMicrophoneInput();
- }
-
- await toggleRecording();
-
- setTimeout(() => {
- isProcessingClick = false;
- }, 300); // Add a 300ms delay before allowing another click
-});
diff --git a/webui/test2.html b/webui/test2.html
new file mode 100644
index 000000000..02e431458
--- /dev/null
+++ b/webui/test2.html
@@ -0,0 +1,122 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
+ <title>Agent Zero</title>
+
+ <script type="module">
+ import { pipeline, read_audio } from './[email protected]';
+
+ let transcriber;
+ let mediaRecorder;
+ let isRecording = false;
+ let audioChunks = [];
+ let chunks_to_process = [{ tokens: [], finalized: false }];
+
+ async function initTranscriber() {
+ try {
+ transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
+ } catch (error) {
+ console.error("Failed to initialize transcriber:", error);
+ }
+ }
+
+ async function toggleRecording() {
+ if (isRecording) {
+ stopRecording();
+ } else {
+ startRecording();
+ }
+ }
+
+ async function startRecording() {
+ isRecording = true;
+ document.getElementById("micButton").innerText = "Stop Recording";
+
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
+
+ // Handle data as it’s available
+ mediaRecorder.ondataavailable = (event) => {
+ audioChunks.push(event.data);
+
+ // Process chunks every 3 seconds
+ if (audioChunks.length >= 3) {
+ processAudioChunks();
+ }
+ };
+
+ // Capture small chunks every second
+ mediaRecorder.start(1000);
+ }
+
+ async function processAudioChunks() {
+ const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
+ const audioUrl = URL.createObjectURL(audioBlob);
+
+ try {
+ const audioData = await read_audio(audioUrl, 16000);
+
+ if (transcriber) {
+ await transcriber(audioData, {
+ callback_function: mergeChunksCallback,
+ chunk_callback: processChunkCallback,
+ return_timestamps: true,
+ force_full_sequences: false,
+ });
+ } else {
+ console.warn("Transcriber is not ready yet.");
+ }
+ } catch (error) {
+ console.error("Error during transcription:", error);
+ }
+
+ // Cleanup and reset buffer
+ URL.revokeObjectURL(audioUrl);
+ audioChunks = [];
+ }
+
+ function processChunkCallback(chunk) {
+ let lastChunk = chunks_to_process[chunks_to_process.length - 1];
+ Object.assign(lastChunk, chunk);
+ lastChunk.finalized = true;
+
+ if (!chunk.is_last) {
+ chunks_to_process.push({ tokens: [], finalized: false });
+ }
+ }
+
+ function mergeChunksCallback(item) {
+ let lastChunk = chunks_to_process[chunks_to_process.length - 1];
+ lastChunk.tokens = [...item[0].output_token_ids];
+
+ // Merge text chunks and update transcript
+ const mergedText = transcriber.tokenizer._decode_asr(chunks_to_process, {
+ time_precision: 1.0 / transcriber.processor.feature_extractor.config.chunk_length,
+ return_timestamps: true,
+ force_full_sequences: false,
+ });
+
+ document.getElementById("transcript").innerText += mergedText + ' ';
+ }
+
+ function stopRecording() {
+ isRecording = false;
+ document.getElementById("micButton").innerText = "Start Recording";
+ if (mediaRecorder && mediaRecorder.state !== 'inactive') {
+ mediaRecorder.stop();
+ }
+ }
+
+ window.toggleRecording = toggleRecording;
+ window.onload = initTranscriber;
+ </script>
+</head>
+
+<body>
+ <h1>Agent Zero Speech Transcription</h1>
+ <button id="micButton" onclick="toggleRecording()">Start Recording</button>
+ <p id="transcript">Transcript will appear here...</p>
+</body>
+</html>