Agent Zero AI framework
feature: openai-whisper voice input
Alessandro82ca0d800ab6
diff --git a/python/helpers/voice_transcription.py b/python/helpers/voice_transcription.pynew file mode 100644index 000000000..97e6196b9--- /dev/null+++ b/python/helpers/voice_transcription.py@@ -0,0 +1,105 @@+import whisper+import io+import base64+import numpy as np+from typing import Optional, Union, BinaryIO+from whisper.audio import load_audio+import tempfile+import os+import subprocess+import warnings++# suppress FutureWarning from torch.load+warnings.filterwarnings('ignore', category=FutureWarning)++class VoiceTranscription:+ @staticmethod+ def load_model(model_size: str = "base"):+ """+ Load a Whisper model with the specified size.+ """+ try:+ return whisper.load_model(model_size)+ except Exception as e:+ print(f"Error loading Whisper model: {e}")+ return None++ @classmethod+ def transcribe_bytes(cls, audio_bytes: Union[str, bytes, BinaryIO],+ model_size: str = "base",+ language: Optional[str] = None) -> str:+ """+ Transcribe audio from bytes or a file-like object.+ """+ model = cls.load_model(model_size)+ if not model:+ raise RuntimeError("Could not load Whisper model")++ # Decode audio bytes if encoded as a base64 string+ if isinstance(audio_bytes, str):+ try:+ audio_bytes = base64.b64decode(audio_bytes)+ except Exception as e:+ print(f"Error decoding base64 audio data: {e}")+ raise++ # Save audio bytes to a temporary file with .webm extension+ with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp_input_file:+ tmp_input_file.write(audio_bytes)+ temp_input_path = tmp_input_file.name++ try:+ # Define the output path with .wav extension+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_output_file:+ temp_output_path = tmp_output_file.name++ # Convert WebM to WAV using FFmpeg+ ffmpeg_cmd = [+ 'ffmpeg', '-y', '-i', temp_input_path,+ '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',+ temp_output_path+ ]++ print(f"Running FFmpeg command: {' '.join(ffmpeg_cmd)}")++ # Run FFmpeg command using subprocess+ try:+ subprocess.run(+ ffmpeg_cmd, stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL, check=True # Suppressed stderr+ )+ except subprocess.CalledProcessError as e:+ error_message = e.stderr.decode().strip()+ print(f"FFmpeg error: {error_message}")++ # Log the temporary file path for debugging+ print(f"Transcribing audio from temporary file: {temp_output_path}")++ # Load audio using Whisper's load_audio+ audio = load_audio(temp_output_path)++ # Transcribe using the Whisper model+ result = model.transcribe(audio, fp16=False, language=language)+ text = result.get("text", "").strip()++ # Log the transcription result+ print(f"Transcription result: {text}")++ return text++ except subprocess.CalledProcessError as e:+ error_message = e.stderr.decode().strip()+ print(f"FFmpeg error: {error_message}")+ # Return empty string or handle as appropriate+ return ""+ except Exception as transcribe_error:+ print(f"Transcription error: {transcribe_error}")+ # Return empty string or handle as appropriate+ return ""+ finally:++ # Clean up temporary files+ if os.path.exists(temp_input_path):+ os.remove(temp_input_path)+ if os.path.exists(temp_output_path):+ os.remove(temp_output_path)diff --git a/requirements.txt b/requirements.txtindex 8e3fac3df..192cdd06c 100644--- a/requirements.txt+++ b/requirements.txt@@ -14,6 +14,7 @@ langchain-huggingface==0.0.3langchain-mistralai==0.1.8langchain-ollama==0.1.3langchain-openai==0.1.15+openai-whisper==20240930lxml_html_clean==0.3.1markdown==3.7newspaper3k==0.2.8diff --git a/run_ui.py b/run_ui.pyindex a286ff94e..22e77e1a5 100644--- a/run_ui.py+++ b/run_ui.py@@ -13,6 +13,8 @@ from python.helpers.files import get_abs_pathfrom python.helpers.print_style import PrintStylefrom python.helpers.dotenv import load_dotenvfrom python.helpers import persist_chat, settings+from python.helpers.voice_transcription import VoiceTranscription+import base64from werkzeug.utils import secure_filename@@ -133,6 +135,78 @@ async def health_check():return "OK"[email protected]('/transcribe', methods=['POST'])+def transcribe_audio():+ """+ Transcribe audio data using Whisper.+ Expected JSON payload:+ {+ 'audio_data': base64 encoded audio,+ 'model_size': 'base', # Optional, defaults to 'base'+ 'language': None, # Optional language code+ 'is_final': False # Optional flag for final transcription+ }+ """+ try:+ # Parse request data+ data = request.json+ audio_data = data.get('audio_data')+ model_size = data.get('model_size', 'base')+ language = data.get('language')+ is_final = data.get('is_final', False)++ # Validate input+ if not audio_data:+ return jsonify({+ "error": "No audio data provided",+ "status": "error"+ }), 400++ # Validate model size+ valid_model_sizes = ['tiny', 'base', 'small', 'medium', 'large']+ if model_size not in valid_model_sizes:+ return jsonify({+ "error": f"Invalid model size. Choose from {valid_model_sizes}",+ "status": "error"+ }), 400++ # Log the received audio data size+ print(f"Received audio data size: {len(audio_data)} characters (base64)")++ try:+ # Transcribe using VoiceTranscription helper+ text = VoiceTranscription.transcribe_bytes(+ audio_data,+ model_size=model_size,+ language=language+ )++ # Return transcription result+ return jsonify({+ "text": text,+ "is_final": is_final,+ "model_size": model_size,+ "status": "success"+ })++ except Exception as transcribe_error:+ # Detailed error logging for transcription failures+ print(f"Transcription error: {transcribe_error}")+ return jsonify({+ "error": "Transcription failed",+ "details": str(transcribe_error),+ "status": "error"+ }), 500++ except Exception as e:+ # Catch-all error handler+ print(f"Unexpected transcription error: {e}")+ return jsonify({+ "error": "Unexpected error during transcription",+ "details": str(e),+ "status": "error"+ }), 500+# # secret page, requires authentication# @app.route('/secret', methods=['GET'])# @requires_authdiff --git a/webui/index.css b/webui/index.cssindex b74817b9c..7f9b8c7b8 100644--- a/webui/index.css+++ b/webui/index.css@@ -917,6 +917,23 @@ pre {}#microphone-button.recording {background-color: #ff4136; /* Red color for recording */+ transition: background-color 0.3s ease;+}++.mic-pulse {+ animation: pulse 1.5s infinite;+}++@keyframes pulse {+ 0% {+ transform: scale(1);+ }+ 50% {+ transform: scale(1.1);+ }+ 100% {+ transform: scale(1);+ }}.chat-button svg {diff --git a/webui/index.js b/webui/index.jsindex 7aaace9b9..fbe642c97 100644--- a/webui/index.js+++ b/webui/index.js@@ -18,6 +18,7 @@ const microphoneButton = document.getElementById('microphone-button');let autoScroll = true;let context = "";let microphoneInput = null;+let isProcessingClick = false;// Initialize the toggle button@@ -150,32 +151,308 @@ sendButton.addEventListener('click', sendMessage);// MICROPHONE INPUT++class MicrophoneInput {+ /**+ * Voice Input Handler with Whisper Transcription+ *+ * Whisper Model Size Configuration:+ * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)+ * - Best for: Quick prototyping, low-resource environments+ * - Pros: Very fast, low memory usage+ * - Cons: Lowest transcription accuracy+ *+ * - 'base': Small model, good balance of speed and accuracy (~74MB)+ * - Best for: General-purpose voice input+ * - Pros: Reasonable accuracy, moderate resource usage+ * - Cons: Less accurate than larger models+ *+ * - 'small': Medium-sized model, better accuracy (~244MB)+ * - Best for: More precise transcription needs+ * - Pros: Improved accuracy over base model+ * - Cons: Slower, more memory-intensive+ *+ * - 'medium': Large model with high accuracy (~769MB)+ * - Best for: Professional transcription, multi-language support+ * - Pros: Very high accuracy+ * - Cons: Significant computational resources required+ *+ * - 'large': Largest model, highest accuracy (~1.5GB)+ * - Best for: Professional, multi-language transcription+ * - Pros: Highest possible accuracy+ * - Cons: Slowest, most resource-intensive+ *+ * Recommended Default: 'base' for most web applications+ */+ constructor(updateCallback, options = {}) {+ this.mediaRecorder = null;+ this.audioChunks = [];+ this.isRecording = false;+ this.updateCallback = updateCallback;+ this.isFinalizing = false;+ this.messageSent = false; // move messageSent into class++ // New properties for silence detection+ this.audioContext = null;+ this.mediaStreamSource = null;+ this.analyserNode = null;+ this.silenceTimer = null;+ this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed+ this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds++ this.options = {+ modelSize: 'base',+ language: null,+ chunkDuration: 3000,+ ...options+ };+ }++ async initialize() {+ try {+ const stream = await navigator.mediaDevices.getUserMedia({+ audio: {+ echoCancellation: true,+ noiseSuppression: true,+ channelCount: 1+ }+ });++ // Configure MediaRecorder+ this.mediaRecorder = new MediaRecorder(stream, {+ mimeType: 'audio/webm;codecs=opus'+ });++ // Handle audio data availability+ this.mediaRecorder.ondataavailable = async (event) => {+ if (event.data.size > 0) {+ this.audioChunks.push(event.data);+ // await this.processAudioChunk(event.data);+ }+ };++ // Handle recording stop+ this.mediaRecorder.onstop = async () => {+ await this.finalizeRecording();+ };++ // Set up AudioContext and AnalyserNode for silence detection+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);+ this.analyserNode = this.audioContext.createAnalyser();+ this.analyserNode.minDecibels = -90;+ this.analyserNode.maxDecibels = -10;+ this.analyserNode.smoothingTimeConstant = 0.85;++ this.mediaStreamSource.connect(this.analyserNode);+ } catch (error) {+ console.error('Microphone initialization error:', error);+ toast('Failed to access microphone. Please check permissions.', 'error');+ }+ }++ startSilenceDetection() {+ const dataArray = new Uint8Array(this.analyserNode.fftSize);+ const checkSilence = () => {+ this.analyserNode.getByteTimeDomainData(dataArray);++ let sum = 0;+ for (let i = 0; i < dataArray.length; i++) {+ const amplitude = (dataArray[i] - 128) / 128;+ sum += amplitude * amplitude;+ }+ const rms = Math.sqrt(sum / dataArray.length);++ if (rms < this.silenceThreshold) {+ if (!this.silenceTimer) {+ this.silenceTimer = setTimeout(() => {+ if (this.isRecording) {+ console.log('Silence detected. Stopping recording.');+ this.stopRecording();+ microphoneButton.classList.remove('recording');+ microphoneButton.classList.remove('mic-pulse');+ }+ }, this.silenceDuration);+ }+ } else {+ if (this.silenceTimer) {+ clearTimeout(this.silenceTimer);+ this.silenceTimer = null;+ }+ }++ if (this.isRecording) {+ requestAnimationFrame(checkSilence);+ }+ };++ if (this.isRecording) {+ requestAnimationFrame(checkSilence);+ }+ }++ startRecording() {+ if (this.mediaRecorder && this.audioContext) {+ this.isRecording = true;+ this.audioChunks = [];+ this.messageSent = false;+ this.mediaRecorder.start(this.options.chunkDuration);+ this.audioContext.resume();+ this.startSilenceDetection();+ }+ }++ stopRecording() {+ if (this.mediaRecorder && this.isRecording) {+ this.isRecording = false;+ if (!this.isFinalizing) {+ this.isFinalizing = true;+ this.mediaRecorder.stop();+ this.audioContext.suspend();+ if (this.silenceTimer) {+ clearTimeout(this.silenceTimer);+ this.silenceTimer = null;+ }+ }+ }+ }+++ async finalizeRecording() {+ if (this.isFinalizing) {+ this.isFinalizing = false;++ if (this.audioChunks.length > 0) {+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });+ this.audioChunks = []; // Clear for next recording++ const reader = new FileReader();+ reader.onloadend = async () => {+ const base64Data = reader.result.split(',')[1];++ try {+ const response = await fetch('/transcribe', {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json'+ },+ body: JSON.stringify({+ audio_data: base64Data,+ model_size: this.options.modelSize,+ language: this.options.language,+ is_final: true+ })+ });++ const result = await response.json();++ if (result.text) {+ console.log('Final transcription received:', result.text);+ await this.updateCallback(result.text, true);+ } else {+ console.warn('Final transcription returned empty text.');+ }+ } catch (transcribeError) {+ console.error('Final transcription error:', transcribeError);+ toast('Final transcription failed.', 'error');+ } finally {+ // Reset the microphone button state+ microphoneButton.classList.remove('recording');+ microphoneButton.classList.remove('mic-pulse');+ microphoneButton.style.backgroundColor = '';+ }+ };+ reader.readAsDataURL(audioBlob);+ }+ }+ }+}++export default MicrophoneInput;++async function initializeMicrophoneInput() {- microphoneInput = new MicrophoneInput(updateChatInput);+ console.log('Initializing microphone input');++ microphoneInput = new MicrophoneInput(+ async (text, isFinal) => {+ if (isFinal) {+ console.log('Final transcription callback received:', text);+ chatInput.value = text;+ adjustTextareaHeight();++ if (!microphoneInput.messageSent) {+ microphoneInput.messageSent = true;+ console.log('Sending message');+ await sendMessage();++ // Clear the chat input after sending the message+ chatInput.value = '';+ adjustTextareaHeight();+ }+ }+ },+ {+ modelSize: 'base',+ language: 'en',+ silenceThreshold: 0.07, // Adjust as needed+ silenceDuration: 2000, // Adjust as needed+ onError: (error) => {+ console.error('Microphone input error:', error);+ toast('Microphone error: ' + error.message, 'error');+ // Reset recording state+ if (microphoneButton.classList.contains('recording')) {+ microphoneButton.classList.remove('recording');+ }+ }+ }+ );+await microphoneInput.initialize();}+function updateChatInput(text) {- chatInput.value += text + ' ';+ console.log('updateChatInput called with:', text);++ // Ensure the text is not undefined or null+ if (!text) {+ console.warn('Received empty transcription text');+ return;+ }++ // Append text with proper spacing+ const currentValue = chatInput.value;+ const needsSpace = currentValue.length > 0 && !currentValue.endsWith(' ');+ chatInput.value = currentValue + (needsSpace ? ' ' : '') + text + ' ';++ // Adjust height and trigger input eventadjustTextareaHeight();+ chatInput.dispatchEvent(new Event('input'));++ console.log('Updated chat input value:', chatInput.value);}-microphoneButton.addEventListener('click', () => {- if (!microphoneInput) {- initializeMicrophoneInput().then(() => {- toggleRecording();- });- } else {- toggleRecording();- }-});++function toggleRecording() {+ console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);+if (microphoneInput.isRecording) {microphoneInput.stopRecording();microphoneButton.classList.remove('recording');+ // Add pulsing animation class+ microphoneButton.classList.remove('mic-pulse');} else {microphoneInput.startRecording();microphoneButton.classList.add('recording');+ // Add pulsing animation class+ microphoneButton.classList.add('mic-pulse');}++ // Add visual feedback+ microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';+ console.log('New recording state:', microphoneInput.isRecording);}+// Some error handling for microphone inputasync function requestMicrophonePermission() {try {@@ -189,15 +466,25 @@ async function requestMicrophonePermission() {}// microphoneButton click event listener modifiermicrophoneButton.addEventListener('click', async () => {+ console.log('Microphone button clicked');+ if (isProcessingClick) {+ console.log('Click already being processed, ignoring');+ return;+ }+ isProcessingClick = true;+const hasPermission = await requestMicrophonePermission();if (!hasPermission) return;+if (!microphoneInput) {- initializeMicrophoneInput().then(() => {- toggleRecording();- });- } else {- toggleRecording();+ await initializeMicrophoneInput();}++ await toggleRecording();++ setTimeout(() => {+ isProcessingClick = false;+ }, 300); // Add a 300ms delay before allowing another click});function updateUserTime() {