Skip to content

Agent Zero AI framework

feature: openai-whisper voice input

Alessandro82ca0d800ab6
diff --git a/python/helpers/voice_transcription.py b/python/helpers/voice_transcription.py
new file mode 100644
index 000000000..97e6196b9
--- /dev/null
+++ b/python/helpers/voice_transcription.py
@@ -0,0 +1,105 @@
+import whisper
+import io
+import base64
+import numpy as np
+from typing import Optional, Union, BinaryIO
+from whisper.audio import load_audio
+import tempfile
+import os
+import subprocess
+import warnings
+
+# suppress FutureWarning from torch.load
+warnings.filterwarnings('ignore', category=FutureWarning)
+
+class VoiceTranscription:
+ @staticmethod
+ def load_model(model_size: str = "base"):
+ """
+ Load a Whisper model with the specified size.
+ """
+ try:
+ return whisper.load_model(model_size)
+ except Exception as e:
+ print(f"Error loading Whisper model: {e}")
+ return None
+
+ @classmethod
+ def transcribe_bytes(cls, audio_bytes: Union[str, bytes, BinaryIO],
+ model_size: str = "base",
+ language: Optional[str] = None) -> str:
+ """
+ Transcribe audio from bytes or a file-like object.
+ """
+ model = cls.load_model(model_size)
+ if not model:
+ raise RuntimeError("Could not load Whisper model")
+
+ # Decode audio bytes if encoded as a base64 string
+ if isinstance(audio_bytes, str):
+ try:
+ audio_bytes = base64.b64decode(audio_bytes)
+ except Exception as e:
+ print(f"Error decoding base64 audio data: {e}")
+ raise
+
+ # Save audio bytes to a temporary file with .webm extension
+ with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp_input_file:
+ tmp_input_file.write(audio_bytes)
+ temp_input_path = tmp_input_file.name
+
+ try:
+ # Define the output path with .wav extension
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_output_file:
+ temp_output_path = tmp_output_file.name
+
+ # Convert WebM to WAV using FFmpeg
+ ffmpeg_cmd = [
+ 'ffmpeg', '-y', '-i', temp_input_path,
+ '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
+ temp_output_path
+ ]
+
+ print(f"Running FFmpeg command: {' '.join(ffmpeg_cmd)}")
+
+ # Run FFmpeg command using subprocess
+ try:
+ subprocess.run(
+ ffmpeg_cmd, stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL, check=True # Suppressed stderr
+ )
+ except subprocess.CalledProcessError as e:
+ error_message = e.stderr.decode().strip()
+ print(f"FFmpeg error: {error_message}")
+
+ # Log the temporary file path for debugging
+ print(f"Transcribing audio from temporary file: {temp_output_path}")
+
+ # Load audio using Whisper's load_audio
+ audio = load_audio(temp_output_path)
+
+ # Transcribe using the Whisper model
+ result = model.transcribe(audio, fp16=False, language=language)
+ text = result.get("text", "").strip()
+
+ # Log the transcription result
+ print(f"Transcription result: {text}")
+
+ return text
+
+ except subprocess.CalledProcessError as e:
+ error_message = e.stderr.decode().strip()
+ print(f"FFmpeg error: {error_message}")
+ # Return empty string or handle as appropriate
+ return ""
+ except Exception as transcribe_error:
+ print(f"Transcription error: {transcribe_error}")
+ # Return empty string or handle as appropriate
+ return ""
+ finally:
+
+ # Clean up temporary files
+ if os.path.exists(temp_input_path):
+ os.remove(temp_input_path)
+ if os.path.exists(temp_output_path):
+ os.remove(temp_output_path)
diff --git a/requirements.txt b/requirements.txt
index 8e3fac3df..192cdd06c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,6 +14,7 @@ langchain-huggingface==0.0.3
langchain-mistralai==0.1.8
langchain-ollama==0.1.3
langchain-openai==0.1.15
+openai-whisper==20240930
lxml_html_clean==0.3.1
markdown==3.7
newspaper3k==0.2.8
diff --git a/run_ui.py b/run_ui.py
index a286ff94e..22e77e1a5 100644
--- a/run_ui.py
+++ b/run_ui.py
@@ -13,6 +13,8 @@ from python.helpers.files import get_abs_path
from python.helpers.print_style import PrintStyle
from python.helpers.dotenv import load_dotenv
from python.helpers import persist_chat, settings
+from python.helpers.voice_transcription import VoiceTranscription
+import base64
from werkzeug.utils import secure_filename
@@ -133,6 +135,78 @@ async def health_check():
return "OK"
[email protected]('/transcribe', methods=['POST'])
+def transcribe_audio():
+ """
+ Transcribe audio data using Whisper.
+ Expected JSON payload:
+ {
+ 'audio_data': base64 encoded audio,
+ 'model_size': 'base', # Optional, defaults to 'base'
+ 'language': None, # Optional language code
+ 'is_final': False # Optional flag for final transcription
+ }
+ """
+ try:
+ # Parse request data
+ data = request.json
+ audio_data = data.get('audio_data')
+ model_size = data.get('model_size', 'base')
+ language = data.get('language')
+ is_final = data.get('is_final', False)
+
+ # Validate input
+ if not audio_data:
+ return jsonify({
+ "error": "No audio data provided",
+ "status": "error"
+ }), 400
+
+ # Validate model size
+ valid_model_sizes = ['tiny', 'base', 'small', 'medium', 'large']
+ if model_size not in valid_model_sizes:
+ return jsonify({
+ "error": f"Invalid model size. Choose from {valid_model_sizes}",
+ "status": "error"
+ }), 400
+
+ # Log the received audio data size
+ print(f"Received audio data size: {len(audio_data)} characters (base64)")
+
+ try:
+ # Transcribe using VoiceTranscription helper
+ text = VoiceTranscription.transcribe_bytes(
+ audio_data,
+ model_size=model_size,
+ language=language
+ )
+
+ # Return transcription result
+ return jsonify({
+ "text": text,
+ "is_final": is_final,
+ "model_size": model_size,
+ "status": "success"
+ })
+
+ except Exception as transcribe_error:
+ # Detailed error logging for transcription failures
+ print(f"Transcription error: {transcribe_error}")
+ return jsonify({
+ "error": "Transcription failed",
+ "details": str(transcribe_error),
+ "status": "error"
+ }), 500
+
+ except Exception as e:
+ # Catch-all error handler
+ print(f"Unexpected transcription error: {e}")
+ return jsonify({
+ "error": "Unexpected error during transcription",
+ "details": str(e),
+ "status": "error"
+ }), 500
+
# # secret page, requires authentication
# @app.route('/secret', methods=['GET'])
# @requires_auth
diff --git a/webui/index.css b/webui/index.css
index b74817b9c..7f9b8c7b8 100644
--- a/webui/index.css
+++ b/webui/index.css
@@ -917,6 +917,23 @@ pre {
}
#microphone-button.recording {
background-color: #ff4136; /* Red color for recording */
+ transition: background-color 0.3s ease;
+}
+
+.mic-pulse {
+ animation: pulse 1.5s infinite;
+}
+
+@keyframes pulse {
+ 0% {
+ transform: scale(1);
+ }
+ 50% {
+ transform: scale(1.1);
+ }
+ 100% {
+ transform: scale(1);
+ }
}
.chat-button svg {
diff --git a/webui/index.js b/webui/index.js
index 7aaace9b9..fbe642c97 100644
--- a/webui/index.js
+++ b/webui/index.js
@@ -18,6 +18,7 @@ const microphoneButton = document.getElementById('microphone-button');
let autoScroll = true;
let context = "";
let microphoneInput = null;
+let isProcessingClick = false;
// Initialize the toggle button
@@ -150,32 +151,308 @@ sendButton.addEventListener('click', sendMessage);
// MICROPHONE INPUT
+
+class MicrophoneInput {
+ /**
+ * Voice Input Handler with Whisper Transcription
+ *
+ * Whisper Model Size Configuration:
+ * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
+ * - Best for: Quick prototyping, low-resource environments
+ * - Pros: Very fast, low memory usage
+ * - Cons: Lowest transcription accuracy
+ *
+ * - 'base': Small model, good balance of speed and accuracy (~74MB)
+ * - Best for: General-purpose voice input
+ * - Pros: Reasonable accuracy, moderate resource usage
+ * - Cons: Less accurate than larger models
+ *
+ * - 'small': Medium-sized model, better accuracy (~244MB)
+ * - Best for: More precise transcription needs
+ * - Pros: Improved accuracy over base model
+ * - Cons: Slower, more memory-intensive
+ *
+ * - 'medium': Large model with high accuracy (~769MB)
+ * - Best for: Professional transcription, multi-language support
+ * - Pros: Very high accuracy
+ * - Cons: Significant computational resources required
+ *
+ * - 'large': Largest model, highest accuracy (~1.5GB)
+ * - Best for: Professional, multi-language transcription
+ * - Pros: Highest possible accuracy
+ * - Cons: Slowest, most resource-intensive
+ *
+ * Recommended Default: 'base' for most web applications
+ */
+ constructor(updateCallback, options = {}) {
+ this.mediaRecorder = null;
+ this.audioChunks = [];
+ this.isRecording = false;
+ this.updateCallback = updateCallback;
+ this.isFinalizing = false;
+ this.messageSent = false; // move messageSent into class
+
+ // New properties for silence detection
+ this.audioContext = null;
+ this.mediaStreamSource = null;
+ this.analyserNode = null;
+ this.silenceTimer = null;
+ this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
+ this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
+
+ this.options = {
+ modelSize: 'base',
+ language: null,
+ chunkDuration: 3000,
+ ...options
+ };
+ }
+
+ async initialize() {
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ channelCount: 1
+ }
+ });
+
+ // Configure MediaRecorder
+ this.mediaRecorder = new MediaRecorder(stream, {
+ mimeType: 'audio/webm;codecs=opus'
+ });
+
+ // Handle audio data availability
+ this.mediaRecorder.ondataavailable = async (event) => {
+ if (event.data.size > 0) {
+ this.audioChunks.push(event.data);
+ // await this.processAudioChunk(event.data);
+ }
+ };
+
+ // Handle recording stop
+ this.mediaRecorder.onstop = async () => {
+ await this.finalizeRecording();
+ };
+
+ // Set up AudioContext and AnalyserNode for silence detection
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
+ this.analyserNode = this.audioContext.createAnalyser();
+ this.analyserNode.minDecibels = -90;
+ this.analyserNode.maxDecibels = -10;
+ this.analyserNode.smoothingTimeConstant = 0.85;
+
+ this.mediaStreamSource.connect(this.analyserNode);
+ } catch (error) {
+ console.error('Microphone initialization error:', error);
+ toast('Failed to access microphone. Please check permissions.', 'error');
+ }
+ }
+
+ startSilenceDetection() {
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
+ const checkSilence = () => {
+ this.analyserNode.getByteTimeDomainData(dataArray);
+
+ let sum = 0;
+ for (let i = 0; i < dataArray.length; i++) {
+ const amplitude = (dataArray[i] - 128) / 128;
+ sum += amplitude * amplitude;
+ }
+ const rms = Math.sqrt(sum / dataArray.length);
+
+ if (rms < this.silenceThreshold) {
+ if (!this.silenceTimer) {
+ this.silenceTimer = setTimeout(() => {
+ if (this.isRecording) {
+ console.log('Silence detected. Stopping recording.');
+ this.stopRecording();
+ microphoneButton.classList.remove('recording');
+ microphoneButton.classList.remove('mic-pulse');
+ }
+ }, this.silenceDuration);
+ }
+ } else {
+ if (this.silenceTimer) {
+ clearTimeout(this.silenceTimer);
+ this.silenceTimer = null;
+ }
+ }
+
+ if (this.isRecording) {
+ requestAnimationFrame(checkSilence);
+ }
+ };
+
+ if (this.isRecording) {
+ requestAnimationFrame(checkSilence);
+ }
+ }
+
+ startRecording() {
+ if (this.mediaRecorder && this.audioContext) {
+ this.isRecording = true;
+ this.audioChunks = [];
+ this.messageSent = false;
+ this.mediaRecorder.start(this.options.chunkDuration);
+ this.audioContext.resume();
+ this.startSilenceDetection();
+ }
+ }
+
+ stopRecording() {
+ if (this.mediaRecorder && this.isRecording) {
+ this.isRecording = false;
+ if (!this.isFinalizing) {
+ this.isFinalizing = true;
+ this.mediaRecorder.stop();
+ this.audioContext.suspend();
+ if (this.silenceTimer) {
+ clearTimeout(this.silenceTimer);
+ this.silenceTimer = null;
+ }
+ }
+ }
+ }
+
+
+ async finalizeRecording() {
+ if (this.isFinalizing) {
+ this.isFinalizing = false;
+
+ if (this.audioChunks.length > 0) {
+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
+ this.audioChunks = []; // Clear for next recording
+
+ const reader = new FileReader();
+ reader.onloadend = async () => {
+ const base64Data = reader.result.split(',')[1];
+
+ try {
+ const response = await fetch('/transcribe', {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json'
+ },
+ body: JSON.stringify({
+ audio_data: base64Data,
+ model_size: this.options.modelSize,
+ language: this.options.language,
+ is_final: true
+ })
+ });
+
+ const result = await response.json();
+
+ if (result.text) {
+ console.log('Final transcription received:', result.text);
+ await this.updateCallback(result.text, true);
+ } else {
+ console.warn('Final transcription returned empty text.');
+ }
+ } catch (transcribeError) {
+ console.error('Final transcription error:', transcribeError);
+ toast('Final transcription failed.', 'error');
+ } finally {
+ // Reset the microphone button state
+ microphoneButton.classList.remove('recording');
+ microphoneButton.classList.remove('mic-pulse');
+ microphoneButton.style.backgroundColor = '';
+ }
+ };
+ reader.readAsDataURL(audioBlob);
+ }
+ }
+ }
+}
+
+export default MicrophoneInput;
+
+
async function initializeMicrophoneInput() {
- microphoneInput = new MicrophoneInput(updateChatInput);
+ console.log('Initializing microphone input');
+
+ microphoneInput = new MicrophoneInput(
+ async (text, isFinal) => {
+ if (isFinal) {
+ console.log('Final transcription callback received:', text);
+ chatInput.value = text;
+ adjustTextareaHeight();
+
+ if (!microphoneInput.messageSent) {
+ microphoneInput.messageSent = true;
+ console.log('Sending message');
+ await sendMessage();
+
+ // Clear the chat input after sending the message
+ chatInput.value = '';
+ adjustTextareaHeight();
+ }
+ }
+ },
+ {
+ modelSize: 'base',
+ language: 'en',
+ silenceThreshold: 0.07, // Adjust as needed
+ silenceDuration: 2000, // Adjust as needed
+ onError: (error) => {
+ console.error('Microphone input error:', error);
+ toast('Microphone error: ' + error.message, 'error');
+ // Reset recording state
+ if (microphoneButton.classList.contains('recording')) {
+ microphoneButton.classList.remove('recording');
+ }
+ }
+ }
+ );
+
await microphoneInput.initialize();
}
+
function updateChatInput(text) {
- chatInput.value += text + ' ';
+ console.log('updateChatInput called with:', text);
+
+ // Ensure the text is not undefined or null
+ if (!text) {
+ console.warn('Received empty transcription text');
+ return;
+ }
+
+ // Append text with proper spacing
+ const currentValue = chatInput.value;
+ const needsSpace = currentValue.length > 0 && !currentValue.endsWith(' ');
+ chatInput.value = currentValue + (needsSpace ? ' ' : '') + text + ' ';
+
+ // Adjust height and trigger input event
adjustTextareaHeight();
+ chatInput.dispatchEvent(new Event('input'));
+
+ console.log('Updated chat input value:', chatInput.value);
}
-microphoneButton.addEventListener('click', () => {
- if (!microphoneInput) {
- initializeMicrophoneInput().then(() => {
- toggleRecording();
- });
- } else {
- toggleRecording();
- }
-});
+
+
function toggleRecording() {
+ console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
+
if (microphoneInput.isRecording) {
microphoneInput.stopRecording();
microphoneButton.classList.remove('recording');
+ // Add pulsing animation class
+ microphoneButton.classList.remove('mic-pulse');
} else {
microphoneInput.startRecording();
microphoneButton.classList.add('recording');
+ // Add pulsing animation class
+ microphoneButton.classList.add('mic-pulse');
}
+
+ // Add visual feedback
+ microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
+ console.log('New recording state:', microphoneInput.isRecording);
}
+
// Some error handling for microphone input
async function requestMicrophonePermission() {
try {
@@ -189,15 +466,25 @@ async function requestMicrophonePermission() {
}
// microphoneButton click event listener modifier
microphoneButton.addEventListener('click', async () => {
+ console.log('Microphone button clicked');
+ if (isProcessingClick) {
+ console.log('Click already being processed, ignoring');
+ return;
+ }
+ isProcessingClick = true;
+
const hasPermission = await requestMicrophonePermission();
if (!hasPermission) return;
+
if (!microphoneInput) {
- initializeMicrophoneInput().then(() => {
- toggleRecording();
- });
- } else {
- toggleRecording();
+ await initializeMicrophoneInput();
}
+
+ await toggleRecording();
+
+ setTimeout(() => {
+ isProcessingClick = false;
+ }, 300); // Add a 300ms delay before allowing another click
});
function updateUserTime() {