Agent Zero AI framework
stt+tts fix
frdel1953cae25890
diff --cc preload.pyindex 0c8118e49,68789e97a..2494a7091--- a/preload.py+++ b/preload.py@@@ -1,8 -1,15 +1,9 @@@import asynciofrom python.helpers import runtime, whisper, settingsfrom python.helpers.print_style import PrintStyle++from python.helpers import kokoro_ttsimport models-try:- from python.helpers import kokoro_tts- KOKORO_AVAILABLE = True-except ImportError:- KOKORO_AVAILABLE = False-PrintStyle().print("Running preload...")runtime.initialize()@@@ -22,19 -29,24 +23,24 @@@ async def preload()async def preload_embedding():if set["embed_model_provider"] == models.ModelProvider.HUGGINGFACE.name:try:- emb_mod = models.get_huggingface_embedding(set["embed_model_name"])+ # Use the new LiteLLM-based model system+ emb_mod = models.get_embedding_model(- models.ModelProvider.HUGGINGFACE,- set["embed_model_name"]++ models.ModelProvider.HUGGINGFACE, set["embed_model_name"]+ )emb_txt = await emb_mod.aembed_query("test")return emb_txtexcept Exception as e:PrintStyle().error(f"Error in preload_embedding: {e}")+ # preload kokoro tts model if enabled+ async def preload_kokoro():- if KOKORO_AVAILABLE and set.get("tts_enabled"):- try:- return await kokoro_tts.preload()- except Exception as e:- PrintStyle().error(f"Error in preload_kokoro: {e}")++ try:++ return await kokoro_tts.preload()++ except Exception as e:++ PrintStyle().error(f"Error in preload_kokoro: {e}")# async tasks to preload-- tasks = [preload_whisper(), preload_embedding()]- if KOKORO_AVAILABLE:- tasks.append(preload_kokoro())++ tasks = [preload_whisper(), preload_embedding(), preload_kokoro()]await asyncio.gather(*tasks, return_exceptions=True)PrintStyle().print("Preload completed")diff --cc python/api/synthesize.pyindex 000000000,c09b000e6..4d33254f8mode 000000,100644..100644--- a/python/api/synthesize.py+++ b/python/api/synthesize.py@@@ -1,0 -1,95 +1,92 @@@+ # api/synthesize.py++ import re+ from python.helpers.api import ApiHandler+ from flask import Request, Response++ from python.helpers import runtime, settings, kokoro_tts++ class Synthesize(ApiHandler):+ async def process(self, input: dict, request: Request) -> dict | Response:+ text = input.get("text", "")+ ctxid = input.get("ctxid", "")+- # DEBUG: Log what we received- print(f"[SYNTHESIZE DEBUG] Received text: '{text[:100]}...' (length: {len(text)})")-+ context = self.get_context(ctxid)+ if await kokoro_tts.is_downloading():+ context.log.log(type="info", content="Kokoro TTS model is currently being downloaded, please wait...")++ try:+ # Clean and chunk text for long responses+ cleaned_text = self._clean_text(text)+ chunks = self._chunk_text(cleaned_text)++ if len(chunks) == 1:+ # Single chunk - return as before+ audio = await kokoro_tts.synthesize_sentences(chunks)+ return {"audio": audio, "success": True}+ else:+ # Multiple chunks - return as sequence+ audio_parts = []+ for chunk in chunks:+ chunk_audio = await kokoro_tts.synthesize_sentences([chunk])+ audio_parts.append(chunk_audio)+ return {"audio_parts": audio_parts, "success": True}+ except Exception as e:+ return {"error": str(e), "success": False}++ def _clean_text(self, text: str) -> str:+ """Clean text by removing markdown, tables, code blocks, and other formatting"""+ # Remove code blocks+ text = re.sub(r'```[\s\S]*?```', '', text)+ text = re.sub(r'`[^`]*`', '', text)++ # Remove markdown links+ text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text)++ # Remove markdown formatting+ text = re.sub(r'[*_#]+', '', text)++ # Remove tables (basic cleanup)+ text = re.sub(r'\|[^\n]*\|', '', text)++ # Remove extra whitespace and newlines+ text = re.sub(r'\n+', ' ', text)+ text = re.sub(r'\s+', ' ', text)++ # Remove URLs+ text = re.sub(r'https?://[^\s]+', '', text)++ # Remove email addresses+ text = re.sub(r'\S+@\S+', '', text)++ return text.strip()++ def _chunk_text(self, text: str) -> list[str]:+ """Split text into manageable chunks for TTS"""+ # If text is short enough, return as single chunk+ if len(text) <= 300:+ return [text]++ # Split into sentences first+ sentences = re.split(r'(?<=[.!?])\s+', text)++ chunks = []+ current_chunk = ""++ for sentence in sentences:+ sentence = sentence.strip()+ if not sentence:+ continue++ # If adding this sentence would make chunk too long, start new chunk+ if current_chunk and len(current_chunk + " " + sentence) > 300:+ chunks.append(current_chunk.strip())+ current_chunk = sentence+ else:+ current_chunk += (" " if current_chunk else "") + sentence++ # Add the last chunk if it has content+ if current_chunk.strip():+ chunks.append(current_chunk.strip())++ return chunks if chunks else [text]diff --cc python/helpers/kokoro_tts.pyindex 000000000,7fe85bd92..20f6036f0mode 000000,100644..100644--- a/python/helpers/kokoro_tts.py+++ b/python/helpers/kokoro_tts.py@@@ -1,0 -1,90 +1,91 @@@+ # kokoro_tts.py++ import base64+ import io+ import warnings+ import asyncio+ import soundfile as sf+ from python.helpers import runtime+ from python.helpers.print_style import PrintStyle++ warnings.filterwarnings("ignore", category=FutureWarning)++ _pipeline = None-_voice = "af_heart"++_voice = "am_puck,am_onyx"++_speed = 1.1+ is_updating_model = False++ async def preload():+ try:+ return await runtime.call_development_function(_preload)+ except Exception as e:+ if not runtime.is_development():+ raise e+ # Fallback to direct execution if RFC fails in development+ PrintStyle.standard("RFC failed, falling back to direct execution...")+ return await _preload()++ async def _preload():+ global _pipeline, is_updating_model++ while is_updating_model:+ await asyncio.sleep(0.1)++ try:+ is_updating_model = True+ if not _pipeline:+ PrintStyle.standard("Loading Kokoro TTS model...")+ from kokoro import KPipeline+ _pipeline = KPipeline(lang_code='a')+ finally:+ is_updating_model = False++ async def is_downloading():+ try:+ return await runtime.call_development_function(_is_downloading)+ except Exception as e:+ if not runtime.is_development():+ raise e+ # Fallback to direct execution if RFC fails in development+ return _is_downloading()++ def _is_downloading():+ return is_updating_model++ async def synthesize_sentences(sentences: list[str]):+ """Generate audio for multiple sentences and return concatenated base64 audio"""+ try:+ return await runtime.call_development_function(_synthesize_sentences, sentences)+ except Exception as e:+ if not runtime.is_development():+ raise e+ # Fallback to direct execution if RFC fails in development+ return await _synthesize_sentences(sentences)++ async def _synthesize_sentences(sentences: list[str]):+ await _preload()++ combined_audio = []++ try:+ for sentence in sentences:+ if sentence.strip():- segments = _pipeline(sentence.strip(), voice=_voice)++ segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed)+ segment_list = list(segments)++ for segment in segment_list:+ audio_tensor = segment.audio+ audio_numpy = audio_tensor.detach().cpu().numpy()+ combined_audio.extend(audio_numpy)++ # Convert combined audio to bytes+ buffer = io.BytesIO()+ sf.write(buffer, combined_audio, 24000, format='WAV')+ audio_bytes = buffer.getvalue()++ # Return base64 encoded audio+ return base64.b64encode(audio_bytes).decode('utf-8')++ except Exception as e:+ PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}")+ raisediff --cc python/helpers/settings.pyindex 1b806de76,452b12d14..2b1ba258f--- a/python/helpers/settings.py+++ b/python/helpers/settings.py@@@ -691,11 -681,24 +691,13 @@@ def convert_out(settings: Settings) ->})+ stt_section: SettingsSection = {- "id": "stt",- "title": "Speech to Text",- "description": "Voice transcription preferences and server turn detection settings.",- "fields": stt_fields,+ # TTS fields+ tts_fields: list[SettingsField] = []++ tts_fields.append(+ {+ "id": "tts_enabled",- "title": "Enable Kokoro TTS",- "description": "Enable server-side AI text-to-speech (Kokoro)",- "type": "switch",- "value": settings["tts_enabled"],- }- )-- speech_section: SettingsSection = {- "id": "speech",- "title": "Speech",- "description": "Voice transcription and speech synthesis settings.",- "fields": stt_fields + tts_fields,"tab": "agent",}@@@ -822,9 -825,10 +824,10 @@@agent_section,chat_model_section,util_model_section,- embed_model_section,browser_model_section,- # memory_section,+ embed_model_section,+ stt_section,+ speech_section,api_keys_section,auth_section,mcp_client_section,@@@ -1016,8 -1003,9 +1019,9 @@@ def get_default_settings() -> Settingsstt_silence_threshold=0.3,stt_silence_duration=1000,stt_waiting_timeout=2000,+ tts_enabled=False,- mcp_servers='{\n "mcpServers": {\n "render": {\n "command": "docker",\n "args": [\n "run",\n "-i",\n "--rm",\n "-e",\n "RENDER_API_KEY",\n "-v",\n "render-mcp-server-config:/config",\n "ghcr.io/render-oss/render-mcp-server"\n ],\n "env": {\n "RENDER_API_KEY": "rnd_C2reJ87CxaM8vQI8DqtBfL2ymwzA"\n }\n }\n }\n}',- mcp_client_init_timeout=5,+ mcp_servers='{\n "mcpServers": {}\n}',+ mcp_client_init_timeout=10,mcp_client_tool_timeout=120,mcp_server_enabled=False,mcp_server_token=create_auth_token(),diff --cc webui/index.htmlindex b782da944,e0f0f94c3..d07691fbd--- a/webui/index.html+++ b/webui/index.html@@@ -100,7 -148,7 +100,6 @@@<!-- Load module scripts first --><script type="module" src="js/scheduler.js"></script>-- <script type="module" src="js/speech.js"></script><script type="module" src="js/history.js"></script><script type="module" src="index.js"></script>@@@ -361,15 -403,15 +360,12 @@@<button class="toast__copy" style="display: none;">Copy</button><button class="toast__close">Close</button></div>-- <div id="progress-bar-box" x-data="{ isSpeaking: false }" x-init="-- // Watch the speech synthesis status to update isSpeaking- setInterval(() => isSpeaking = window.speech && window.speech.isSpeaking(), 1000)- setInterval(() => isSpeaking = window.speech.isSpeaking(), 100)-- ">++ <div id="progress-bar-box"><h4 id="progress-bar-h"><span id="progress-bar-i">|></span><span id="progress-bar"></span></h4>-- <h4 id="progress-bar-stop-speech" x-show="isSpeaking">-- <span id="stop-speech" @click="window.speech.stop()" style="cursor: pointer">Stop Speech</span>++ <h4 id="progress-bar-stop-speech" x-data x-cloak x-show="$store.speech.isSpeaking">++ <span id="stop-speech" @click="$store.speech.stop()" style="cursor: pointer">Stop Speech</span></h4></div><div id="input-section" x-data="{@@@ -424,7 -517,7 +420,9 @@@<!-- Microphone button --><button class="chat-button mic-inactive" id="microphone-button"-- aria-label="Start/Stop recording">++ aria-label="Start/Stop recording"++ @click="$store.speech.handleMicrophoneClick()"++ x-effect="$store.speech.updateMicrophoneButtonUI()"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor"><pathd="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z" />diff --cc webui/index.jsindex 8354dc100,a60f230bd..db01fc256--- a/webui/index.js+++ b/webui/index.js@@@ -1,9 -1,6 +1,9 @@@import * as msgs from "./js/messages.js";--import { speech } from "./js/speech.js";import * as api from "./js/api.js";+import * as css from "./js/css.js";+import { sleep } from "./js/sleep.js";+import { store as attachmentsStore } from "./components/chat/attachments/attachmentsStore.js";++import { store as speechStore } from "./js/speech-store.js";window.fetchApi = api.fetchApi; // TODO - backward compatibility for non-modular scripts, remove once refactored to alpine@@@ -502,28 -517,17 +502,28 @@@ function afterMessagesUpdate(logs)}function speakMessages(logs) {- // EXTENDED FIX: Only speak VERY long responses to ensure completeness+ // log.no, log.type, log.heading, log.contentfor (let i = logs.length - 1; i >= 0; i--) {const log = logs[i];- if (log.type == "response" && !log.temp) {- if (log.no > lastSpokenNo && log.content.length > 300) { // Much higher threshold+ // finished response+ if (log.type == "response" && log.kvps && log.kvps.finished) {+ if (log.no > lastSpokenNo) {lastSpokenNo = log.no;- console.log(`[SPEECH] Speaking full response (${log.content.length} chars): "${log.content.substring(0, 200)}..."`);-- speech.speak(log.content);++ speechStore.speak(log.content);+ return;+ }+ // finished LLM headline, not response+ } else if (+ log.type == "agent" &&+ log.kvps &&+ log.kvps.headline &&+ log.kvps.tool_args &&+ log.kvps.tool_name != "response"+ ) {+ if (log.no > lastSpokenNo) {+ lastSpokenNo = log.no;- speech.speak(log.kvps.headline);++ speechStore.speak(log.kvps.headline);return;- } else if (log.content.length <= 300) {- console.log(`[SPEECH] Skipping short response (${log.content.length} chars): "${log.content.substring(0, 100)}..."`);}}}@@@ -715,6 -717,12 +715,9 @@@ export const setContext = function (idlastLogGuid = "";lastLogVersion = 0;lastSpokenNo = 0;+- // CRITICAL FIX: Clear speech buffer and stop any playing audio- pendingSpeechContent.clear();- if (window.speechStore) {- speechStore.stop();- }++ // Stop speech when switching chats++ speechStore.stop();// Clear the chat history immediately to avoid showing stale contentchatHistory.innerHTML = "";@@@ -775,7 -779,7 +778,7 @@@ window.toggleDarkMode = function (isDarwindow.toggleSpeech = function (isOn) {console.log("Speech:", isOn);localStorage.setItem("speech", isOn);-- if (!isOn) speech.stop();++ if (!isOn) speechStore.stop();};window.nudge = async function () {diff --cc webui/js/speech-store.jsindex 000000000,489a36146..1cbab090emode 000000,100644..100644--- a/webui/js/speech-store.js+++ b/webui/js/speech-store.js@@@ -1,0 -1,522 +1,580 @@@+ import { createStore } from "./AlpineStore.js";+ import { updateChatInput, sendMessage } from "../index.js";++ const Status = {+ INACTIVE: "inactive",+ ACTIVATING: "activating",+ LISTENING: "listening",+ RECORDING: "recording",+ WAITING: "waiting",+ PROCESSING: "processing",+ };++ // Create the speech store-export const speechStore = createStore('speech', {++const model = {+ // STT Settings+ stt_model_size: "tiny",+ stt_language: "en",+ stt_silence_threshold: 0.05,+ stt_silence_duration: 1000,+ stt_waiting_timeout: 2000,++ // TTS Settings+ tts_enabled: false,++ // TTS State+ isSpeaking: false,+ currentAudio: null,+ audioContext: null,+ userHasInteracted: false,++ // STT State+ microphoneInput: null,- micStatus: Status.INACTIVE,++ isProcessingClick: false,++++ // Getter for micStatus - delegates to microphoneInput++ get micStatus() {++ return this.microphoneInput?.status || Status.INACTIVE;++ },++++ updateMicrophoneButtonUI() {++ const microphoneButton = document.getElementById("microphone-button");++ if (!microphoneButton) return;++ const status = this.micStatus;++ microphoneButton.classList.remove(++ 'mic-inactive', 'mic-activating', 'mic-listening', 'mic-recording', 'mic-waiting', 'mic-processing'++ );++ microphoneButton.classList.add(`mic-${status.toLowerCase()}`);++ microphoneButton.setAttribute("data-status", status);++ },++++ async handleMicrophoneClick() {++ if (this.isProcessingClick) return;++ this.isProcessingClick = true;++ try {++ if (!this.microphoneInput) {++ await this.initMicrophone();++ }++++ if (this.microphoneInput) {++ await this.microphoneInput.toggle();++ }++ } finally {++ setTimeout(() => {++ this.isProcessingClick = false;++ }, 300);++ }++ },++++ // Initialize speech functionality+ async init() {+ await this.loadSettings();+ this.setupBrowserTTS();+ this.setupUserInteractionHandling();+ },++ // Load settings from server+ async loadSettings() {+ try {+ const response = await fetchApi("/settings_get", { method: "POST" });+ const data = await response.json();+ const speechSection = data.settings.sections.find(s => s.title === "Speech");++ if (speechSection) {+ speechSection.fields.forEach(field => {+ if (this.hasOwnProperty(field.id)) {+ this[field.id] = field.value;+ }+ });+ }+ } catch (error) {+ window.toastFetchError("Failed to load speech settings", error);+ console.error("Failed to load speech settings:", error);+ }+ },++ // Setup browser TTS+ setupBrowserTTS() {+ this.synth = window.speechSynthesis;+ this.browserUtterance = null;+ },++ // Setup user interaction handling for autoplay policy+ setupUserInteractionHandling() {+ const enableAudio = () => {+ if (!this.userHasInteracted) {+ this.userHasInteracted = true;+ console.log("User interaction detected - audio playback enabled");++ // Create a dummy audio context to "unlock" audio+ try {+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();+ this.audioContext.resume();+ } catch (e) {+ console.log("AudioContext not available");+ }+ }+ };++ // Listen for any user interaction+ const events = ['click', 'touchstart', 'keydown', 'mousedown'];+ events.forEach(event => {+ document.addEventListener(event, enableAudio, { once: true, passive: true });+ });+ },++ // Main speak function+ async speak(text) {+ if (!this.tts_enabled) return;+ if (this.isSpeaking) return;++ text = this.cleanText(text);+ if (!text.trim()) return;++ if (!this.userHasInteracted) {+ this.showAudioPermissionPrompt();+ return;+ }++ try {+ await this.speakWithKokoro(text);+ } catch (error) {+ console.error("TTS error:", error);+ this.speakWithBrowser(text);+ }+ },++ // Show a prompt to user to enable audio+ showAudioPermissionPrompt() {+ if (window.toast) {+ window.toast("Click anywhere to enable audio playback", "info", 5000);+ } else {+ console.log("Please click anywhere on the page to enable audio playback");+ }+ },++ // Browser TTS+ speakWithBrowser(text) {+ this.browserUtterance = new SpeechSynthesisUtterance(text);+ this.browserUtterance.onstart = () => { this.isSpeaking = true; };+ this.browserUtterance.onend = () => { this.isSpeaking = false; };+ this.synth.speak(this.browserUtterance);+ },++ // Kokoro TTS+ async speakWithKokoro(text) {+ try {+ const response = await sendJsonData("/synthesize", { text });++ if (response.success) {+ if (response.audio_parts) {+ // Multiple chunks - play sequentially+ for (const audioPart of response.audio_parts) {+ await this.playAudio(audioPart);+ await new Promise(resolve => setTimeout(resolve, 100)); // Brief pause+ }+ } else if (response.audio) {+ // Single audio+ await this.playAudio(response.audio);+ }+ } else {+ console.error("Kokoro TTS error:", response.error);+ this.speakWithBrowser(text);+ }+ } catch (error) {+ console.error("Kokoro TTS error:", error);+ this.speakWithBrowser(text);+ }+ },+++ // Play base64 audio+ async playAudio(base64Audio) {+ return new Promise((resolve, reject) => {+ const audio = new Audio();++ audio.onplay = () => {+ this.isSpeaking = true;+ };+ audio.onended = () => {+ this.isSpeaking = false;+ this.currentAudio = null;+ resolve();+ };+ audio.onerror = (error) => {+ this.isSpeaking = false;+ this.currentAudio = null;+ reject(error);+ };++ audio.src = `data:audio/wav;base64,${base64Audio}`;+ this.currentAudio = audio;++ audio.play().catch(error => {+ this.isSpeaking = false;+ this.currentAudio = null;++ if (error.name === 'NotAllowedError') {+ this.showAudioPermissionPrompt();+ this.userHasInteracted = false;+ }+ reject(error);+ });+ });+ },++ // Stop all speech+ stop() {+ if (this.synth?.speaking) {+ this.synth.cancel();+ }++ if (this.currentAudio) {+ this.currentAudio.pause();+ this.currentAudio.currentTime = 0;+ this.currentAudio = null;+ }++ this.isSpeaking = false;+ },++ // Clean text for TTS+ cleanText(text) {+ return text+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, "")+ .replace(/https?:\/\/[^\s]+/g, "")+ .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "")+ .replace(/\s+/g, " ")+ .trim();+ },++ // Initialize microphone input+ async initMicrophone() {+ if (this.microphoneInput) return this.microphoneInput;++ this.microphoneInput = new MicrophoneInput(+ async (text, isFinal) => {+ if (isFinal) {+ updateChatInput(text);+ if (!this.microphoneInput.messageSent) {+ this.microphoneInput.messageSent = true;+ await sendMessage();+ }+ }+ }+ );+- this.micStatus = Status.ACTIVATING;+ const initialized = await this.microphoneInput.initialize();+ return initialized ? this.microphoneInput : null;+ },+- // Toggle microphone- async toggleMicrophone() {- const hasPermission = await this.requestMicrophonePermission();- if (!hasPermission) return;-- if (!this.microphoneInput && !(await this.initMicrophone())) {- return;- }-- this.micStatus = this.micStatus === Status.INACTIVE || this.micStatus === Status.ACTIVATING- ? Status.LISTENING- : Status.INACTIVE;- },-- // Request microphone permission++ // Request microphone permission - delegate to MicrophoneInput+ async requestMicrophonePermission() {- try {- await navigator.mediaDevices.getUserMedia({ audio: true });- return true;- } catch (err) {- console.error("Error accessing microphone:", err);- toast("Microphone access denied. Please enable microphone access in your browser settings.", "error");- return false;- }++ return this.microphoneInput ?++ this.microphoneInput.requestPermission() :++ MicrophoneInput.prototype.requestPermission.call(null);+ }-});++};++ // Microphone Input Class (simplified for store integration)+ class MicrophoneInput {+ constructor(updateCallback) {+ this.mediaRecorder = null;+ this.audioChunks = [];+ this.lastChunk = [];+ this.updateCallback = updateCallback;+ this.messageSent = false;+ this.audioContext = null;+ this.mediaStreamSource = null;+ this.analyserNode = null;+ this._status = Status.INACTIVE;+ this.lastAudioTime = null;+ this.waitingTimer = null;+ this.silenceStartTime = null;+ this.hasStartedRecording = false;+ this.analysisFrame = null;+ }++ get status() {+ return this._status;+ }++ set status(newStatus) {+ if (this._status === newStatus) return;++ const oldStatus = this._status;+ this._status = newStatus;- speechStore.micStatus = newStatus;+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);++ this.handleStatusChange(oldStatus, newStatus);+ }++ async initialize() {++ // Set status to activating at the start of initialization++ this.status = Status.ACTIVATING;+ try {+ const stream = await navigator.mediaDevices.getUserMedia({+ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 }+ });+++ // Log which device is being used++ const audioTrack = stream.getAudioTracks()[0];++ const deviceId = audioTrack.getSettings().deviceId;++ if (deviceId) {++ const devices = await navigator.mediaDevices.enumerateDevices();++ const device = devices.find(d => d.deviceId === deviceId);++ if (device) {++ console.log(`Microphone input device: ${device.label} (ID: ${device.deviceId})`);++ } else {++ console.log(`Microphone input device ID: ${deviceId} (label not found)`);++ }++ } else {++ console.log('Microphone input deviceId not available');++ }+++ this.mediaRecorder = new MediaRecorder(stream);+ this.mediaRecorder.ondataavailable = (event) => {+ if (event.data.size > 0 && (this.status === Status.RECORDING || this.status === Status.WAITING)) {+ if (this.lastChunk) {+ this.audioChunks.push(this.lastChunk);+ this.lastChunk = null;+ }+ this.audioChunks.push(event.data);+ } else if (this.status === Status.LISTENING) {+ this.lastChunk = event.data;+ }+ };++ this.setupAudioAnalysis(stream);+ return true;+ } catch (error) {+ console.error("Microphone initialization error:", error);+ toast("Failed to access microphone. Please check permissions.", "error");+ return false;+ }+ }++ handleStatusChange(oldStatus, newStatus) {+ if (newStatus != Status.RECORDING) {+ this.lastChunk = null;+ }++ switch (newStatus) {+ case Status.INACTIVE:+ this.handleInactiveState();+ break;+ case Status.LISTENING:+ this.handleListeningState();+ break;+ case Status.RECORDING:+ this.handleRecordingState();+ break;+ case Status.WAITING:+ this.handleWaitingState();+ break;+ case Status.PROCESSING:+ this.handleProcessingState();+ break;+ }+ }++ handleInactiveState() {+ this.stopRecording();+ this.stopAudioAnalysis();+ if (this.waitingTimer) {+ clearTimeout(this.waitingTimer);+ this.waitingTimer = null;+ }+ }++ handleListeningState() {+ this.stopRecording();+ this.audioChunks = [];+ this.hasStartedRecording = false;+ this.silenceStartTime = null;+ this.lastAudioTime = null;+ this.messageSent = false;+ this.startAudioAnalysis();+ }++ handleRecordingState() {+ if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") {+ this.hasStartedRecording = true;+ this.mediaRecorder.start(1000);+ console.log("Speech started");+ }+ if (this.waitingTimer) {+ clearTimeout(this.waitingTimer);+ this.waitingTimer = null;+ }+ }++ handleWaitingState() {+ this.waitingTimer = setTimeout(() => {+ if (this.status === Status.WAITING) {+ this.status = Status.PROCESSING;+ }- }, speechStore.stt_waiting_timeout);++ }, store.stt_waiting_timeout);+ }++ handleProcessingState() {+ this.stopRecording();+ this.process();+ }++ setupAudioAnalysis(stream) {+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);+ this.analyserNode = this.audioContext.createAnalyser();+ this.analyserNode.fftSize = 2048;+ this.analyserNode.minDecibels = -90;+ this.analyserNode.maxDecibels = -10;+ this.analyserNode.smoothingTimeConstant = 0.85;+ this.mediaStreamSource.connect(this.analyserNode);+ }++ startAudioAnalysis() {+ const analyzeFrame = () => {+ if (this.status === Status.INACTIVE) return;++ const dataArray = new Uint8Array(this.analyserNode.fftSize);+ this.analyserNode.getByteTimeDomainData(dataArray);++ let sum = 0;+ for (let i = 0; i < dataArray.length; i++) {+ const amplitude = (dataArray[i] - 128) / 128;+ sum += amplitude * amplitude;+ }+ const rms = Math.sqrt(sum / dataArray.length);+ const now = Date.now();++ // Update status based on audio level (ignore if TTS is speaking)- if (rms > this.densify(speechStore.stt_silence_threshold)) {++ if (rms > this.densify(store.stt_silence_threshold)) {+ this.lastAudioTime = now;+ this.silenceStartTime = null;+- if ((this.status === Status.LISTENING || this.status === Status.WAITING) && !speechStore.isSpeaking && !speechStore.isGenerating) {++ if ((this.status === Status.LISTENING || this.status === Status.WAITING) && !store.isSpeaking && !store.isGenerating) {+ this.status = Status.RECORDING;+ }+ } else if (this.status === Status.RECORDING) {+ if (!this.silenceStartTime) {+ this.silenceStartTime = now;+ }++ const silenceDuration = now - this.silenceStartTime;- if (silenceDuration >= speechStore.stt_silence_duration) {++ if (silenceDuration >= store.stt_silence_duration) {+ this.status = Status.WAITING;+ }+ }++ this.analysisFrame = requestAnimationFrame(analyzeFrame);+ };++ this.analysisFrame = requestAnimationFrame(analyzeFrame);+ }++ stopAudioAnalysis() {+ if (this.analysisFrame) {+ cancelAnimationFrame(this.analysisFrame);+ this.analysisFrame = null;+ }+ }++ stopRecording() {+ if (this.mediaRecorder?.state === "recording") {+ this.mediaRecorder.stop();+ this.hasStartedRecording = false;+ }+ }++ densify(x) {+ return Math.exp(-5 * (1 - x));+ }++ async process() {+ if (this.audioChunks.length === 0) {+ this.status = Status.LISTENING;+ return;+ }++ const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" });+ const base64 = await this.convertBlobToBase64Wav(audioBlob);++ try {+ const result = await sendJsonData("/transcribe", { audio: base64 });+ const text = this.filterResult(result.text || "");++ if (text) {+ console.log("Transcription:", result.text);+ await this.updateCallback(result.text, true);+ }+ } catch (error) {+ window.toastFetchError("Transcription error", error);+ console.error("Transcription error:", error);+ } finally {+ this.audioChunks = [];+ this.status = Status.LISTENING;+ }+ }++ convertBlobToBase64Wav(audioBlob) {+ return new Promise((resolve, reject) => {+ const reader = new FileReader();+ reader.onloadend = () => {+ const base64Data = reader.result.split(",")[1];+ resolve(base64Data);+ };+ reader.onerror = (error) => reject(error);+ reader.readAsDataURL(audioBlob);+ });+ }++ filterResult(text) {+ text = text.trim();+ let ok = false;+ while (!ok) {+ if (!text) break;+ if (text[0] === "{" && text[text.length - 1] === "}") break;+ if (text[0] === "(" && text[text.length - 1] === ")") break;+ if (text[0] === "[" && text[text.length - 1] === "]") break;+ ok = true;+ }+ if (ok) return text;+ else console.log(`Discarding transcription: ${text}`);+ }++++ // Toggle microphone between active and inactive states++ async toggle() {++ const hasPermission = await this.requestPermission();++ if (!hasPermission) return;++++ // Toggle between listening and inactive++ if (this.status === Status.INACTIVE || this.status === Status.ACTIVATING) {++ this.status = Status.LISTENING;++ } else {++ this.status = Status.INACTIVE;++ }++ }++++ // Request microphone permission++ async requestPermission() {++ try {++ await navigator.mediaDevices.getUserMedia({ audio: true });++ return true;++ } catch (err) {++ console.error("Error accessing microphone:", err);++ toast("Microphone access denied. Please enable microphone access in your browser settings.", "error");++ return false;++ }++ }+ }+++export const store = createStore('speech', model);+++ // Initialize speech store-window.speechStore = speechStore;++// window.speechStore = speechStore;++ // Event listeners-document.addEventListener("settings-updated", () => speechStore.loadSettings());-document.addEventListener("DOMContentLoaded", () => speechStore.init());++document.addEventListener("settings-updated", () => store.loadSettings());++// document.addEventListener("DOMContentLoaded", () => speechStore.init());diff --cc webui/js/speech.jsindex 040d0b783,23ade3145..000000000deleted file mode 100644,100644--- a/webui/js/speech.js+++ /dev/null@@@ -1,492 -1,52 +1,0 @@@- // import { pipeline, read_audio } from '../[email protected]';- import { updateChatInput, sendMessage } from "../index.js";- import { fetchApi } from "./api.js";-import { speechStore } from "./speech-store.js";----const microphoneButton = document.getElementById("microphone-button");- let microphoneInput = null;--let isProcessingClick = false;-- const Status = {- INACTIVE: "inactive",- ACTIVATING: "activating",- LISTENING: "listening",- RECORDING: "recording",- WAITING: "waiting",- PROCESSING: "processing",- };-- const micSettings = {- stt_model_size: "tiny",- stt_language: "en",- stt_silence_threshold: 0.05,- stt_silence_duration: 1000,- stt_waiting_timeout: 2000,- };- window.micSettings = micSettings;- loadMicSettings();-- function densify(x) {- return Math.exp(-5 * (1 - x));- }-- async function loadMicSettings() {- try {- const response = await fetchApi("/settings_get", {- method: "POST",- });- const data = await response.json();- const sttSettings = data.settings.sections.find(- (s) => s.title === "Speech to Text"- );-- if (sttSettings) {- // Update options from server settings- sttSettings.fields.forEach((field) => {- const key = field.id; //.split('.')[1]; // speech_to_text.model_size -> model_size- micSettings[key] = field.value;- });- }- } catch (error) {- window.toastFetchError("Failed to load speech settings", error);- console.error("Failed to load speech settings:", error);- }- }-- class MicrophoneInput {- constructor(updateCallback, options = {}) {- this.mediaRecorder = null;- this.audioChunks = [];- this.lastChunk = [];- this.updateCallback = updateCallback;- this.messageSent = false;-- // Audio analysis properties- this.audioContext = null;- this.mediaStreamSource = null;- this.analyserNode = null;- this._status = Status.INACTIVE;-- // Timing properties- this.lastAudioTime = null;- this.waitingTimer = null;- this.silenceStartTime = null;- this.hasStartedRecording = false;- this.analysisFrame = null;- }-- get status() {- return this._status;- }-- set status(newStatus) {- if (this._status === newStatus) return;-- const oldStatus = this._status;- this._status = newStatus;- console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);-- // Update UI- microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);- microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);- microphoneButton.setAttribute("data-status", newStatus);-- // Handle state-specific behaviors- this.handleStatusChange(oldStatus, newStatus);- }-- handleStatusChange(oldStatus, newStatus) {- //last chunk kept only for transition to recording status- if (newStatus != Status.RECORDING) {- this.lastChunk = null;- }-- switch (newStatus) {- case Status.INACTIVE:- this.handleInactiveState();- break;- case Status.LISTENING:- this.handleListeningState();- break;- case Status.RECORDING:- this.handleRecordingState();- break;- case Status.WAITING:- this.handleWaitingState();- break;- case Status.PROCESSING:- this.handleProcessingState();- break;- }- }-- handleInactiveState() {- this.stopRecording();- this.stopAudioAnalysis();- if (this.waitingTimer) {- clearTimeout(this.waitingTimer);- this.waitingTimer = null;- }- }-- handleListeningState() {- this.stopRecording();- this.audioChunks = [];- this.hasStartedRecording = false;- this.silenceStartTime = null;- this.lastAudioTime = null;- this.messageSent = false;- this.startAudioAnalysis();- }-- handleRecordingState() {- if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") {- this.hasStartedRecording = true;- this.mediaRecorder.start(1000);- console.log("Speech started");- }- if (this.waitingTimer) {- clearTimeout(this.waitingTimer);- this.waitingTimer = null;- }- }-- handleWaitingState() {- // Don't stop recording during waiting state- this.waitingTimer = setTimeout(() => {- if (this.status === Status.WAITING) {- this.status = Status.PROCESSING;- }- }, micSettings.stt_waiting_timeout);- }-- handleProcessingState() {- this.stopRecording();- this.process();- }-- stopRecording() {- if (this.mediaRecorder?.state === "recording") {- this.mediaRecorder.stop();- this.hasStartedRecording = false;- }- }-- async initialize() {- try {- const stream = await navigator.mediaDevices.getUserMedia({- audio: {- echoCancellation: true,- noiseSuppression: true,- channelCount: 1,- },- });-- this.mediaRecorder = new MediaRecorder(stream);- this.mediaRecorder.ondataavailable = (event) => {- if (- event.data.size > 0 &&- (this.status === Status.RECORDING || this.status === Status.WAITING)- ) {- if (this.lastChunk) {- this.audioChunks.push(this.lastChunk);- this.lastChunk = null;- }- this.audioChunks.push(event.data);- console.log(- "Audio chunk received, total chunks:",- this.audioChunks.length- );- } else if (this.status === Status.LISTENING) {- this.lastChunk = event.data;- }- };-- this.setupAudioAnalysis(stream);- return true;- } catch (error) {- console.error("Microphone initialization error:", error);- toast("Failed to access microphone. Please check permissions.", "error");- return false;- }- }-- setupAudioAnalysis(stream) {- this.audioContext = new (window.AudioContext ||- window.webkitAudioContext)();- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);- this.analyserNode = this.audioContext.createAnalyser();- this.analyserNode.fftSize = 2048;- this.analyserNode.minDecibels = -90;- this.analyserNode.maxDecibels = -10;- this.analyserNode.smoothingTimeConstant = 0.85;- this.mediaStreamSource.connect(this.analyserNode);- }-- startAudioAnalysis() {- const analyzeFrame = () => {- if (this.status === Status.INACTIVE) return;-- const dataArray = new Uint8Array(this.analyserNode.fftSize);- this.analyserNode.getByteTimeDomainData(dataArray);-- // Calculate RMS volume- let sum = 0;- for (let i = 0; i < dataArray.length; i++) {- const amplitude = (dataArray[i] - 128) / 128;- sum += amplitude * amplitude;- }- const rms = Math.sqrt(sum / dataArray.length);-- const now = Date.now();-- // Update status based on audio level- if (rms > densify(micSettings.stt_silence_threshold)) {- this.lastAudioTime = now;- this.silenceStartTime = null;-- if (- this.status === Status.LISTENING ||- this.status === Status.WAITING- ) {- if (!speech.isSpeaking())- // TODO? a better way to ignore agent's voice?- this.status = Status.RECORDING;- }- } else if (this.status === Status.RECORDING) {- if (!this.silenceStartTime) {- this.silenceStartTime = now;- }-- const silenceDuration = now - this.silenceStartTime;- if (silenceDuration >= micSettings.stt_silence_duration) {- this.status = Status.WAITING;- }- }-- this.analysisFrame = requestAnimationFrame(analyzeFrame);- };-- this.analysisFrame = requestAnimationFrame(analyzeFrame);- }-- stopAudioAnalysis() {- if (this.analysisFrame) {- cancelAnimationFrame(this.analysisFrame);- this.analysisFrame = null;- }- }-- async process() {- if (this.audioChunks.length === 0) {- this.status = Status.LISTENING;- return;- }-- const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" });- const base64 = await this.convertBlobToBase64Wav(audioBlob);-- try {- const result = await sendJsonData("/transcribe", { audio: base64 });-- const text = this.filterResult(result.text || "");-- if (text) {- console.log("Transcription:", result.text);- await this.updateCallback(result.text, true);- }- } catch (error) {- window.toastFetchError("Transcription error", error);- console.error("Transcription error:", error);- } finally {- this.audioChunks = [];- this.status = Status.LISTENING;- }- }-- convertBlobToBase64Wav(audioBlob) {- return new Promise((resolve, reject) => {- const reader = new FileReader();-- // Read the Blob as a Data URL- reader.onloadend = () => {- const base64Data = reader.result.split(",")[1]; // Extract Base64 data- resolve(base64Data);- };-- reader.onerror = (error) => {- reject(error);- };-- reader.readAsDataURL(audioBlob); // Start reading the Blob- });- }--- filterResult(text) {- text = text.trim();- let ok = false;- while (!ok) {- if (!text) break;- if (text[0] === "{" && text[text.length - 1] === "}") break;- if (text[0] === "(" && text[text.length - 1] === ")") break;- if (text[0] === "[" && text[text.length - 1] === "]") break;- ok = true;- }- if (ok) return text;- else console.log(`Discarding transcription: ${text}`);- }-// Update microphone button UI based on store status-function updateMicrophoneButtonUI() {- const status = speechStore.micStatus;-- microphoneButton.classList.remove('mic-inactive', 'mic-activating', 'mic-listening', 'mic-recording', 'mic-waiting', 'mic-processing');- microphoneButton.classList.add(`mic-${status.toLowerCase()}`);- microphoneButton.setAttribute("data-status", status);--}-- // Initialize and handle click events- async function initializeMicrophoneInput() {- window.microphoneInput = microphoneInput = new MicrophoneInput(- async (text, isFinal) => {- if (isFinal) {- updateChatInput(text);- if (!microphoneInput.messageSent) {- microphoneInput.messageSent = true;- await sendMessage();- }- }- }- );- microphoneInput.status = Status.ACTIVATING;--- return await microphoneInput.initialize();- }-// Watch store for status changes-document.addEventListener("alpine:init", () => {- Alpine.effect(() => {- updateMicrophoneButtonUI();- });-});---// Microphone button click handler--microphoneButton.addEventListener("click", async () => {-- if (isProcessingClick) return;-- isProcessingClick = true;-- const hasPermission = await requestMicrophonePermission();- if (!hasPermission) return;---- try {- if (!microphoneInput && !(await initializeMicrophoneInput())) {- return;- }-- // Simply toggle between INACTIVE and LISTENING states- microphoneInput.status =- microphoneInput.status === Status.INACTIVE ||- microphoneInput.status === Status.ACTIVATING- ? Status.LISTENING- : Status.INACTIVE;- await speechStore.toggleMicrophone();-- } finally {-- setTimeout(() => {-- isProcessingClick = false;-- }, 300);-- }--});-- // Some error handling for microphone input- async function requestMicrophonePermission() {- try {- await navigator.mediaDevices.getUserMedia({ audio: true });- return true;- } catch (err) {- console.error("Error accessing microphone:", err);- toast(- "Microphone access denied. Please enable microphone access in your browser settings.",- "error"- );- return false;- }- }---// Create simplified Speech class for backward compatibility--class Speech {- constructor() {- this.synth = window.speechSynthesis;- this.utterance = null;- }-- stripEmojis(str) {- // Remove emojis- return str.replace(- /([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g,- ""- );- }-- normalizeWhitespace(str) {- // Replace multiple spaces with a single space while preserving newlines- str = str.split('\n').map(line => {- return line.replace(/\s+/g, ' ').trim();- }).join('\n');-- return str.trim();- }-- speak(text) {- console.log("Starting speak:", text);- // Stop any current utterance- this.stop();-- // Process text for speech synthesis- text = this.stripEmojis(text); // remove emojis- text = this.normalizeWhitespace(text); // normalize whitespace while preserving newlines- text = this.replaceURLs(text);- text = this.replaceGuids(text);-- console.log("Speaking text:", text);- this.utterance = new SpeechSynthesisUtterance(text);-- // Speak the new utterance- this.synth.speak(this.utterance);- }-- replaceURLs(text) {- const urlRegex =- /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b(www\.)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b[-A-Z0-9+&@#\/%?=~_|!:,.;]*\.(?:[A-Z]{2,})[-A-Z0-9+&@#\/%?=~_|])/gi;- return text.replace(urlRegex, (url) => {- let text = url;- // if contains ://, split by it- if (text.includes("://")) text = text.split("://")[1];- // if contains /, split by it- if (text.includes("/")) text = text.split("/")[0];-- // if contains ., split by it- if (text.includes(".")) {- const doms = text.split(".");- //up to last two- return doms[doms.length - 2] + "." + doms[doms.length - 1];- } else {- return text;- }- });- }-- replaceGuids(text) {- const guidRegex =- /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;- return text.replace(guidRegex, "");- }-- replaceNonText(text) {- const nonTextRegex = /\w[^\w\s]*\w(?=\s|$)|[^\w\s]+/g;- text = text.replace(nonTextRegex, (match) => {- return ``;- });- const longStringRegex = /\S{25,}/g;- text = text.replace(longStringRegex, (match) => {- return ``;- });- return text;- async speak(text) {- return speechStore.speak(text);-- }---- stop() {- if (this.isSpeaking()) {- this.synth.cancel();- }- speechStore.stop();-- }---- isSpeaking() {- return this.synth?.speaking || false;- return speechStore.isSpeaking;-- }--}----export const speech = new Speech();--window.speech = speech;-- // Add event listener for settings changes- document.addEventListener("settings-updated", loadMicSettings);