Agent Zero AI framework
native models, log trunc fix, speech skip
frdeldcae2b74ac8e
diff --git a/preload.py b/preload.pyindex 2494a7091..4da2a05dc 100644--- a/preload.py+++ b/preload.py@@ -40,7 +40,11 @@ async def preload():PrintStyle().error(f"Error in preload_kokoro: {e}")# async tasks to preload- tasks = [preload_whisper(), preload_embedding(), preload_kokoro()]+ tasks = [+ preload_embedding(),+ # preload_whisper(),+ # preload_kokoro()+ ] # no longer preload kokoro and whisper, do it JITawait asyncio.gather(*tasks, return_exceptions=True)PrintStyle().print("Preload completed")diff --git a/python/api/synthesize.py b/python/api/synthesize.pyindex 106800b75..0a91833a0 100644--- a/python/api/synthesize.py+++ b/python/api/synthesize.py@@ -12,7 +12,7 @@ class Synthesize(ApiHandler):ctxid = input.get("ctxid", "")context = self.get_context(ctxid)- if await kokoro_tts.is_downloading():+ if not await kokoro_tts.is_downloaded():context.log.log(type="info", content="Kokoro TTS model is currently being downloaded, please wait...")try:diff --git a/python/api/transcribe.py b/python/api/transcribe.pyindex 34ccb9a9e..f81e6aa3d 100644--- a/python/api/transcribe.py+++ b/python/api/transcribe.py@@ -9,8 +9,8 @@ class Transcribe(ApiHandler):ctxid = input.get("ctxid", "")context = self.get_context(ctxid)- if await whisper.is_downloading():- context.log.log(type="info", content="Whisper model is currently being downloaded, please wait...")+ if not await whisper.is_downloaded():+ context.log.log(type="info", content="Whisper STT model is currently being downloaded, please wait...")set = settings.get_settings()result = await whisper.transcribe(set["stt_model_size"], audio) # type: ignorediff --git a/python/helpers/kokoro_tts.py b/python/helpers/kokoro_tts.pyindex 20f6036f0..17a57c385 100644--- a/python/helpers/kokoro_tts.py+++ b/python/helpers/kokoro_tts.py@@ -15,52 +15,74 @@ _voice = "am_puck,am_onyx"_speed = 1.1is_updating_model = False+async def preload():try:- return await runtime.call_development_function(_preload)+ # return await runtime.call_development_function(_preload)+ return await _preload()except Exception as e:- if not runtime.is_development():- raise e+ # if not runtime.is_development():+ raise e# Fallback to direct execution if RFC fails in development- PrintStyle.standard("RFC failed, falling back to direct execution...")- return await _preload()+ # PrintStyle.standard("RFC failed, falling back to direct execution...")+ # return await _preload()+async def _preload():global _pipeline, is_updating_model-+while is_updating_model:await asyncio.sleep(0.1)-+try:is_updating_model = Trueif not _pipeline:PrintStyle.standard("Loading Kokoro TTS model...")from kokoro import KPipeline- _pipeline = KPipeline(lang_code='a')+ _pipeline = KPipeline(lang_code="a")finally:is_updating_model = False+async def is_downloading():try:- return await runtime.call_development_function(_is_downloading)+ # return await runtime.call_development_function(_is_downloading)+ return _is_downloading()except Exception as e:- if not runtime.is_development():- raise e+ # if not runtime.is_development():+ raise e# Fallback to direct execution if RFC fails in development- return _is_downloading()+ # return _is_downloading()+def _is_downloading():return is_updating_model+async def is_downloaded():+ try:+ # return await runtime.call_development_function(_is_downloaded)+ return _is_downloaded()+ except Exception as e:+ # if not runtime.is_development():+ raise e+ # Fallback to direct execution if RFC fails in development+ # return _is_downloaded()++def _is_downloaded():+ return _pipeline is not None++async def synthesize_sentences(sentences: list[str]):"""Generate audio for multiple sentences and return concatenated base64 audio"""try:- return await runtime.call_development_function(_synthesize_sentences, sentences)+ # return await runtime.call_development_function(_synthesize_sentences, sentences)+ return await _synthesize_sentences(sentences)except Exception as e:- if not runtime.is_development():- raise e+ # if not runtime.is_development():+ raise e# Fallback to direct execution if RFC fails in development- return await _synthesize_sentences(sentences)+ # return await _synthesize_sentences(sentences)+async def _synthesize_sentences(sentences: list[str]):await _preload()@@ -70,22 +92,22 @@ async def _synthesize_sentences(sentences: list[str]):try:for sentence in sentences:if sentence.strip():- segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed)+ segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed) # type: ignoresegment_list = list(segments)-+for segment in segment_list:audio_tensor = segment.audio- audio_numpy = audio_tensor.detach().cpu().numpy()+ audio_numpy = audio_tensor.detach().cpu().numpy() # type: ignorecombined_audio.extend(audio_numpy)# Convert combined audio to bytesbuffer = io.BytesIO()- sf.write(buffer, combined_audio, 24000, format='WAV')+ sf.write(buffer, combined_audio, 24000, format="WAV")audio_bytes = buffer.getvalue()# Return base64 encoded audio- return base64.b64encode(audio_bytes).decode('utf-8')+ return base64.b64encode(audio_bytes).decode("utf-8")except Exception as e:PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}")- raise\ No newline at end of file+ raise\ No newline at end of filediff --git a/python/helpers/log.py b/python/helpers/log.pyindex 075661c40..e66b7107d 100644--- a/python/helpers/log.py+++ b/python/helpers/log.py@@ -4,6 +4,7 @@ from typing import Any, Literal, Optional, Dictimport uuidfrom collections import OrderedDict # Import OrderedDictfrom python.helpers.strings import truncate_text_by_ratio+import copyType = Literal["agent",@@ -25,9 +26,9 @@ ProgressUpdate = Literal["persistent", "temporary", "none"]HEADING_MAX_LEN: int = 120-CONTENT_MAX_LEN: int = 4000+CONTENT_MAX_LEN: int = 10000KEY_MAX_LEN: int = 60-VALUE_MAX_LEN: int = 1000+VALUE_MAX_LEN: int = 3000PROGRESS_MAX_LEN: int = 120@@ -44,29 +45,36 @@ def _truncate_progress(text: str | None) -> str:def _truncate_key(text: str) -> str:return truncate_text_by_ratio(str(text), KEY_MAX_LEN, "...", ratio=1.0)-def _truncate_value(text: Any) -> Any:+def _truncate_value(val: Any) -> Any:+ # If dict, recursively truncate each value+ if isinstance(val, dict):+ for k in list(val.keys()):+ val[k] = _truncate_value(val[k])+ return val+ # If list or tuple, recursively truncate each item+ if isinstance(val, list):+ for i in range(len(val)):+ val[i] = _truncate_value(val[i])+ return val+ if isinstance(val, tuple):+ return tuple(_truncate_value(x) for x in val)+# Convert non-str values to json for consistent length measurement- if isinstance(text, str):- raw = text+ if isinstance(val, str):+ raw = valelse:try:- raw = json.dumps(text, ensure_ascii=False)+ raw = json.dumps(val, ensure_ascii=False)except Exception:- raw = str(text)+ raw = str(val)if len(raw) <= VALUE_MAX_LEN:- return text # No truncation needed, preserve original type+ return val # No truncation needed, preserve original type- # Determine removed characters dynamically to build replacement string+ # Do a single truncation calculationremoved = len(raw) - VALUE_MAX_LEN- while True:- replacement = f"\n\n<< {removed} Characters hidden >>\n\n"- truncated = truncate_text_by_ratio(raw, VALUE_MAX_LEN, replacement, ratio=0.3)- new_removed = len(raw) - (len(truncated) - len(replacement))- if new_removed == removed:- break- removed = new_removed-+ replacement = f"\n\n<< {removed} Characters hidden >>\n\n"+ truncated = truncate_text_by_ratio(raw, VALUE_MAX_LEN, replacement, ratio=0.3)return truncateddef _truncate_content(text: str | None) -> str:@@ -177,10 +185,13 @@ class Log:# Truncate kvpsif kvps is not None:+ kvps = copy.deepcopy(kvps) # deep copy to avoid modifying the original kvpskvps = OrderedDict({_truncate_key(k): _truncate_value(v) for k, v in kvps.items()})# Apply truncation to kwargs merged into kvps later+ if kwargs is not None:+ kwargs = copy.deepcopy(kwargs) # deep copy to avoid modifying the original kwargskwargs = { _truncate_key(k): _truncate_value(v) for k, v in (kwargs or {}).items() }# Ensure kvps is OrderedDict even if None@@ -231,6 +242,7 @@ class Log:item.content = _truncate_content(content)if kvps is not None:+ kvps = copy.deepcopy(kvps) # deep copy to avoid modifying the original kvpsitem.kvps = OrderedDict({_truncate_key(k): _truncate_value(v) for k, v in kvps.items()}) # Ensure order@@ -239,11 +251,13 @@ class Log:item.temp = tempif kwargs:+ kwargs = copy.deepcopy(kwargs) # deep copy to avoid modifying the original kwargsif item.kvps is None:item.kvps = OrderedDict() # Ensure kvps is an OrderedDictfor k, v in kwargs.items():item.kvps[_truncate_key(k)] = _truncate_value(v)+self.updates += [item.no]self._update_progress_from_item(item)diff --git a/python/helpers/whisper.py b/python/helpers/whisper.pyindex 0f644487d..bbe26cf1c 100644--- a/python/helpers/whisper.py+++ b/python/helpers/whisper.py@@ -3,7 +3,7 @@ import warningsimport whisperimport tempfileimport asyncio-from python.helpers import runtime, rfc, settings+from python.helpers import runtime, rfc, settings, filesfrom python.helpers.print_style import PrintStyle# Suppress FutureWarning from torch.load@@ -15,10 +15,11 @@ is_updating_model = False # Tracks whether the model is currently updatingasync def preload(model_name:str):try:- return await runtime.call_development_function(_preload, model_name)+ # return await runtime.call_development_function(_preload, model_name)+ return await _preload(model_name)except Exception as e:- if not runtime.is_development():- raise e+ # if not runtime.is_development():+ raise easync def _preload(model_name:str):global _model, _model_name, is_updating_model@@ -30,19 +31,34 @@ async def _preload(model_name:str):is_updating_model = Trueif not _model or _model_name != model_name:PrintStyle.standard(f"Loading Whisper model: {model_name}")- _model = whisper.load_model(name=model_name) # type: ignore+ _model = whisper.load_model(name=model_name, download_root=files.get_abs_path("/tmp/models/whisper")) # type: ignore_model_name = model_namefinally:is_updating_model = Falseasync def is_downloading():- return await runtime.call_development_function(_is_downloading)+ # return await runtime.call_development_function(_is_downloading)+ return _is_downloading()def _is_downloading():return is_updating_model+async def is_downloaded():+ try:+ # return await runtime.call_development_function(_is_downloaded)+ return _is_downloaded()+ except Exception as e:+ # if not runtime.is_development():+ raise e+ # Fallback to direct execution if RFC fails in development+ # return _is_downloaded()++def _is_downloaded():+ return _model is not None+async def transcribe(model_name:str, audio_bytes_b64: str):- return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)+ # return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)+ return await _transcribe(model_name, audio_bytes_b64)async def _transcribe(model_name:str, audio_bytes_b64: str):diff --git a/webui/index.js b/webui/index.jsindex 20fe5e8ac..9195a2ccd 100644--- a/webui/index.js+++ b/webui/index.js@@ -24,6 +24,7 @@ const timeDate = document.getElementById("time-date-container");let autoScroll = true;let context = "";let resetCounter = 0;+let skipOneSpeech = false;let connectionStatus = false;// Initialize the toggle button@@ -503,6 +504,10 @@ function afterMessagesUpdate(logs) {}function speakMessages(logs) {+ if (skipOneSpeech) {+ skipOneSpeech = false;+ return;+ }// log.no, log.type, log.heading, log.contentfor (let i = logs.length - 1; i >= 0; i--) {const log = logs[i];@@ -512,10 +517,14 @@ function speakMessages(logs) {// finished responseif (log.type == "response") {- // lastSpokenNo = log.no;- speechStore.speakStream(getChatBasedId(log.no), log.content, log.kvps?.finished);- return;-+ // lastSpokenNo = log.no;+ speechStore.speakStream(+ getChatBasedId(log.no),+ log.content,+ log.kvps?.finished+ );+ return;+// finished LLM headline, not response} else if (log.type == "agent" &&@@ -524,9 +533,9 @@ function speakMessages(logs) {log.kvps.tool_args &&log.kvps.tool_name != "response") {- // lastSpokenNo = log.no;- speechStore.speakStream(getChatBasedId(log.no), log.kvps.headline, true);- return;+ // lastSpokenNo = log.no;+ speechStore.speakStream(getChatBasedId(log.no), log.kvps.headline, true);+ return;}}}@@ -736,6 +745,9 @@ export const setContext = function (id) {if (tasksAD) tasksAD.selected = id;}}++ //skip one speech if enabled when switching context+ if (localStorage.getItem("speech") == "true") skipOneSpeech = true;};export const getContext = function () {@@ -743,7 +755,7 @@ export const getContext = function () {};export const getChatBasedId = function (id) {- return context+"-"+resetCounter+"-"+id;+ return context + "-" + resetCounter + "-" + id;};window.toggleAutoScroll = async function (_autoScroll) {