Agent Zero AI framework
whisper remote preload
frdel357909c16a66
diff --git a/preload.py b/preload.pyindex 10aeecead..d1556755d 100644--- a/preload.py+++ b/preload.py@@ -1,7 +1,18 @@-from python.helpers import runtime, whisper+import asyncio+from python.helpers import runtime, whisper, settingsprint("Running preload...")runtime.initialize()++async def preload():+ set = settings.get_settings()++ # async tasks to preload+ tasks = [whisper.preload(set["stt_model_size"])]++ return asyncio.gather(*tasks, return_exceptions=True)++# preload transcription model-whisper.preload()\ No newline at end of file+asyncio.run(preload())diff --git a/python/api/transcribe.py b/python/api/transcribe.pyindex 4c2ae3d5b..34ccb9a9e 100644--- a/python/api/transcribe.py+++ b/python/api/transcribe.py@@ -1,10 +1,17 @@from python.helpers.api import ApiHandlerfrom flask import Request, Response-from python.helpers import runtime, whisper+from python.helpers import runtime, settings, whisperclass Transcribe(ApiHandler):async def process(self, input: dict, request: Request) -> dict | Response:audio = input.get("audio")- result = await whisper.transcribe(audio) # type: ignore+ ctxid = input.get("ctxid", "")++ context = self.get_context(ctxid)+ if await whisper.is_downloading():+ context.log.log(type="info", content="Whisper model is currently being downloaded, please wait...")++ set = settings.get_settings()+ result = await whisper.transcribe(set["stt_model_size"], audio) # type: ignorereturn resultdiff --git a/python/helpers/settings.py b/python/helpers/settings.pyindex 16a703786..f5376537a 100644--- a/python/helpers/settings.py+++ b/python/helpers/settings.py@@ -1,3 +1,4 @@+import asyncioimport jsonimport osimport re@@ -5,12 +6,13 @@ import subprocessfrom typing import Any, Literal, TypedDictimport models-from python.helpers import runtime, whisper+from python.helpers import runtime, whisper, deferfrom . import files, dotenvfrom models import get_model, ModelProvider, ModelTypefrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.embeddings import Embeddings+class Settings(TypedDict):chat_model_provider: strchat_model_name: str@@ -743,8 +745,8 @@ def _apply_settings():agent.config = ctx.configagent = agent.get_data(agent.DATA_NAME_SUBORDINATE)- # reload whisper model if necessary- whisper.preload()+ # reload whisper model if necessary+ task = defer.DeferredTask(whisper.preload, _settings["stt_model_size"])def _env_to_dict(data: str):@@ -776,7 +778,6 @@ def set_root_password(password: str):raise Exception("root password can only be set in dockerized environments")subprocess.run(f"echo 'root:{password}' | chpasswd", shell=True, check=True)dotenv.save_dotenv_value(dotenv.KEY_ROOT_PASSWORD, password)-def get_runtime_config(set: Settings):diff --git a/python/helpers/whisper.py b/python/helpers/whisper.pyindex 97bb89b99..acfb819db 100644--- a/python/helpers/whisper.py+++ b/python/helpers/whisper.py@@ -1,39 +1,59 @@-# Import the necessary librariesimport base64import warningsimport whisperimport tempfile+import asynciofrom python.helpers import runtime, rfc, settings-# suppress FutureWarning from torch.load-warnings.filterwarnings('ignore', category=FutureWarning)+# Suppress FutureWarning from torch.load+warnings.filterwarnings("ignore", category=FutureWarning)-model = None-model_name = ""+_model = None+_model_name = ""+is_updating_model = False # Tracks whether the model is currently updating-def preload():- global model, model_name- set = settings.get_settings()- if not model or model_name != set["stt_model_size"]:- model = whisper.load_model(set["stt_model_size"])- model_name = set["stt_model_size"]- return model+async def preload(model_name:str):+ try:+ return await runtime.call_development_function(_preload, model_name)+ except Exception as e:+ if not runtime.is_development():+ raise e++async def _preload(model_name:str):+ global _model, _model_name, is_updating_model-async def transcribe(audio_bytes_b64: str):- return await runtime.call_development_function(_transcribe, audio_bytes_b64)+ while is_updating_model:+ await asyncio.sleep(0.1)-def _transcribe(audio_bytes_b64: str):- global model- if model is None:- model = preload()+ try:+ is_updating_model = True+ if not _model or _model_name != model_name:+ print(f"Loading Whisper model: {model_name}")+ _model = whisper.load_model(model_name)+ _model_name = model_name+ finally:+ is_updating_model = False+async def is_downloading():+ return await runtime.call_development_function(_is_downloading)++def _is_downloading():+ return is_updating_model++async def transcribe(model_name:str, audio_bytes_b64: str):+ return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)+++async def _transcribe(model_name:str, audio_bytes_b64: str):+ await _preload(model_name)+# Decode audio bytes if encoded as a base64 stringaudio_bytes = base64.b64decode(audio_bytes_b64)- #create temp audio file+ # Create temp audio filewith tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:audio_file.write(audio_bytes)# Transcribe the audio file- result = model.transcribe(audio_file.name, fp16=False )- return result\ No newline at end of file+ result = _model.transcribe(audio_file.name, fp16=False) # type: ignore+ return result