Skip to content

Agent Zero AI framework

whisper remote preload

frdel357909c16a66
diff --git a/preload.py b/preload.py
index 10aeecead..d1556755d 100644
--- a/preload.py
+++ b/preload.py
@@ -1,7 +1,18 @@
-from python.helpers import runtime, whisper
+import asyncio
+from python.helpers import runtime, whisper, settings
print("Running preload...")
runtime.initialize()
+
+async def preload():
+ set = settings.get_settings()
+
+ # async tasks to preload
+ tasks = [whisper.preload(set["stt_model_size"])]
+
+ return asyncio.gather(*tasks, return_exceptions=True)
+
+
# preload transcription model
-whisper.preload()
\ No newline at end of file
+asyncio.run(preload())
diff --git a/python/api/transcribe.py b/python/api/transcribe.py
index 4c2ae3d5b..34ccb9a9e 100644
--- a/python/api/transcribe.py
+++ b/python/api/transcribe.py
@@ -1,10 +1,17 @@
from python.helpers.api import ApiHandler
from flask import Request, Response
-from python.helpers import runtime, whisper
+from python.helpers import runtime, settings, whisper
class Transcribe(ApiHandler):
async def process(self, input: dict, request: Request) -> dict | Response:
audio = input.get("audio")
- result = await whisper.transcribe(audio) # type: ignore
+ ctxid = input.get("ctxid", "")
+
+ context = self.get_context(ctxid)
+ if await whisper.is_downloading():
+ context.log.log(type="info", content="Whisper model is currently being downloaded, please wait...")
+
+ set = settings.get_settings()
+ result = await whisper.transcribe(set["stt_model_size"], audio) # type: ignore
return result
diff --git a/python/helpers/settings.py b/python/helpers/settings.py
index 16a703786..f5376537a 100644
--- a/python/helpers/settings.py
+++ b/python/helpers/settings.py
@@ -1,3 +1,4 @@
+import asyncio
import json
import os
import re
@@ -5,12 +6,13 @@ import subprocess
from typing import Any, Literal, TypedDict
import models
-from python.helpers import runtime, whisper
+from python.helpers import runtime, whisper, defer
from . import files, dotenv
from models import get_model, ModelProvider, ModelType
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.embeddings import Embeddings
+
class Settings(TypedDict):
chat_model_provider: str
chat_model_name: str
@@ -743,8 +745,8 @@ def _apply_settings():
agent.config = ctx.config
agent = agent.get_data(agent.DATA_NAME_SUBORDINATE)
- # reload whisper model if necessary
- whisper.preload()
+ # reload whisper model if necessary
+ task = defer.DeferredTask(whisper.preload, _settings["stt_model_size"])
def _env_to_dict(data: str):
@@ -776,7 +778,6 @@ def set_root_password(password: str):
raise Exception("root password can only be set in dockerized environments")
subprocess.run(f"echo 'root:{password}' | chpasswd", shell=True, check=True)
dotenv.save_dotenv_value(dotenv.KEY_ROOT_PASSWORD, password)
-
def get_runtime_config(set: Settings):
diff --git a/python/helpers/whisper.py b/python/helpers/whisper.py
index 97bb89b99..acfb819db 100644
--- a/python/helpers/whisper.py
+++ b/python/helpers/whisper.py
@@ -1,39 +1,59 @@
-# Import the necessary libraries
import base64
import warnings
import whisper
import tempfile
+import asyncio
from python.helpers import runtime, rfc, settings
-# suppress FutureWarning from torch.load
-warnings.filterwarnings('ignore', category=FutureWarning)
+# Suppress FutureWarning from torch.load
+warnings.filterwarnings("ignore", category=FutureWarning)
-model = None
-model_name = ""
+_model = None
+_model_name = ""
+is_updating_model = False # Tracks whether the model is currently updating
-def preload():
- global model, model_name
- set = settings.get_settings()
- if not model or model_name != set["stt_model_size"]:
- model = whisper.load_model(set["stt_model_size"])
- model_name = set["stt_model_size"]
- return model
+async def preload(model_name:str):
+ try:
+ return await runtime.call_development_function(_preload, model_name)
+ except Exception as e:
+ if not runtime.is_development():
+ raise e
+
+async def _preload(model_name:str):
+ global _model, _model_name, is_updating_model
-async def transcribe(audio_bytes_b64: str):
- return await runtime.call_development_function(_transcribe, audio_bytes_b64)
+ while is_updating_model:
+ await asyncio.sleep(0.1)
-def _transcribe(audio_bytes_b64: str):
- global model
- if model is None:
- model = preload()
+ try:
+ is_updating_model = True
+ if not _model or _model_name != model_name:
+ print(f"Loading Whisper model: {model_name}")
+ _model = whisper.load_model(model_name)
+ _model_name = model_name
+ finally:
+ is_updating_model = False
+async def is_downloading():
+ return await runtime.call_development_function(_is_downloading)
+
+def _is_downloading():
+ return is_updating_model
+
+async def transcribe(model_name:str, audio_bytes_b64: str):
+ return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)
+
+
+async def _transcribe(model_name:str, audio_bytes_b64: str):
+ await _preload(model_name)
+
# Decode audio bytes if encoded as a base64 string
audio_bytes = base64.b64decode(audio_bytes_b64)
- #create temp audio file
+ # Create temp audio file
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:
audio_file.write(audio_bytes)
# Transcribe the audio file
- result = model.transcribe(audio_file.name, fp16=False )
- return result
\ No newline at end of file
+ result = _model.transcribe(audio_file.name, fp16=False) # type: ignore
+ return result