Skip to content

The open-source AI voice studio. Clone, dictate, create.

Keep Models page on local Qwen, mark remote endpoint ready, surface endpoint failures clearly

jamiepine6d6f6b3455ad
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 221d6fdf..62a149b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@
## [Unreleased]
+### Dictation and agents
+
+- **Bring your own LLM for refinement and personalities.** Settings → Captures → Refinement gains a *Custom LLM endpoint (advanced)* section: point it at any server that speaks OpenAI's `/v1/chat/completions` (llama.cpp, vLLM, LM Studio, LocalAI, Ollama's OpenAI shim, OpenRouter, or OpenAI itself) with a model name and an optional API key, and dictation refinement and personality compose/rewrite go there instead of the bundled Qwen3. Leave the endpoint blank and nothing changes. The key is write-only: the settings API reports only whether one is stored. The dictation readiness checklist treats a configured endpoint as ready, the Models page keeps managing the on-device Qwen regardless, and a failing endpoint surfaces as a clear error naming the URL and status. — @ext-sakamoro ([#947](https://github.com/jamiepine/voicebox/pull/947))
+
## [0.6.0] - 2026-10-04
**The stability release.** 0.5 made Voicebox a dictation and agent-voice studio; 0.6 makes it hold up. Sixty-plus contributor pull requests landed since 0.5 and most of them fix something that used to go wrong in daily use: MLX crashed with `There is no Stream(gpu, 1) in current thread` when a model was loaded on one thread and used on another, memory grew with every generation and never came back on unload, a fully-cached model retried HuggingFace five times per file before it would load offline, the Docker image could not write its own data volume, dictation refused to arm without an LLM it was never going to call, and the backend test suite would not even collect. All of that is fixed. The headline features ride on top: native AMD ROCm on Windows and in Docker, five new interface languages, and Chatterbox multilingual running on Apple Silicon through MLX.
diff --git a/backend/backends/__init__.py b/backend/backends/__init__.py
index 833bdc23..b92c5554 100644
--- a/backend/backends/__init__.py
+++ b/backend/backends/__init__.py
@@ -603,7 +603,7 @@ async def ensure_model_cached_or_raise(engine: str, model_size: str = "default")
def unload_model_by_config(config: ModelConfig) -> bool:
"""Unload a model given its config. Returns True if it was loaded, False otherwise."""
from . import get_tts_backend_for_engine
- from ..services import tts, transcribe, llm as llm_service
+ from ..services import tts, transcribe
from ..utils.cache import clear_voice_prompt_memory_cache
if config.engine == "whisper":
@@ -614,7 +614,11 @@ def unload_model_by_config(config: ModelConfig) -> bool:
return False
if config.engine == "qwen_llm":
- backend = llm_service.get_llm_model()
+ # Always address the local Qwen backend here: when a custom
+ # OpenAI-compatible endpoint is active, ``get_llm_model()`` returns
+ # the remote backend, and the Models page must still manage the
+ # on-device model.
+ backend = get_llm_backend_for_engine("qwen_llm")
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
if backend.is_loaded() and loaded_size == config.model_size:
backend.unload_model()
@@ -650,7 +654,7 @@ def unload_model_by_config(config: ModelConfig) -> bool:
def check_model_loaded(config: ModelConfig) -> bool:
"""Check if a model is currently loaded."""
from . import get_tts_backend_for_engine
- from ..services import tts, transcribe, llm as llm_service
+ from ..services import tts, transcribe
try:
if config.engine == "whisper":
@@ -658,7 +662,7 @@ def check_model_loaded(config: ModelConfig) -> bool:
return whisper_model.is_loaded() and getattr(whisper_model, "model_size", None) == config.model_size
if config.engine == "qwen_llm":
- backend = llm_service.get_llm_model()
+ backend = get_llm_backend_for_engine("qwen_llm")
loaded_size = getattr(backend, "_current_model_size", None) or getattr(backend, "model_size", None)
return backend.is_loaded() and loaded_size == config.model_size
@@ -681,7 +685,7 @@ def check_model_loaded(config: ModelConfig) -> bool:
def get_model_load_func(config: ModelConfig):
"""Return a callable that loads/downloads the model."""
from . import get_tts_backend_for_engine
- from ..services import tts, transcribe, llm as llm_service
+ from ..services import tts, transcribe
if config.engine == "whisper":
return lambda: transcribe.get_whisper_model().load_model(config.model_size)
@@ -693,7 +697,7 @@ def get_model_load_func(config: ModelConfig):
return lambda: get_tts_backend_for_engine(config.engine).load_model(config.model_size)
if config.engine == "qwen_llm":
- return lambda: llm_service.get_llm_model().load_model(config.model_size)
+ return lambda: get_llm_backend_for_engine("qwen_llm").load_model(config.model_size)
return lambda: get_tts_backend_for_engine(config.engine).load_model()
diff --git a/backend/backends/openai_compat_backend.py b/backend/backends/openai_compat_backend.py
index b60cd457..ce52891f 100644
--- a/backend/backends/openai_compat_backend.py
+++ b/backend/backends/openai_compat_backend.py
@@ -97,9 +97,11 @@ class OpenAICompatLLMBackend:
remote model selection is pinned at construction time.
Raises:
- httpx.HTTPStatusError: The remote returned a non-2xx status.
- httpx.RequestError: Transport failure (timeout, DNS, TLS).
- ValueError: The response was well-formed HTTP but carried no
+ RuntimeError: The remote returned a non-2xx status, could not be
+ reached (timeout, DNS, TLS), or sent a non-JSON body. The
+ message names the endpoint so the settings UI and the
+ dictation pill show something actionable.
+ ValueError: The response was well-formed JSON but carried no
content in either the chat or legacy completion shapes.
"""
messages = _build_messages(prompt, system, examples)
@@ -129,12 +131,22 @@ class OpenAICompatLLMBackend:
exc.response.status_code,
body_preview,
)
- raise
+ raise RuntimeError(
+ f"Custom LLM endpoint {url} returned HTTP {exc.response.status_code}: "
+ f"{body_preview[:200] or exc.response.reason_phrase}"
+ ) from exc
except httpx.RequestError as exc:
logger.error("OpenAI-compat endpoint %s request failed: %s", url, exc)
- raise
+ raise RuntimeError(
+ f"Custom LLM endpoint {url} is unreachable: {exc.__class__.__name__}: {exc}"
+ ) from exc
- data = response.json()
+ try:
+ data = response.json()
+ except ValueError as exc:
+ raise RuntimeError(
+ f"Custom LLM endpoint {url} returned a non-JSON body: {response.text[:200]}"
+ ) from exc
return _extract_content(data, url)
diff --git a/backend/routes/captures.py b/backend/routes/captures.py
index 40a5adc1..7761f955 100644
--- a/backend/routes/captures.py
+++ b/backend/routes/captures.py
@@ -190,13 +190,31 @@ async def capture_readiness_endpoint(db: Session = Depends(get_db)):
size=stt_cfg.model_size,
size_mb=stt_cfg.size_mb or None,
),
- llm=models.ModelReadiness(
- ready=is_model_cached(llm_cfg.hf_repo_id),
+ llm=_llm_readiness(saved, llm_cfg),
+ )
+
+
+def _llm_readiness(saved, llm_cfg) -> models.ModelReadiness:
+ """LLM row of the readiness checklist.
+
+ A configured custom OpenAI-compatible endpoint owns the model, so there
+ is nothing to download locally and the gate is green regardless of the
+ Qwen cache; the row names the remote model instead.
+ """
+ if saved.custom_llm_endpoint and saved.custom_llm_model:
+ return models.ModelReadiness(
+ ready=True,
model_name=llm_cfg.model_name,
- display_name=llm_cfg.display_name,
- size=llm_cfg.model_size,
- size_mb=llm_cfg.size_mb or None,
- ),
+ display_name=saved.custom_llm_model,
+ size=saved.custom_llm_model,
+ size_mb=None,
+ )
+ return models.ModelReadiness(
+ ready=is_model_cached(llm_cfg.hf_repo_id),
+ model_name=llm_cfg.model_name,
+ display_name=llm_cfg.display_name,
+ size=llm_cfg.model_size,
+ size_mb=llm_cfg.size_mb or None,
)
diff --git a/backend/routes/llm.py b/backend/routes/llm.py
index 49c4bd88..785200a6 100644
--- a/backend/routes/llm.py
+++ b/backend/routes/llm.py
@@ -87,6 +87,15 @@ async def llm_generate(request: models.LLMGenerateRequest):
examples=examples,
)
return models.LLMGenerateResponse(text=text, model_size=model_size)
+ except RuntimeError as e:
+ if isinstance(backend, OpenAICompatLLMBackend):
+ # The remote backend raises a deliberately short message naming
+ # the endpoint and status (no local paths), so the UI can show
+ # the user what to fix.
+ logger.error("Custom LLM endpoint failed: %s", e)
+ raise HTTPException(status_code=502, detail=str(e)) from e
+ logger.exception("LLM generate failed")
+ raise HTTPException(status_code=500, detail="LLM generation failed") from e
except Exception as e:
# The backend exception text can include filesystem paths and stack
# frames — log it server-side and hand the client a generic message.
diff --git a/backend/tests/test_openai_compat_backend.py b/backend/tests/test_openai_compat_backend.py
new file mode 100644
index 00000000..225681dc
--- /dev/null
+++ b/backend/tests/test_openai_compat_backend.py
@@ -0,0 +1,320 @@
+"""
+Tests for the custom OpenAI-compatible LLM endpoint against a fake server.
+
+Unlike ``test_openai_compat_integration.py`` (which needs a real model
+server and skips otherwise), these boot a tiny in-process HTTP server that
+speaks just enough of ``/v1/chat/completions`` to check the request shape
+Voicebox sends, the response shapes it accepts, how failures surface, and
+how the settings layer switches dispatch between the remote backend and
+the built-in Qwen path.
+"""
+
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import pytest
+
+from backend import models
+from backend.backends import (
+ _OPENAI_COMPAT_CACHE_KEY,
+ _llm_backends,
+ get_llm_backend,
+ reset_backends,
+ set_llm_config,
+)
+from backend.backends.openai_compat_backend import OpenAICompatLLMBackend
+
+
+class FakeOpenAIServer:
+ """Minimal OpenAI-compatible chat server that records every request.
+
+ ``script`` is a callable ``(path, headers, body) -> (status, payload)``;
+ ``payload`` is JSON-encoded unless it is a ``bytes`` object, which is
+ sent verbatim (to simulate non-JSON bodies).
+ """
+
+ def __init__(self, script):
+ self.script = script
+ self.requests: list[dict] = []
+ server = self
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ length = int(self.headers.get("Content-Length") or 0)
+ raw = self.rfile.read(length)
+ body = json.loads(raw) if raw else None
+ server.requests.append(
+ {"path": self.path, "headers": dict(self.headers), "body": body}
+ )
+ status, payload = server.script(self.path, self.headers, body)
+ data = payload if isinstance(payload, bytes) else json.dumps(payload).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(data)))
+ self.end_headers()
+ self.wfile.write(data)
+
+ def log_message(self, *args):
+ pass
+
+ self.httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ self.thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+
+ @property
+ def base_url(self) -> str:
+ host, port = self.httpd.server_address[:2]
+ return f"http://{host}:{port}/v1"
+
+ def __enter__(self):
+ self.thread.start()
+ return self
+
+ def __exit__(self, *exc):
+ self.httpd.shutdown()
+ self.httpd.server_close()
+
+
+def _chat_reply(text: str):
+ return 200, {"choices": [{"message": {"role": "assistant", "content": text}}]}
+
+
[email protected](autouse=True)
+def _clean_backends():
+ reset_backends()
+ yield
+ reset_backends()
+
+
+async def test_generate_sends_openai_chat_shape_with_bearer_and_examples():
+ with FakeOpenAIServer(lambda *_: _chat_reply(" Refined text. ")) as server:
+ backend = OpenAICompatLLMBackend(
+ endpoint=server.base_url + "/",
+ model="my-model",
+ api_key="sk-test",
+ )
+ out = await backend.generate(
+ prompt="um hello there",
+ system="You clean transcripts.",
+ max_tokens=64,
+ temperature=0.2,
+ model_size="0.6B",
+ examples=[("um hi", "Hi.")],
+ )
+
+ assert out == "Refined text."
+ (req,) = server.requests
+ assert req["path"] == "/v1/chat/completions"
+ assert req["headers"]["Authorization"] == "Bearer sk-test"
+ assert req["body"]["model"] == "my-model"
+ assert req["body"]["stream"] is False
+ assert req["body"]["max_tokens"] == 64
+ assert req["body"]["temperature"] == 0.2
+ assert req["body"]["messages"] == [
+ {"role": "system", "content": "You clean transcripts."},
+ {"role": "user", "content": "um hi"},
+ {"role": "assistant", "content": "Hi."},
+ {"role": "user", "content": "um hello there"},
+ ]
+
+
+async def test_generate_without_api_key_sends_no_authorization_header():
+ with FakeOpenAIServer(lambda *_: _chat_reply("ok")) as server:
+ backend = OpenAICompatLLMBackend(endpoint=server.base_url, model="m", api_key="")
+ assert await backend.generate(prompt="x") == "ok"
+ assert "Authorization" not in server.requests[0]["headers"]
+
+
+async def test_generate_accepts_legacy_text_completion_shape():
+ with FakeOpenAIServer(lambda *_: (200, {"choices": [{"text": " legacy "}]})) as server:
+ backend = OpenAICompatLLMBackend(endpoint=server.base_url, model="m")
+ assert await backend.generate(prompt="x") == "legacy"
+
+
+async def test_http_error_surfaces_as_runtime_error_naming_the_endpoint():
+ with FakeOpenAIServer(lambda *_: (401, {"error": {"message": "bad key"}})) as server:
+ backend = OpenAICompatLLMBackend(endpoint=server.base_url, model="m", api_key="nope")
+ with pytest.raises(RuntimeError) as excinfo:
+ await backend.generate(prompt="x")
+ message = str(excinfo.value)
+ assert "HTTP 401" in message
+ assert server.base_url in message
+ assert "bad key" in message
+
+
+async def test_unreachable_endpoint_surfaces_as_runtime_error():
+ with FakeOpenAIServer(lambda *_: _chat_reply("ok")) as server:
+ dead_url = server.base_url
+ backend = OpenAICompatLLMBackend(endpoint=dead_url, model="m", timeout=2.0)
+ with pytest.raises(RuntimeError) as excinfo:
+ await backend.generate(prompt="x")
+ assert "unreachable" in str(excinfo.value)
+
+
+async def test_non_json_body_surfaces_as_runtime_error():
+ with FakeOpenAIServer(lambda *_: (200, b"<html>proxy page</html>")) as server:
+ backend = OpenAICompatLLMBackend(endpoint=server.base_url, model="m")
+ with pytest.raises(RuntimeError) as excinfo:
+ await backend.generate(prompt="x")
+ assert "non-JSON" in str(excinfo.value)
+
+
+async def test_empty_choices_raises_value_error():
+ with FakeOpenAIServer(lambda *_: (200, {"choices": []})) as server:
+ backend = OpenAICompatLLMBackend(endpoint=server.base_url, model="m")
+ with pytest.raises(ValueError, match="No choices"):
+ await backend.generate(prompt="x")
+
+
+def test_dispatch_switches_between_remote_and_builtin():
+ set_llm_config("http://localhost:1/v1", "m", None)
+ remote = get_llm_backend()
+ assert isinstance(remote, OpenAICompatLLMBackend)
+ assert get_llm_backend() is remote, "same config must reuse the cached instance"
+
+ set_llm_config("http://localhost:2/v1", "m", "key")
+ rotated = get_llm_backend()
+ assert rotated is not remote
+ assert rotated.endpoint == "http://localhost:2/v1"
+ assert rotated.api_key == "key"
+
+ set_llm_config("", "", "")
+ assert _OPENAI_COMPAT_CACHE_KEY not in _llm_backends
+ assert not isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+
+
+def test_dispatch_requires_both_endpoint_and_model():
+ set_llm_config("http://localhost:1/v1", None, None)
+ assert not isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+ set_llm_config(None, "m", None)
+ assert not isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+
+
+def test_capture_settings_response_never_echoes_the_api_key():
+ stored = models.CaptureSettingsResponse(
+ custom_llm_endpoint="http://localhost:1/v1",
+ custom_llm_model="m",
+ custom_llm_api_key="sk-secret",
+ )
+ assert stored.custom_llm_api_key_configured is True
+ assert "sk-secret" not in stored.model_dump_json()
+
+ empty = models.CaptureSettingsResponse(custom_llm_api_key="")
+ assert empty.custom_llm_api_key_configured is False
+
+
+def test_settings_write_syncs_dispatch_and_readiness(tmp_path):
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+
+ from backend.database.models import Base
+ from backend.services import settings as settings_service
+
+ engine = create_engine(f"sqlite:///{tmp_path / 'settings.db'}")
+ Base.metadata.create_all(engine)
+ db = sessionmaker(bind=engine)()
+ try:
+ row = settings_service.update_capture_settings(
+ db,
+ {"custom_llm_endpoint": "http://localhost:1/v1", "custom_llm_model": "m", "custom_llm_api_key": "k"},
+ )
+ assert isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+
+ from backend.backends import get_llm_model_configs
+ from backend.routes.captures import _llm_readiness
+
+ llm_cfg = next(c for c in get_llm_model_configs() if c.model_size == row.llm_model)
+ readiness = _llm_readiness(row, llm_cfg)
+ assert readiness.ready is True
+ assert readiness.display_name == "m"
+
+ reset_backends()
+ settings_service.bootstrap_llm_backend_config(db)
+ assert isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+
+ row = settings_service.update_capture_settings(db, {"custom_llm_endpoint": None})
+ assert not isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+ assert _llm_readiness(row, llm_cfg).display_name == llm_cfg.display_name
+ finally:
+ db.close()
+
+
+def test_routes_round_trip_through_fake_server(tmp_path):
+ """PUT /settings/captures → GET masks the key → /llm/generate and
+ /capture/readiness honour the remote endpoint → clearing it reverts."""
+ from fastapi import FastAPI
+ from sqlalchemy import create_engine
+ from sqlalchemy.orm import sessionmaker
+ from starlette.testclient import TestClient
+
+ from backend.database import get_db
+ from backend.database.models import Base
+ from backend.routes import captures as captures_routes, llm as llm_routes, settings as settings_routes
+
+ engine = create_engine(
+ f"sqlite:///{tmp_path / 'routes.db'}", connect_args={"check_same_thread": False}
+ )
+ Base.metadata.create_all(engine)
+ session_factory = sessionmaker(bind=engine)
+
+ def _db():
+ db = session_factory()
+ try:
+ yield db
+ finally:
+ db.close()
+
+ app = FastAPI()
+ app.include_router(settings_routes.router)
+ app.include_router(llm_routes.router)
+ app.include_router(captures_routes.router)
+ app.dependency_overrides[get_db] = _db
+ client = TestClient(app)
+
+ with FakeOpenAIServer(lambda *_: _chat_reply("Hello from remote.")) as server:
+ res = client.put(
+ "/settings/captures",
+ json={
+ "custom_llm_endpoint": server.base_url,
+ "custom_llm_model": "remote-model",
+ "custom_llm_api_key": "sk-secret",
+ },
+ )
+ assert res.status_code == 200, res.text
+ body = res.json()
+ assert body["custom_llm_endpoint"] == server.base_url
+ assert body["custom_llm_api_key_configured"] is True
+ assert "sk-secret" not in res.text
+
+ assert "sk-secret" not in client.get("/settings/captures").text
+
+ readiness = client.get("/capture/readiness").json()
+ assert readiness["llm"]["ready"] is True
+ assert readiness["llm"]["display_name"] == "remote-model"
+
+ res = client.post("/llm/generate", json={"prompt": "um hi"})
+ assert res.status_code == 200, res.text
+ assert res.json() == {"text": "Hello from remote.", "model_size": "remote-model"}
+ assert server.requests[-1]["headers"]["Authorization"] == "Bearer sk-secret"
+
+ res = client.put("/settings/captures", json={"custom_llm_api_key": None})
+ assert res.json()["custom_llm_api_key_configured"] is False
+ client.post("/llm/generate", json={"prompt": "again"})
+ assert "Authorization" not in server.requests[-1]["headers"]
+
+ # Server is gone: the failure is a clean 502 naming the endpoint, not a hang.
+ res = client.post("/llm/generate", json={"prompt": "um hi"})
+ assert res.status_code == 502
+ assert "unreachable" in res.text
+
+ client.put("/settings/captures", json={"custom_llm_endpoint": None, "custom_llm_model": None})
+ assert not isinstance(get_llm_backend(), OpenAICompatLLMBackend)
+ readiness = client.get("/capture/readiness").json()
+ assert readiness["llm"]["display_name"] != "remote-model"