VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.
Python aiaudiobookcudadubbingelevenlabs-alternativehuggingfacelocal-firstmlxomnivoice-studiospeech-to-texttext-to-speechtranscriptiontranslatettsvoice-aivoice-cloningvoice-generationvoicestudioworkflow
fix(backend): keep health probes responsive when the log reader is slow
velixiode2513d6b43a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex abbe0d4ad..58f6760b7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,10 @@ metadata and the backend fallback mirror it.- Preview iOS and Android companion apps: drive your own VoiceStudio from your phone (speak, voices and cloning, dubbing, dictation) after pairing by QR code; build from source for now ([guide](docs/mobile.md)) (#2965)+### Fixed++- The backend no longer freezes ("temporarily not responding", startup never finishing) when the app reads its log slowly, for example during a model download (#PRN) — thanks @tigerscurrying-prog!+### Docs- Contributor guides and the pull request template explain how to sign the CLA before opening a first pull request (#2676)diff --git a/backend/core/loop_stall_monitor.py b/backend/core/loop_stall_monitor.pynew file mode 100644index 000000000..dd421b798--- /dev/null+++ b/backend/core/loop_stall_monitor.py@@ -0,0 +1,80 @@+"""Name the code that blocks the event loop when /health stops answering.++The desktop shell polls ``/health`` with a 1.5 s deadline. When a synchronous+call (or a GIL-hungry thread) holds the loop longer than that, the shell+reports "backend busy / not responding" while the log shows only unrelated 200s+- nothing names the culprit (#2970, #2876, #2840, #2794, #2756). A daemon+thread pings the loop and, if a ping goes unanswered for the threshold, dumps+every thread's Python stack once per stall. Pure diagnostics: never raises into+startup, and ``OMNIVOICE_LOOP_STALL_S=0`` disables it.+"""+from __future__ import annotations++import asyncio+import logging+import os+import sys+import threading+import time++logger = logging.getLogger("omnivoice.loop_stall")+_DEFAULT_THRESHOLD_S = 4.0+_POLL_S = 1.0+++def stall_threshold() -> float:+ try:+ return float(os.environ.get("OMNIVOICE_LOOP_STALL_S", _DEFAULT_THRESHOLD_S))+ except ValueError:+ return _DEFAULT_THRESHOLD_S+++def start_loop_stall_monitor(+ loop: asyncio.AbstractEventLoop | None = None,+ *,+ threshold: float | None = None,+ poll: float = _POLL_S,+ dump=None,+) -> threading.Event | None:+ """Start the monitor; returns its stop event, or None when disabled."""+ limit = stall_threshold() if threshold is None else threshold+ if limit <= 0:+ return None+ loop = loop or asyncio.get_running_loop()+ stop = threading.Event()+ last_seen = [time.monotonic()]++ def _dump_threads() -> None:+ import faulthandler++ faulthandler.dump_traceback(file=sys.stderr, all_threads=True)++ emit = dump or _dump_threads++ def _beat() -> None:+ last_seen[0] = time.monotonic()++ def _run() -> None:+ reported = False+ while not stop.wait(poll):+ try:+ loop.call_soon_threadsafe(_beat)+ except RuntimeError: # loop closed: the process is exiting+ return+ lag = time.monotonic() - last_seen[0]+ if lag < limit:+ reported = False+ elif not reported:+ reported = True+ logger.warning(+ "Event loop has not run for %.1fs; /health cannot answer. "+ "Stacks of all threads follow - the main thread's frame is "+ "the blocking call.", lag,+ )+ try:+ emit()+ except Exception: # diagnostics must never take the app down+ logger.debug("stall thread dump failed", exc_info=True)++ threading.Thread(target=_run, name="loop-stall-monitor", daemon=True).start()+ return stopdiff --git a/backend/core/nonblocking_stdio.py b/backend/core/nonblocking_stdio.pynew file mode 100644index 000000000..1375e2f1f--- /dev/null+++ b/backend/core/nonblocking_stdio.py@@ -0,0 +1,155 @@+"""Never let a slow log reader freeze the event loop.++The desktop shell starts the backend with stdout and stderr connected to pipes+it reads asynchronously. A pipe holds only a few tens of KB; once the reader+falls behind (a model download's progress bar writes hundreds of lines a+second, or the shell is busy) the next ``write`` blocks. uvicorn writes its+access log from the event loop thread, so the whole loop stops, ``/health``+misses its deadline and the shell reports "backend not responding" while the+log's last lines are ordinary 200s (#2970, #2876, #2840, #2794, #2756).++``install()`` puts a bounded in-memory buffer between producers and a pipe: a+daemon thread owns the real (possibly blocking) write, producers return at+once, and when the buffer overflows the oldest text is dropped and one notice+says how much. Terminals and regular files cannot stall like that and are left+untouched, so behaviour is identical on every platform.+"""+from __future__ import annotations++import atexit+import collections+import io+import os+import stat+import sys+import threading++DEFAULT_LIMIT_BYTES = 2 * 1024 * 1024+_EXIT_DRAIN_S = 2.0+++def _is_pipe(stream) -> bool:+ try:+ mode = os.fstat(stream.fileno()).st_mode+ except (AttributeError, OSError, ValueError):+ return False+ return stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)+++class DecoupledStream(io.TextIOBase):+ """Text stream whose ``write`` never waits on the underlying sink."""++ def __init__(self, real, *, limit_bytes: int = DEFAULT_LIMIT_BYTES) -> None:+ self._real = real+ self._limit = max(1024, int(limit_bytes))+ self._chunks: collections.deque[str] = collections.deque()+ self._size = 0+ self._dropped = 0+ self._broken = False+ self._busy = False+ self._cond = threading.Condition()+ self._thread = threading.Thread(+ target=self._pump, name="stdio-pump", daemon=True+ )+ self._thread.start()++ # -- producer side: constant time, never blocks on the sink ------------+ def write(self, text) -> int:+ if not isinstance(text, str):+ text = str(text)+ if not text:+ return 0+ with self._cond:+ if self._broken:+ return len(text)+ self._chunks.append(text)+ self._size += len(text)+ while self._size > self._limit and len(self._chunks) > 1:+ self._size -= len(self._chunks.popleft())+ self._dropped += 1+ self._cond.notify()+ return len(text)++ def flush(self) -> None:+ """No-op by design: waiting on the sink is exactly what must not happen."""++ def writable(self) -> bool:+ return True++ def isatty(self) -> bool:+ return False++ def fileno(self) -> int:+ return self._real.fileno()++ def __getattr__(self, name): # encoding, errors, buffer, reconfigure, ...+ return getattr(self._real, name)++ # -- consumer side: the only thread that may block ---------------------+ def _pump(self) -> None:+ while True:+ with self._cond:+ while not self._chunks and not self._dropped:+ self._busy = False+ self._cond.notify_all()+ self._cond.wait()+ self._busy = True+ parts = list(self._chunks)+ self._chunks.clear()+ self._size = 0+ dropped, self._dropped = self._dropped, 0+ text = "".join(parts)+ if dropped:+ text = (+ f"[log] {dropped} output chunk(s) dropped: the log reader "+ "was not keeping up\n" + text+ )+ if not self._emit(text):+ with self._cond:+ self._broken = True+ self._chunks.clear()+ self._size = 0+ self._busy = False+ self._cond.notify_all()+ return++ def _emit(self, text: str) -> bool:+ try:+ try:+ self._real.write(text)+ except UnicodeEncodeError:+ enc = getattr(self._real, "encoding", None) or "utf-8"+ self._real.write(text.encode(enc, "replace").decode(enc, "replace"))+ self._real.flush()+ return True+ except (OSError, ValueError): # reader gone or stream closed+ return False++ def drain(self, timeout: float = _EXIT_DRAIN_S) -> None:+ """Best-effort wait (bounded) for buffered text to reach the sink."""+ with self._cond:+ self._cond.wait_for(+ lambda: self._broken or (not self._chunks and not self._busy),+ timeout,+ )+++def install(*, limit_bytes: int = DEFAULT_LIMIT_BYTES) -> list[DecoupledStream]:+ """Wrap ``sys.stdout``/``sys.stderr`` when they are pipes. Idempotent."""+ if os.environ.get("OMNIVOICE_BLOCKING_STDIO", "").strip().lower() in ("1", "true", "yes", "on"):+ return []+ wrapped: list[DecoupledStream] = []+ for name in ("stdout", "stderr"):+ stream = getattr(sys, name, None)+ if stream is None or isinstance(stream, DecoupledStream) or not _is_pipe(stream):+ continue+ replacement = DecoupledStream(stream, limit_bytes=limit_bytes)+ setattr(sys, name, replacement)+ wrapped.append(replacement)+ if wrapped:+ def _drain_all() -> None:+ for s in wrapped:+ s.drain()++ atexit.register(_drain_all)+ return wrappeddiff --git a/backend/main.py b/backend/main.pyindex d17f0a667..7d3d2197e 100644--- a/backend/main.py+++ b/backend/main.py@@ -33,6 +33,13 @@ from core.contained_subprocess import secure_backend_drain_fd # noqa: E402secure_backend_drain_fd()+# The shell reads our stdout/stderr through pipes; a reader that falls behind+# would otherwise block uvicorn's access-log write on the event loop and freeze+# /health (#2970 class). Producers get a bounded buffer instead.+from core.nonblocking_stdio import install as _install_nonblocking_stdio # noqa: E402++_install_nonblocking_stdio()+import math # noqa: E402# Windows: run every child process (ffmpeg, engine sidecars, yt-dlp, demucs, …)@@ -1218,7 +1225,15 @@ async def lifespan(app: FastAPI):# StartupGate. The watchdog stays armed until the deferred task# disarms it, so a hang inside Phase A/B still gets its thread dump.app.state.startup_task = asyncio.create_task(_deferred_startup(app))+ try:+ from core.loop_stall_monitor import start_loop_stall_monitor++ _loop_stall_stop = start_loop_stall_monitor()+ except Exception:+ _loop_stall_stop = None # diagnostics-onlyyield+ if _loop_stall_stop is not None:+ _loop_stall_stop.set()# ── Graceful shutdown (SIGTERM from Tauri, Ctrl+C, etc.) ────────────# Retire the run sentinel FIRST, before any bounded wait below (#1895):# once uvicorn has begun graceful shutdown the exit is deliberate bydiff --git a/docs/install/troubleshooting.md b/docs/install/troubleshooting.mdindex a11e1cc23..b70745eab 100644--- a/docs/install/troubleshooting.md+++ b/docs/install/troubleshooting.md@@ -844,6 +844,14 @@ rather than failing with "Can't reach the local backend". Nothing to do — itclears itself as soon as the job releases the event loop. If the dot turns**red** and names an exit code, that is a real crash; use the sections above.+The backend also never waits on its own log output: stdout/stderr go through a+bounded buffer when the shell reads them through pipes, so a slow log reader (a+model download's progress bar, a busy shell) can no longer freeze the event+loop. If the buffer overflows, the log shows one `[log] N output chunk(s)+dropped` line. Set `OMNIVOICE_BLOCKING_STDIO=1` to restore plain blocking+writes. When the loop does stall, the backend log names the blocking call+(`OMNIVOICE_LOOP_STALL_S`, default 4 s, `0` disables).+**Two things changed here** ([#1190](https://github.com/debpalash/VoiceStudio/issues/1190)):- **Waiting in line is no longer counted as compute.** The generate budget useddiff --git a/tests/test_loop_stall_monitor.py b/tests/test_loop_stall_monitor.pynew file mode 100644index 000000000..f2cf17f57--- /dev/null+++ b/tests/test_loop_stall_monitor.py@@ -0,0 +1,36 @@+"""A blocked event loop must leave a thread dump naming the blocker."""+import asyncio+import time++from core.loop_stall_monitor import start_loop_stall_monitor+++def test_dumps_once_when_loop_is_blocked():+ dumps = []++ async def scenario():+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))+ await asyncio.sleep(0.2)+ time.sleep(0.9) # the synchronous call that blocks /health+ await asyncio.sleep(0.2)+ stop.set()++ asyncio.run(scenario())+ assert len(dumps) == 1+++def test_silent_when_loop_is_healthy():+ dumps = []++ async def scenario():+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))+ await asyncio.sleep(0.6)+ stop.set()++ asyncio.run(scenario())+ assert dumps == []+++def test_disabled_by_zero(monkeypatch):+ monkeypatch.setenv("OMNIVOICE_LOOP_STALL_S", "0")+ assert start_loop_stall_monitor(loop=object()) is Nonediff --git a/tests/test_nonblocking_stdio.py b/tests/test_nonblocking_stdio.pynew file mode 100644index 000000000..ac093f9b6--- /dev/null+++ b/tests/test_nonblocking_stdio.py@@ -0,0 +1,129 @@+"""A slow log reader must never block the backend (#2970, #2876, #2840, #2794, #2756).++The shell reads stdout/stderr through pipes. uvicorn writes its access log on+the event loop thread, so a full pipe froze ``/health`` while the log showed+only 200s. Measured on the real backend with an unread stderr pipe: it stopped+answering after ~1100 logged requests; with the buffer it served 6000.+"""+import subprocess+import sys+import textwrap+import threading+import time+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[1]+BACKEND = ROOT / "backend"+sys.path.insert(0, str(BACKEND))++from core.nonblocking_stdio import DecoupledStream # noqa: E402+++class _BlockedSink:+ encoding = "utf-8"++ def __init__(self):+ self.gate = threading.Event()+ self.received = []++ def write(self, text):+ self.gate.wait()+ self.received.append(text)++ def flush(self):+ pass+++def test_write_returns_immediately_while_sink_is_blocked():+ sink = _BlockedSink()+ stream = DecoupledStream(sink, limit_bytes=4096)+ start = time.monotonic()+ for i in range(20000):+ stream.write(f"line {i}\n")+ assert time.monotonic() - start < 2.0+ sink.gate.set()+ stream.drain(5)+ out = "".join(sink.received)+ assert "dropped" in out and "line 19999\n" in out # newest text survives+ assert len(out) < 2 * 4096 + 200 # one batch in flight + one buffered+++def test_text_is_delivered_in_order_when_reader_keeps_up():+ sink = _BlockedSink()+ sink.gate.set()+ stream = DecoupledStream(sink)+ for i in range(50):+ stream.write(f"{i}\n")+ stream.drain(5)+ assert "".join(sink.received) == "".join(f"{i}\n" for i in range(50))+++def test_broken_sink_never_raises_into_the_producer():+ class Gone(_BlockedSink):+ def write(self, text):+ raise BrokenPipeError++ sink = Gone()+ stream = DecoupledStream(sink)+ stream.write("x\n")+ stream.drain(2)+ assert stream.write("y\n") == 2+++_CHILD = textwrap.dedent(+ """+ import sys+ sys.path.insert(0, {backend!r})+ from core.nonblocking_stdio import install+ install()+ chunk = "x" * 1023 + "\\n"+ for _ in range(8192): # 8 MB, far beyond any pipe buffer+ sys.stderr.write(chunk)+ print("finished", flush=True)+ """+)+++def _run_child(env_extra: dict[str, str], timeout: float):+ import os++ proc = subprocess.Popen(+ [sys.executable, "-I", "-c", _CHILD.format(backend=str(BACKEND))],+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE, # never read: this is the slow log reader+ env={**os.environ, **env_extra},+ text=True,+ )+ try:+ out = proc.stdout.readline() # type: ignore[union-attr]+ return out.strip()+ finally:+ proc.kill()+ proc.wait()+++def test_unread_stderr_pipe_does_not_block_the_process():+ result: list[str] = []+ t = threading.Thread(target=lambda: result.append(_run_child({}, 30)), daemon=True)+ t.start()+ t.join(20)+ assert result == ["finished"], "writing to an unread stderr pipe blocked the process"+++def test_control_unwrapped_stderr_pipe_blocks():+ """Proves the test above measures the real hazard, not a trivial pass."""+ result: list[str] = []+ t = threading.Thread(+ target=lambda: result.append(_run_child({"OMNIVOICE_BLOCKING_STDIO": "1"}, 30)),+ daemon=True,+ )+ t.start()+ t.join(4)+ assert t.is_alive() and result == []+++def test_main_installs_it_before_logging_and_servers_start():+ src = (BACKEND / "main.py").read_text(encoding="utf-8")+ assert src.index("_install_nonblocking_stdio()") < src.index("import math")