Skip to content

VoiceStudio is the open-source, fully-local ElevenLabs alternative — voice cloning, voice design, video dubbing, dictation, transcription & audiobook creation in 646 languages.

fix(backend): keep health probes responsive when the log reader is slow

velixiode2513d6b43a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index abbe0d4ad..58f6760b7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,6 +16,10 @@ metadata and the backend fallback mirror it.
- Preview iOS and Android companion apps: drive your own VoiceStudio from your phone (speak, voices and cloning, dubbing, dictation) after pairing by QR code; build from source for now ([guide](docs/mobile.md)) (#2965)
+### Fixed
+
+- The backend no longer freezes ("temporarily not responding", startup never finishing) when the app reads its log slowly, for example during a model download (#PRN) — thanks @tigerscurrying-prog!
+
### Docs
- Contributor guides and the pull request template explain how to sign the CLA before opening a first pull request (#2676)
diff --git a/backend/core/loop_stall_monitor.py b/backend/core/loop_stall_monitor.py
new file mode 100644
index 000000000..dd421b798
--- /dev/null
+++ b/backend/core/loop_stall_monitor.py
@@ -0,0 +1,80 @@
+"""Name the code that blocks the event loop when /health stops answering.
+
+The desktop shell polls ``/health`` with a 1.5 s deadline. When a synchronous
+call (or a GIL-hungry thread) holds the loop longer than that, the shell
+reports "backend busy / not responding" while the log shows only unrelated 200s
+- nothing names the culprit (#2970, #2876, #2840, #2794, #2756). A daemon
+thread pings the loop and, if a ping goes unanswered for the threshold, dumps
+every thread's Python stack once per stall. Pure diagnostics: never raises into
+startup, and ``OMNIVOICE_LOOP_STALL_S=0`` disables it.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import sys
+import threading
+import time
+
+logger = logging.getLogger("omnivoice.loop_stall")
+_DEFAULT_THRESHOLD_S = 4.0
+_POLL_S = 1.0
+
+
+def stall_threshold() -> float:
+ try:
+ return float(os.environ.get("OMNIVOICE_LOOP_STALL_S", _DEFAULT_THRESHOLD_S))
+ except ValueError:
+ return _DEFAULT_THRESHOLD_S
+
+
+def start_loop_stall_monitor(
+ loop: asyncio.AbstractEventLoop | None = None,
+ *,
+ threshold: float | None = None,
+ poll: float = _POLL_S,
+ dump=None,
+) -> threading.Event | None:
+ """Start the monitor; returns its stop event, or None when disabled."""
+ limit = stall_threshold() if threshold is None else threshold
+ if limit <= 0:
+ return None
+ loop = loop or asyncio.get_running_loop()
+ stop = threading.Event()
+ last_seen = [time.monotonic()]
+
+ def _dump_threads() -> None:
+ import faulthandler
+
+ faulthandler.dump_traceback(file=sys.stderr, all_threads=True)
+
+ emit = dump or _dump_threads
+
+ def _beat() -> None:
+ last_seen[0] = time.monotonic()
+
+ def _run() -> None:
+ reported = False
+ while not stop.wait(poll):
+ try:
+ loop.call_soon_threadsafe(_beat)
+ except RuntimeError: # loop closed: the process is exiting
+ return
+ lag = time.monotonic() - last_seen[0]
+ if lag < limit:
+ reported = False
+ elif not reported:
+ reported = True
+ logger.warning(
+ "Event loop has not run for %.1fs; /health cannot answer. "
+ "Stacks of all threads follow - the main thread's frame is "
+ "the blocking call.", lag,
+ )
+ try:
+ emit()
+ except Exception: # diagnostics must never take the app down
+ logger.debug("stall thread dump failed", exc_info=True)
+
+ threading.Thread(target=_run, name="loop-stall-monitor", daemon=True).start()
+ return stop
diff --git a/backend/core/nonblocking_stdio.py b/backend/core/nonblocking_stdio.py
new file mode 100644
index 000000000..1375e2f1f
--- /dev/null
+++ b/backend/core/nonblocking_stdio.py
@@ -0,0 +1,155 @@
+"""Never let a slow log reader freeze the event loop.
+
+The desktop shell starts the backend with stdout and stderr connected to pipes
+it reads asynchronously. A pipe holds only a few tens of KB; once the reader
+falls behind (a model download's progress bar writes hundreds of lines a
+second, or the shell is busy) the next ``write`` blocks. uvicorn writes its
+access log from the event loop thread, so the whole loop stops, ``/health``
+misses its deadline and the shell reports "backend not responding" while the
+log's last lines are ordinary 200s (#2970, #2876, #2840, #2794, #2756).
+
+``install()`` puts a bounded in-memory buffer between producers and a pipe: a
+daemon thread owns the real (possibly blocking) write, producers return at
+once, and when the buffer overflows the oldest text is dropped and one notice
+says how much. Terminals and regular files cannot stall like that and are left
+untouched, so behaviour is identical on every platform.
+"""
+from __future__ import annotations
+
+import atexit
+import collections
+import io
+import os
+import stat
+import sys
+import threading
+
+DEFAULT_LIMIT_BYTES = 2 * 1024 * 1024
+_EXIT_DRAIN_S = 2.0
+
+
+def _is_pipe(stream) -> bool:
+ try:
+ mode = os.fstat(stream.fileno()).st_mode
+ except (AttributeError, OSError, ValueError):
+ return False
+ return stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)
+
+
+class DecoupledStream(io.TextIOBase):
+ """Text stream whose ``write`` never waits on the underlying sink."""
+
+ def __init__(self, real, *, limit_bytes: int = DEFAULT_LIMIT_BYTES) -> None:
+ self._real = real
+ self._limit = max(1024, int(limit_bytes))
+ self._chunks: collections.deque[str] = collections.deque()
+ self._size = 0
+ self._dropped = 0
+ self._broken = False
+ self._busy = False
+ self._cond = threading.Condition()
+ self._thread = threading.Thread(
+ target=self._pump, name="stdio-pump", daemon=True
+ )
+ self._thread.start()
+
+ # -- producer side: constant time, never blocks on the sink ------------
+ def write(self, text) -> int:
+ if not isinstance(text, str):
+ text = str(text)
+ if not text:
+ return 0
+ with self._cond:
+ if self._broken:
+ return len(text)
+ self._chunks.append(text)
+ self._size += len(text)
+ while self._size > self._limit and len(self._chunks) > 1:
+ self._size -= len(self._chunks.popleft())
+ self._dropped += 1
+ self._cond.notify()
+ return len(text)
+
+ def flush(self) -> None:
+ """No-op by design: waiting on the sink is exactly what must not happen."""
+
+ def writable(self) -> bool:
+ return True
+
+ def isatty(self) -> bool:
+ return False
+
+ def fileno(self) -> int:
+ return self._real.fileno()
+
+ def __getattr__(self, name): # encoding, errors, buffer, reconfigure, ...
+ return getattr(self._real, name)
+
+ # -- consumer side: the only thread that may block ---------------------
+ def _pump(self) -> None:
+ while True:
+ with self._cond:
+ while not self._chunks and not self._dropped:
+ self._busy = False
+ self._cond.notify_all()
+ self._cond.wait()
+ self._busy = True
+ parts = list(self._chunks)
+ self._chunks.clear()
+ self._size = 0
+ dropped, self._dropped = self._dropped, 0
+ text = "".join(parts)
+ if dropped:
+ text = (
+ f"[log] {dropped} output chunk(s) dropped: the log reader "
+ "was not keeping up\n" + text
+ )
+ if not self._emit(text):
+ with self._cond:
+ self._broken = True
+ self._chunks.clear()
+ self._size = 0
+ self._busy = False
+ self._cond.notify_all()
+ return
+
+ def _emit(self, text: str) -> bool:
+ try:
+ try:
+ self._real.write(text)
+ except UnicodeEncodeError:
+ enc = getattr(self._real, "encoding", None) or "utf-8"
+ self._real.write(text.encode(enc, "replace").decode(enc, "replace"))
+ self._real.flush()
+ return True
+ except (OSError, ValueError): # reader gone or stream closed
+ return False
+
+ def drain(self, timeout: float = _EXIT_DRAIN_S) -> None:
+ """Best-effort wait (bounded) for buffered text to reach the sink."""
+ with self._cond:
+ self._cond.wait_for(
+ lambda: self._broken or (not self._chunks and not self._busy),
+ timeout,
+ )
+
+
+def install(*, limit_bytes: int = DEFAULT_LIMIT_BYTES) -> list[DecoupledStream]:
+ """Wrap ``sys.stdout``/``sys.stderr`` when they are pipes. Idempotent."""
+ if os.environ.get("OMNIVOICE_BLOCKING_STDIO", "").strip().lower() in ("1", "true", "yes", "on"):
+ return []
+ wrapped: list[DecoupledStream] = []
+ for name in ("stdout", "stderr"):
+ stream = getattr(sys, name, None)
+ if stream is None or isinstance(stream, DecoupledStream) or not _is_pipe(stream):
+ continue
+ replacement = DecoupledStream(stream, limit_bytes=limit_bytes)
+ setattr(sys, name, replacement)
+ wrapped.append(replacement)
+ if wrapped:
+ def _drain_all() -> None:
+ for s in wrapped:
+ s.drain()
+
+ atexit.register(_drain_all)
+ return wrapped
diff --git a/backend/main.py b/backend/main.py
index d17f0a667..7d3d2197e 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -33,6 +33,13 @@ from core.contained_subprocess import secure_backend_drain_fd # noqa: E402
secure_backend_drain_fd()
+# The shell reads our stdout/stderr through pipes; a reader that falls behind
+# would otherwise block uvicorn's access-log write on the event loop and freeze
+# /health (#2970 class). Producers get a bounded buffer instead.
+from core.nonblocking_stdio import install as _install_nonblocking_stdio # noqa: E402
+
+_install_nonblocking_stdio()
+
import math # noqa: E402
# Windows: run every child process (ffmpeg, engine sidecars, yt-dlp, demucs, …)
@@ -1218,7 +1225,15 @@ async def lifespan(app: FastAPI):
# StartupGate. The watchdog stays armed until the deferred task
# disarms it, so a hang inside Phase A/B still gets its thread dump.
app.state.startup_task = asyncio.create_task(_deferred_startup(app))
+ try:
+ from core.loop_stall_monitor import start_loop_stall_monitor
+
+ _loop_stall_stop = start_loop_stall_monitor()
+ except Exception:
+ _loop_stall_stop = None # diagnostics-only
yield
+ if _loop_stall_stop is not None:
+ _loop_stall_stop.set()
# ── Graceful shutdown (SIGTERM from Tauri, Ctrl+C, etc.) ────────────
# Retire the run sentinel FIRST, before any bounded wait below (#1895):
# once uvicorn has begun graceful shutdown the exit is deliberate by
diff --git a/docs/install/troubleshooting.md b/docs/install/troubleshooting.md
index a11e1cc23..b70745eab 100644
--- a/docs/install/troubleshooting.md
+++ b/docs/install/troubleshooting.md
@@ -844,6 +844,14 @@ rather than failing with "Can't reach the local backend". Nothing to do — it
clears itself as soon as the job releases the event loop. If the dot turns
**red** and names an exit code, that is a real crash; use the sections above.
+The backend also never waits on its own log output: stdout/stderr go through a
+bounded buffer when the shell reads them through pipes, so a slow log reader (a
+model download's progress bar, a busy shell) can no longer freeze the event
+loop. If the buffer overflows, the log shows one `[log] N output chunk(s)
+dropped` line. Set `OMNIVOICE_BLOCKING_STDIO=1` to restore plain blocking
+writes. When the loop does stall, the backend log names the blocking call
+(`OMNIVOICE_LOOP_STALL_S`, default 4 s, `0` disables).
+
**Two things changed here** ([#1190](https://github.com/debpalash/VoiceStudio/issues/1190)):
- **Waiting in line is no longer counted as compute.** The generate budget used
diff --git a/tests/test_loop_stall_monitor.py b/tests/test_loop_stall_monitor.py
new file mode 100644
index 000000000..f2cf17f57
--- /dev/null
+++ b/tests/test_loop_stall_monitor.py
@@ -0,0 +1,36 @@
+"""A blocked event loop must leave a thread dump naming the blocker."""
+import asyncio
+import time
+
+from core.loop_stall_monitor import start_loop_stall_monitor
+
+
+def test_dumps_once_when_loop_is_blocked():
+ dumps = []
+
+ async def scenario():
+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))
+ await asyncio.sleep(0.2)
+ time.sleep(0.9) # the synchronous call that blocks /health
+ await asyncio.sleep(0.2)
+ stop.set()
+
+ asyncio.run(scenario())
+ assert len(dumps) == 1
+
+
+def test_silent_when_loop_is_healthy():
+ dumps = []
+
+ async def scenario():
+ stop = start_loop_stall_monitor(threshold=0.3, poll=0.05, dump=lambda: dumps.append(1))
+ await asyncio.sleep(0.6)
+ stop.set()
+
+ asyncio.run(scenario())
+ assert dumps == []
+
+
+def test_disabled_by_zero(monkeypatch):
+ monkeypatch.setenv("OMNIVOICE_LOOP_STALL_S", "0")
+ assert start_loop_stall_monitor(loop=object()) is None
diff --git a/tests/test_nonblocking_stdio.py b/tests/test_nonblocking_stdio.py
new file mode 100644
index 000000000..ac093f9b6
--- /dev/null
+++ b/tests/test_nonblocking_stdio.py
@@ -0,0 +1,129 @@
+"""A slow log reader must never block the backend (#2970, #2876, #2840, #2794, #2756).
+
+The shell reads stdout/stderr through pipes. uvicorn writes its access log on
+the event loop thread, so a full pipe froze ``/health`` while the log showed
+only 200s. Measured on the real backend with an unread stderr pipe: it stopped
+answering after ~1100 logged requests; with the buffer it served 6000.
+"""
+import subprocess
+import sys
+import textwrap
+import threading
+import time
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+BACKEND = ROOT / "backend"
+sys.path.insert(0, str(BACKEND))
+
+from core.nonblocking_stdio import DecoupledStream # noqa: E402
+
+
+class _BlockedSink:
+ encoding = "utf-8"
+
+ def __init__(self):
+ self.gate = threading.Event()
+ self.received = []
+
+ def write(self, text):
+ self.gate.wait()
+ self.received.append(text)
+
+ def flush(self):
+ pass
+
+
+def test_write_returns_immediately_while_sink_is_blocked():
+ sink = _BlockedSink()
+ stream = DecoupledStream(sink, limit_bytes=4096)
+ start = time.monotonic()
+ for i in range(20000):
+ stream.write(f"line {i}\n")
+ assert time.monotonic() - start < 2.0
+ sink.gate.set()
+ stream.drain(5)
+ out = "".join(sink.received)
+ assert "dropped" in out and "line 19999\n" in out # newest text survives
+ assert len(out) < 2 * 4096 + 200 # one batch in flight + one buffered
+
+
+def test_text_is_delivered_in_order_when_reader_keeps_up():
+ sink = _BlockedSink()
+ sink.gate.set()
+ stream = DecoupledStream(sink)
+ for i in range(50):
+ stream.write(f"{i}\n")
+ stream.drain(5)
+ assert "".join(sink.received) == "".join(f"{i}\n" for i in range(50))
+
+
+def test_broken_sink_never_raises_into_the_producer():
+ class Gone(_BlockedSink):
+ def write(self, text):
+ raise BrokenPipeError
+
+ sink = Gone()
+ stream = DecoupledStream(sink)
+ stream.write("x\n")
+ stream.drain(2)
+ assert stream.write("y\n") == 2
+
+
+_CHILD = textwrap.dedent(
+ """
+ import sys
+ sys.path.insert(0, {backend!r})
+ from core.nonblocking_stdio import install
+ install()
+ chunk = "x" * 1023 + "\\n"
+ for _ in range(8192): # 8 MB, far beyond any pipe buffer
+ sys.stderr.write(chunk)
+ print("finished", flush=True)
+ """
+)
+
+
+def _run_child(env_extra: dict[str, str], timeout: float):
+ import os
+
+ proc = subprocess.Popen(
+ [sys.executable, "-I", "-c", _CHILD.format(backend=str(BACKEND))],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, # never read: this is the slow log reader
+ env={**os.environ, **env_extra},
+ text=True,
+ )
+ try:
+ out = proc.stdout.readline() # type: ignore[union-attr]
+ return out.strip()
+ finally:
+ proc.kill()
+ proc.wait()
+
+
+def test_unread_stderr_pipe_does_not_block_the_process():
+ result: list[str] = []
+ t = threading.Thread(target=lambda: result.append(_run_child({}, 30)), daemon=True)
+ t.start()
+ t.join(20)
+ assert result == ["finished"], "writing to an unread stderr pipe blocked the process"
+
+
+def test_control_unwrapped_stderr_pipe_blocks():
+ """Proves the test above measures the real hazard, not a trivial pass."""
+ result: list[str] = []
+ t = threading.Thread(
+ target=lambda: result.append(_run_child({"OMNIVOICE_BLOCKING_STDIO": "1"}, 30)),
+ daemon=True,
+ )
+ t.start()
+ t.join(4)
+ assert t.is_alive() and result == []
+
+
+def test_main_installs_it_before_logging_and_servers_start():
+ src = (BACKEND / "main.py").read_text(encoding="utf-8")
+ assert src.index("_install_nonblocking_stdio()") < src.index("import math")