Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: free PyAV's per-thread scalers before a fork so preexec_fn spawns still exec (#12863)

Daniel Han8435a749be1d
diff --git a/studio/backend/tests/test_process_lifetime.py b/studio/backend/tests/test_process_lifetime.py
index 56e6196e23c..317d2857020 100644
--- a/studio/backend/tests/test_process_lifetime.py
+++ b/studio/backend/tests/test_process_lifetime.py
@@ -240,6 +240,176 @@ def test_pdeathsig_child_dies_when_parent_sigkilled(tmp_path):
proc.kill()
+def _fake_pyav_frame(monkeypatch):
+ import threading
+ import types
+
+ frame = types.ModuleType("av.video.frame")
+ frame._thread_local = threading.local()
+ monkeypatch.setitem(sys.modules, "av.video.frame", frame)
+ return frame
+
+
+def test_a_fork_frees_other_threads_cached_scalers_in_the_parent(monkeypatch):
+ """PyAV 19+ caches a scaler per thread; one freed in a fork child waits on FFmpeg threads it does not have."""
+ import threading
+
+ frame = _fake_pyav_frame(monkeypatch)
+ freed_in = []
+
+ class Scaler:
+ def __del__(self):
+ freed_in.append(os.getpid())
+
+ cached, stop = threading.Event(), threading.Event()
+
+ def video_worker():
+ frame._thread_local.reformatter = Scaler()
+ cached.set()
+ stop.wait() # an idle pool thread keeps its cache
+
+ worker = threading.Thread(target = video_worker, daemon = True)
+ worker.start()
+ try:
+ assert cached.wait(5), "the worker never cached its scaler"
+ pl._suspend_native_caches()
+ # Freed now, in this process, though the thread that cached it is still alive.
+ assert freed_in == [os.getpid()]
+ finally:
+ pl._resume_native_caches_in_parent()
+ stop.set()
+ worker.join(5)
+ assert type(frame._thread_local) is threading.local
+
+
+def test_nothing_is_cached_into_a_fork_while_the_old_scalers_are_freed(monkeypatch):
+ """Freeing a scaler releases the GIL, so another thread can reformat (and cache) before the fork happens."""
+ frame = _fake_pyav_frame(monkeypatch)
+
+ class Scaler:
+ def __del__(self):
+ # What a reformat on another thread does while this one is in sws_freeContext.
+ frame._thread_local.reformatter = "cached during the fork"
+
+ frame._thread_local.reformatter = Scaler()
+ try:
+ pl._suspend_native_caches()
+ assert getattr(frame._thread_local, "reformatter", None) is None
+ finally:
+ pl._resume_native_caches_in_parent()
+ frame._thread_local.reformatter = "cached after the fork"
+ assert frame._thread_local.reformatter == "cached after the fork"
+
+
+def test_overlapping_forks_keep_caching_off_until_the_last_returns(monkeypatch):
+ import threading
+
+ frame = _fake_pyav_frame(monkeypatch)
+ pl._suspend_native_caches()
+ pl._suspend_native_caches() # a second thread forking before the first one's fork returned
+ pl._resume_native_caches_in_parent()
+ assert type(frame._thread_local) is pl._UncachedReformatters
+ pl._resume_native_caches_in_parent()
+ assert type(frame._thread_local) is threading.local
+ pl._resume_native_caches_in_parent() # an unpaired resume cannot go negative
+ pl._suspend_native_caches()
+ assert type(frame._thread_local) is pl._UncachedReformatters
+ pl._resume_native_caches_in_parent()
+
+
+def test_the_hooks_are_registered_at_import(tmp_path):
+ """At import, so forks that bring their own preexec_fn or call os.fork directly are covered too."""
+ if not hasattr(os, "register_at_fork"):
+ pytest.skip("os.register_at_fork is POSIX-only")
+ probe = (
+ "import os, sys\n"
+ "registered = []\n"
+ "os.register_at_fork = lambda **kw: registered.append(sorted((k, v.__name__) for k, v in kw.items()))\n"
+ f"sys.path.insert(0, {str(_BACKEND)!r})\n"
+ "import utils.process_lifetime\n"
+ "print(registered)\n"
+ )
+ out = subprocess.run([sys.executable, "-c", probe], capture_output = True, text = True, timeout = 60)
+ ours = [
+ ("after_in_child", "_resume_native_caches_in_child"),
+ ("after_in_parent", "_resume_native_caches_in_parent"),
+ ("before", "_suspend_native_caches"),
+ ]
+ assert str(ours) in out.stdout, out.stdout + out.stderr
+
+
+_SPAWN_AFTER_REFORMAT = """
+import os, signal, subprocess, sys, threading
+sys.path.insert(0, {backend!r})
+import av, numpy as np
+from utils.process_lifetime import child_popen_kwargs, spawn_on_lifetime_thread
+
+ready, stop, errors = threading.Event(), threading.Event(), []
+
+
+def video_worker():
+ try:
+ frame = av.VideoFrame.from_ndarray(np.zeros((720, 1280, 3), dtype = np.uint8), format = "rgb24")
+ frame.reformat(format = "yuv420p", width = 640, height = 360)
+ except Exception as exc:
+ errors.append(repr(exc))
+ ready.set()
+ stop.wait()
+
+
+def fork_once(how):
+ if how == "spawner":
+ return spawn_on_lifetime_thread(lambda: subprocess.Popen(["true"], **child_popen_kwargs())).wait()
+ if how == "own_preexec":
+ return subprocess.Popen(["true"], preexec_fn = lambda: None).wait()
+ pid = os.fork()
+ if pid == 0:
+ os._exit(0)
+ return os.waitpid(pid, 0)[1]
+
+
+threading.Thread(target = video_worker, daemon = True).start()
+if not ready.wait(30) or errors:
+ print("worker failed", errors, flush = True)
+ os._exit(4)
+box = []
+t = threading.Thread(target = lambda: box.append(fork_once({how!r})), daemon = True)
+t.start()
+t.join(30)
+if t.is_alive():
+ print("hung", flush = True)
+ # The stuck child never reached PDEATHSIG, so it would outlive this script.
+ for tid in os.listdir("/proc/self/task"):
+ try:
+ with open(f"/proc/self/task/{{tid}}/children") as handle:
+ kids = handle.read().split()
+ except OSError:
+ continue
+ for kid in kids:
+ try:
+ os.kill(int(kid), signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ os._exit(3)
+print("exit", box[0], flush = True)
+"""
+
+
[email protected](not IS_LINUX, reason = "the fork-then-Python spawn paths are Linux-only here")
[email protected]("how", ["spawner", "own_preexec", "os_fork"])
+def test_a_fork_after_a_video_reformat_still_completes(tmp_path, how):
+ """A pool thread that reformatted one frame used to hang the next fork whose child runs Python before exec."""
+ pytest.importorskip("av")
+ pytest.importorskip("numpy")
+ script = tmp_path / "fork_after_reformat.py"
+ script.write_text(_SPAWN_AFTER_REFORMAT.format(backend = str(_BACKEND), how = how))
+ # The hang is in a child that never execs, so bound the whole scenario from outside.
+ result = subprocess.run(
+ [sys.executable, str(script)], capture_output = True, text = True, timeout = 120
+ )
+ assert result.stdout.strip() == "exit 0", result.stdout + result.stderr
+
+
@pytest.mark.skipif(sys.platform != "win32", reason = "Windows Job Object")
def test_windows_job_kills_child_when_parent_dies(tmp_path):
# Real kill-on-job-close: the parent installs the job and assigns itself, a
diff --git a/studio/backend/utils/process_lifetime.py b/studio/backend/utils/process_lifetime.py
index 00d447f422e..5fb08552b7f 100644
--- a/studio/backend/utils/process_lifetime.py
+++ b/studio/backend/utils/process_lifetime.py
@@ -379,6 +379,69 @@ def _reset_after_fork() -> None:
_spawner = None
+class _UncachedReformatters(threading.local):
+ """PyAV's per-thread scaler cache while a fork is in flight: reads find nothing, writes are dropped."""
+
+ def __setattr__(self, name: str, value: object) -> None:
+ if name != "reformatter":
+ super().__setattr__(name, value)
+
+
+_forks_in_flight = 0
+
+
+def _suspend_native_caches() -> None:
+ """Before a fork: free the FFmpeg scaler PyAV 19+ caches for every thread that ever reformatted a frame.
+
+ A child that runs Python before exec (a preexec_fn spawn, or a bare os.fork) goes through PyOS_AfterFork_Child,
+ which frees the other threads' state. A scaler freed there waits on FFmpeg slice threads that do not exist in
+ the child, so the child never execs and the spawner, with every launch queued behind it, blocks forever. Freed
+ here instead, where those threads still run. The replacement drops writes until the fork is done: freeing the
+ old scalers releases the GIL, and a reformat on another thread must not cache a new one into this fork."""
+ global _forks_in_flight
+ _forks_in_flight += 1
+ try:
+ frame = sys.modules.get("av.video.frame")
+ cache = getattr(frame, "_thread_local", None)
+ if isinstance(cache, threading.local) and not isinstance(cache, _UncachedReformatters):
+ frame._thread_local = _UncachedReformatters()
+ except Exception: # noqa: BLE001 - a fork hook must not get in the fork's way
+ pass
+
+
+def _resume_native_caches() -> None:
+ try:
+ frame = sys.modules.get("av.video.frame")
+ if isinstance(getattr(frame, "_thread_local", None), _UncachedReformatters):
+ frame._thread_local = threading.local()
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _resume_native_caches_in_parent() -> None:
+ global _forks_in_flight
+ _forks_in_flight = max(0, _forks_in_flight - 1)
+ # Another thread's fork may still be between its own hooks.
+ if _forks_in_flight == 0:
+ _resume_native_caches()
+
+
+def _resume_native_caches_in_child() -> None:
+ global _forks_in_flight
+ _forks_in_flight = 0
+ _resume_native_caches()
+
+
+# At import, not lazily: run_server imports this module before anything spawns, and every fork needs it, including
+# the ones that bring their own preexec_fn or call os.fork directly.
+if hasattr(os, "register_at_fork"):
+ os.register_at_fork(
+ before = _suspend_native_caches,
+ after_in_parent = _resume_native_caches_in_parent,
+ after_in_child = _resume_native_caches_in_child,
+ )
+
+
def _adopt_fork_reset() -> None:
"""Register the child-side reset once, lazily. Best-effort like the rest of
this module: os.register_at_fork is POSIX-only and absent on Windows."""