Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: free PyAV's per-thread scalers before a fork so preexec_fn spawns still exec (#12863)
Daniel Han8435a749be1d
diff --git a/studio/backend/tests/test_process_lifetime.py b/studio/backend/tests/test_process_lifetime.pyindex 56e6196e23c..317d2857020 100644--- a/studio/backend/tests/test_process_lifetime.py+++ b/studio/backend/tests/test_process_lifetime.py@@ -240,6 +240,176 @@ def test_pdeathsig_child_dies_when_parent_sigkilled(tmp_path):proc.kill()+def _fake_pyav_frame(monkeypatch):+ import threading+ import types++ frame = types.ModuleType("av.video.frame")+ frame._thread_local = threading.local()+ monkeypatch.setitem(sys.modules, "av.video.frame", frame)+ return frame+++def test_a_fork_frees_other_threads_cached_scalers_in_the_parent(monkeypatch):+ """PyAV 19+ caches a scaler per thread; one freed in a fork child waits on FFmpeg threads it does not have."""+ import threading++ frame = _fake_pyav_frame(monkeypatch)+ freed_in = []++ class Scaler:+ def __del__(self):+ freed_in.append(os.getpid())++ cached, stop = threading.Event(), threading.Event()++ def video_worker():+ frame._thread_local.reformatter = Scaler()+ cached.set()+ stop.wait() # an idle pool thread keeps its cache++ worker = threading.Thread(target = video_worker, daemon = True)+ worker.start()+ try:+ assert cached.wait(5), "the worker never cached its scaler"+ pl._suspend_native_caches()+ # Freed now, in this process, though the thread that cached it is still alive.+ assert freed_in == [os.getpid()]+ finally:+ pl._resume_native_caches_in_parent()+ stop.set()+ worker.join(5)+ assert type(frame._thread_local) is threading.local+++def test_nothing_is_cached_into_a_fork_while_the_old_scalers_are_freed(monkeypatch):+ """Freeing a scaler releases the GIL, so another thread can reformat (and cache) before the fork happens."""+ frame = _fake_pyav_frame(monkeypatch)++ class Scaler:+ def __del__(self):+ # What a reformat on another thread does while this one is in sws_freeContext.+ frame._thread_local.reformatter = "cached during the fork"++ frame._thread_local.reformatter = Scaler()+ try:+ pl._suspend_native_caches()+ assert getattr(frame._thread_local, "reformatter", None) is None+ finally:+ pl._resume_native_caches_in_parent()+ frame._thread_local.reformatter = "cached after the fork"+ assert frame._thread_local.reformatter == "cached after the fork"+++def test_overlapping_forks_keep_caching_off_until_the_last_returns(monkeypatch):+ import threading++ frame = _fake_pyav_frame(monkeypatch)+ pl._suspend_native_caches()+ pl._suspend_native_caches() # a second thread forking before the first one's fork returned+ pl._resume_native_caches_in_parent()+ assert type(frame._thread_local) is pl._UncachedReformatters+ pl._resume_native_caches_in_parent()+ assert type(frame._thread_local) is threading.local+ pl._resume_native_caches_in_parent() # an unpaired resume cannot go negative+ pl._suspend_native_caches()+ assert type(frame._thread_local) is pl._UncachedReformatters+ pl._resume_native_caches_in_parent()+++def test_the_hooks_are_registered_at_import(tmp_path):+ """At import, so forks that bring their own preexec_fn or call os.fork directly are covered too."""+ if not hasattr(os, "register_at_fork"):+ pytest.skip("os.register_at_fork is POSIX-only")+ probe = (+ "import os, sys\n"+ "registered = []\n"+ "os.register_at_fork = lambda **kw: registered.append(sorted((k, v.__name__) for k, v in kw.items()))\n"+ f"sys.path.insert(0, {str(_BACKEND)!r})\n"+ "import utils.process_lifetime\n"+ "print(registered)\n"+ )+ out = subprocess.run([sys.executable, "-c", probe], capture_output = True, text = True, timeout = 60)+ ours = [+ ("after_in_child", "_resume_native_caches_in_child"),+ ("after_in_parent", "_resume_native_caches_in_parent"),+ ("before", "_suspend_native_caches"),+ ]+ assert str(ours) in out.stdout, out.stdout + out.stderr+++_SPAWN_AFTER_REFORMAT = """+import os, signal, subprocess, sys, threading+sys.path.insert(0, {backend!r})+import av, numpy as np+from utils.process_lifetime import child_popen_kwargs, spawn_on_lifetime_thread++ready, stop, errors = threading.Event(), threading.Event(), []+++def video_worker():+ try:+ frame = av.VideoFrame.from_ndarray(np.zeros((720, 1280, 3), dtype = np.uint8), format = "rgb24")+ frame.reformat(format = "yuv420p", width = 640, height = 360)+ except Exception as exc:+ errors.append(repr(exc))+ ready.set()+ stop.wait()+++def fork_once(how):+ if how == "spawner":+ return spawn_on_lifetime_thread(lambda: subprocess.Popen(["true"], **child_popen_kwargs())).wait()+ if how == "own_preexec":+ return subprocess.Popen(["true"], preexec_fn = lambda: None).wait()+ pid = os.fork()+ if pid == 0:+ os._exit(0)+ return os.waitpid(pid, 0)[1]+++threading.Thread(target = video_worker, daemon = True).start()+if not ready.wait(30) or errors:+ print("worker failed", errors, flush = True)+ os._exit(4)+box = []+t = threading.Thread(target = lambda: box.append(fork_once({how!r})), daemon = True)+t.start()+t.join(30)+if t.is_alive():+ print("hung", flush = True)+ # The stuck child never reached PDEATHSIG, so it would outlive this script.+ for tid in os.listdir("/proc/self/task"):+ try:+ with open(f"/proc/self/task/{{tid}}/children") as handle:+ kids = handle.read().split()+ except OSError:+ continue+ for kid in kids:+ try:+ os.kill(int(kid), signal.SIGKILL)+ except ProcessLookupError:+ pass+ os._exit(3)+print("exit", box[0], flush = True)+"""++[email protected](not IS_LINUX, reason = "the fork-then-Python spawn paths are Linux-only here")[email protected]("how", ["spawner", "own_preexec", "os_fork"])+def test_a_fork_after_a_video_reformat_still_completes(tmp_path, how):+ """A pool thread that reformatted one frame used to hang the next fork whose child runs Python before exec."""+ pytest.importorskip("av")+ pytest.importorskip("numpy")+ script = tmp_path / "fork_after_reformat.py"+ script.write_text(_SPAWN_AFTER_REFORMAT.format(backend = str(_BACKEND), how = how))+ # The hang is in a child that never execs, so bound the whole scenario from outside.+ result = subprocess.run(+ [sys.executable, str(script)], capture_output = True, text = True, timeout = 120+ )+ assert result.stdout.strip() == "exit 0", result.stdout + result.stderr++@pytest.mark.skipif(sys.platform != "win32", reason = "Windows Job Object")def test_windows_job_kills_child_when_parent_dies(tmp_path):# Real kill-on-job-close: the parent installs the job and assigns itself, adiff --git a/studio/backend/utils/process_lifetime.py b/studio/backend/utils/process_lifetime.pyindex 00d447f422e..5fb08552b7f 100644--- a/studio/backend/utils/process_lifetime.py+++ b/studio/backend/utils/process_lifetime.py@@ -379,6 +379,69 @@ def _reset_after_fork() -> None:_spawner = None+class _UncachedReformatters(threading.local):+ """PyAV's per-thread scaler cache while a fork is in flight: reads find nothing, writes are dropped."""++ def __setattr__(self, name: str, value: object) -> None:+ if name != "reformatter":+ super().__setattr__(name, value)+++_forks_in_flight = 0+++def _suspend_native_caches() -> None:+ """Before a fork: free the FFmpeg scaler PyAV 19+ caches for every thread that ever reformatted a frame.++ A child that runs Python before exec (a preexec_fn spawn, or a bare os.fork) goes through PyOS_AfterFork_Child,+ which frees the other threads' state. A scaler freed there waits on FFmpeg slice threads that do not exist in+ the child, so the child never execs and the spawner, with every launch queued behind it, blocks forever. Freed+ here instead, where those threads still run. The replacement drops writes until the fork is done: freeing the+ old scalers releases the GIL, and a reformat on another thread must not cache a new one into this fork."""+ global _forks_in_flight+ _forks_in_flight += 1+ try:+ frame = sys.modules.get("av.video.frame")+ cache = getattr(frame, "_thread_local", None)+ if isinstance(cache, threading.local) and not isinstance(cache, _UncachedReformatters):+ frame._thread_local = _UncachedReformatters()+ except Exception: # noqa: BLE001 - a fork hook must not get in the fork's way+ pass+++def _resume_native_caches() -> None:+ try:+ frame = sys.modules.get("av.video.frame")+ if isinstance(getattr(frame, "_thread_local", None), _UncachedReformatters):+ frame._thread_local = threading.local()+ except Exception: # noqa: BLE001+ pass+++def _resume_native_caches_in_parent() -> None:+ global _forks_in_flight+ _forks_in_flight = max(0, _forks_in_flight - 1)+ # Another thread's fork may still be between its own hooks.+ if _forks_in_flight == 0:+ _resume_native_caches()+++def _resume_native_caches_in_child() -> None:+ global _forks_in_flight+ _forks_in_flight = 0+ _resume_native_caches()+++# At import, not lazily: run_server imports this module before anything spawns, and every fork needs it, including+# the ones that bring their own preexec_fn or call os.fork directly.+if hasattr(os, "register_at_fork"):+ os.register_at_fork(+ before = _suspend_native_caches,+ after_in_parent = _resume_native_caches_in_parent,+ after_in_child = _resume_native_caches_in_child,+ )++def _adopt_fork_reset() -> None:"""Register the child-side reset once, lazily. Best-effort like the rest ofthis module: os.register_at_fork is POSIX-only and absent on Windows."""