Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Drain worker threads without racing their start (#11400)
Daniel Han655c1df61793
diff --git a/studio/backend/tests/test_mcp_http_sessions.py b/studio/backend/tests/test_mcp_http_sessions.pyindex 2add86ac35d..3d59e97428f 100644--- a/studio/backend/tests/test_mcp_http_sessions.py+++ b/studio/backend/tests/test_mcp_http_sessions.py@@ -20,6 +20,8 @@ from types import SimpleNamespaceimport pytest+from .thread_drain import join_when_started+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)@@ -982,7 +984,7 @@ def test_the_queue_of_pending_closes_is_bounded(monkeypatch, clients):release.set()for t in threading.enumerate():if t.name.startswith("mcp-") and t is not threading.current_thread():- t.join(5)+ join_when_started(t, timeout = 5)def test_closing_many_sessions_does_not_spawn_a_thread_each(monkeypatch, clients):diff --git a/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py b/studio/backend/tests/test_mlx_autorepair_no_torch_optout.pyindex 96df0cc7fdf..efeac5c84a9 100644--- a/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py+++ b/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py@@ -24,6 +24,8 @@ from pathlib import Pathimport pytest+from .thread_drain import join_when_started+_BACKEND = Path(__file__).resolve().parent.parentif str(_BACKEND) not in sys.path:sys.path.insert(0, str(_BACKEND))@@ -41,8 +43,7 @@ def _isolated(monkeypatch):yieldfor thread in threading.enumerate():if thread.name == "mlx-autorepair":- thread.join(timeout = 5)- assert not thread.is_alive()+ assert join_when_started(thread, timeout = 5)def _install_manifest_returning(monkeypatch, value):@@ -220,7 +221,7 @@ def test_a_normal_install_still_repairs(monkeypatch):assert mr.start_mlx_autorepair_if_needed() is Truefor thread in threading.enumerate():if thread.name == "mlx-autorepair":- thread.join(timeout = 5)+ join_when_started(thread, timeout = 5)assert attempts == [1]@@ -236,7 +237,7 @@ def test_an_older_install_with_no_recorded_mode_still_repairs(monkeypatch):assert mr.start_mlx_autorepair_if_needed() is Truefor thread in threading.enumerate():if thread.name == "mlx-autorepair":- thread.join(timeout = 5)+ join_when_started(thread, timeout = 5)assert attempts == [1]diff --git a/studio/backend/tests/test_mlx_repair.py b/studio/backend/tests/test_mlx_repair.pyindex e74baa1d7ea..12e9eea9ae4 100644--- a/studio/backend/tests/test_mlx_repair.py+++ b/studio/backend/tests/test_mlx_repair.py@@ -15,6 +15,8 @@ from pathlib import Pathimport pytest+from .thread_drain import join_when_started+_BACKEND = Path(__file__).resolve().parent.parentif str(_BACKEND) not in sys.path:sys.path.insert(0, str(_BACKEND))@@ -45,8 +47,7 @@ def _reset_attempt_guard(monkeypatch):# against the next test's globals.for thread in threading.enumerate():if thread.name == "mlx-autorepair":- thread.join(timeout = 5)- assert not thread.is_alive(), (+ assert join_when_started(thread, timeout = 5), ("an mlx-autorepair worker outlived its test; once these stubs are ""restored it runs the real repair and detection against another test")@@ -402,7 +403,7 @@ def test_apple_silicon_missing_mlx_starts_repair_and_redetects(monkeypatch):# Join the daemon thread deterministically.for thread in threading.enumerate():if thread.name == "mlx-autorepair":- thread.join(timeout = 5)+ join_when_started(thread, timeout = 5)assert repaired["called"] is Trueassert redetected["called"] is True@@ -596,8 +597,7 @@ def _recorded_announcements(monkeypatch):def _join_the_repair_worker():for thread in threading.enumerate():if thread.name == "mlx-autorepair":- thread.join(timeout = 5)- assert not thread.is_alive()+ assert join_when_started(thread, timeout = 5)def test_a_stack_that_measures_usable_overturns_the_verdict(monkeypatch):diff --git a/studio/backend/tests/test_sandbox_files_and_storage_roots.py b/studio/backend/tests/test_sandbox_files_and_storage_roots.pyindex 027b1b3f8b0..1650364f008 100644--- a/studio/backend/tests/test_sandbox_files_and_storage_roots.py+++ b/studio/backend/tests/test_sandbox_files_and_storage_roots.py@@ -25,6 +25,8 @@ from pathlib import Pathimport pytest+from .thread_drain import join_when_started+def _shared_setup_1(monkeypatch, tmp_path):monkeypatch.setenv("UNSLOTH_STUDIO_SANDBOX_HOME", str(tmp_path / "sb"))@@ -346,7 +348,7 @@ def test_legacy_sandbox_is_migrated(tmp_path, monkeypatch):# waits on the whole tree.for thread in threading.enumerate():if thread.name == "sandbox-migrate":- thread.join(30)+ join_when_started(thread, timeout = 30)moved = wd.parent / "__LOCALID_old1234" / "results.csv"print(f"\nmigrated to {moved}")assert moved.is_file()diff --git a/studio/backend/tests/test_thread_drain.py b/studio/backend/tests/test_thread_drain.pynew file mode 100644index 00000000000..1bd34a72879--- /dev/null+++ b/studio/backend/tests/test_thread_drain.py@@ -0,0 +1,82 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""`join_when_started` absorbs the not-started-yet window and nothing else.++The window is real but tiny, so it cannot be hit on demand by starting threads and hoping. It+is reproduced here directly instead: a thread that has never been started is in exactly the+state `threading.enumerate()` can hand a drain, and `join()` on it raises the same+`RuntimeError` for the same reason.++What has to stay true is both halves. The helper must not raise on an unstarted thread, which+is the bug it exists for, and it must still report a thread that is genuinely still running,+which is what every caller asserts on. A helper that swallowed both would turn a hung worker+into a green test.+"""++from __future__ import annotations++import threading+import time++import pytest++from .thread_drain import join_when_started+++def test_join_raises_on_its_own_for_a_thread_that_never_started():+ """The premise. Without this the first test below could pass for the wrong reason."""+ never = threading.Thread(target = lambda: None)+ with pytest.raises(RuntimeError, match = "before it is started"):+ never.join(timeout = 0.01)+++def test_an_unstarted_thread_does_not_raise_and_is_not_called_drained():+ """Absorbing the exception must not turn into claiming the thread is done.++ A thread that `join()` still refuses at the deadline has not run yet and, if it came out+ of `threading.enumerate()`, still will. `is_alive()` cannot tell that apart from finished,+ since it is False on both sides, so the answer here is taken from `join()` rather than+ from `is_alive()`. Callers restore their monkeypatches on the strength of this answer.+ """+ never = threading.Thread(target = lambda: None)+ started = time.monotonic()+ assert join_when_started(never, timeout = 0.05) is False+ # It gives up at the deadline rather than waiting for a thread that will never run.+ assert time.monotonic() - started < 5+++def test_a_thread_that_runs_is_waited_for():+ done = threading.Event()++ def work():+ time.sleep(0.05)+ done.set()++ worker = threading.Thread(target = work)+ worker.start()+ assert join_when_started(worker, timeout = 5) is True+ assert done.is_set(), "it returned True while the work had not finished"+ assert not worker.is_alive()+++def test_a_thread_still_running_at_the_deadline_is_reported():+ """The half that must NOT be absorbed: callers assert on this to catch a leaked worker."""+ release = threading.Event()+ worker = threading.Thread(target = lambda: release.wait(timeout = 30))+ worker.start()+ try:+ assert join_when_started(worker, timeout = 0.05) is False+ finally:+ release.set()+ worker.join(timeout = 5)+++def test_joining_the_current_thread_is_refused_rather_than_retried():+ """join() raises RuntimeError for this too, and it is not a window that closes.++ Retrying it would spend the whole timeout and then report the caller as still alive, which+ is true and useless. Fail loudly instead.+ """+ with pytest.raises(RuntimeError, match = "current thread"):+ join_when_started(threading.current_thread(), timeout = 30)diff --git a/studio/backend/tests/thread_drain.py b/studio/backend/tests/thread_drain.pynew file mode 100644index 00000000000..8bfbc94a510--- /dev/null+++ b/studio/backend/tests/thread_drain.py@@ -0,0 +1,64 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Join a thread found through ``threading.enumerate()`` without racing its start.++``Thread.join()`` raises ``RuntimeError: cannot join thread before it is started`` when the+thread has not reached the point where CPython considers it running. ``threading.enumerate()``+returns threads in exactly that state: ``start()`` puts a thread in ``_limbo`` and it stays+enumerable-but-unjoinable until the interpreter schedules it. The window is short, which is why+a test that drains workers this way passes for months and then fails once on a loaded runner:++ tests/test_diffusion_backend.py:2509: in test_begin_load_rejects_concurrent+ thread.join(timeout = 5)+ RuntimeError: cannot join thread before it is started++Filtering does not close the window. ``Thread.name`` is set at construction, so a name match says+nothing about whether the thread has started. ``Thread.ident`` is assigned in ``_bootstrap_inner``+BEFORE the ``_started`` event that ``join()`` waits on, so an ident check can pass and the very+next line still raise. ``is_alive()`` is False both before a thread starts and after it finishes,+so skipping on it silently drops the worker the drain existed to wait for.++What is reliable is the exception itself: it means "not started YET", and a thread that has been+started always gets there. So retry until the thread joins or the caller's deadline passes.++Where a test owns the worker outright, recording it from inside the thread is better than+enumerating at all, since a thread running its own target is past the window by definition. This+helper is for the drains that legitimately cannot name their thread in advance.+"""++from __future__ import annotations++import threading+import time+++def join_when_started(thread: threading.Thread, timeout: float = 5.0) -> bool:+ """Join ``thread``, waiting out the not-started-yet window. True if it finished.++ Never raises for an unstarted thread: that is the condition this exists to absorb. It does+ not report one as drained either. A thread still in `_limbo` at the deadline has not run+ yet and still will, so True is reserved for a thread that actually finished.+ """+ # join() raises RuntimeError for two unrelated reasons, and only one of them is a window+ # that closes. Joining yourself never becomes possible, so retrying it would burn the whole+ # timeout and then report a thread that is trivially still alive.+ if thread is threading.current_thread():+ raise RuntimeError("cannot join current thread")+ deadline = time.monotonic() + timeout+ while True:+ remaining = deadline - time.monotonic()+ try:+ thread.join(timeout = max(remaining, 0.0))+ except RuntimeError:+ if remaining <= 0:+ # Still not started at the deadline, so it is UNDRAINED, and is_alive() must+ # not be consulted here: it is False before a thread starts exactly as it is+ # after one finishes, so asking it would report a worker that is still+ # scheduled and will run later as one that is already done. Callers restore+ # their monkeypatches on the strength of this answer, which is how that+ # worker would end up running against the next test.+ return False+ time.sleep(0.005)+ continue+ return not thread.is_alive()