Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Drain worker threads without racing their start (#11400)

Daniel Han655c1df61793
diff --git a/studio/backend/tests/test_mcp_http_sessions.py b/studio/backend/tests/test_mcp_http_sessions.py
index 2add86ac35d..3d59e97428f 100644
--- a/studio/backend/tests/test_mcp_http_sessions.py
+++ b/studio/backend/tests/test_mcp_http_sessions.py
@@ -20,6 +20,8 @@ from types import SimpleNamespace
import pytest
+from .thread_drain import join_when_started
+
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
@@ -982,7 +984,7 @@ def test_the_queue_of_pending_closes_is_bounded(monkeypatch, clients):
release.set()
for t in threading.enumerate():
if t.name.startswith("mcp-") and t is not threading.current_thread():
- t.join(5)
+ join_when_started(t, timeout = 5)
def test_closing_many_sessions_does_not_spawn_a_thread_each(monkeypatch, clients):
diff --git a/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py b/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py
index 96df0cc7fdf..efeac5c84a9 100644
--- a/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py
+++ b/studio/backend/tests/test_mlx_autorepair_no_torch_optout.py
@@ -24,6 +24,8 @@ from pathlib import Path
import pytest
+from .thread_drain import join_when_started
+
_BACKEND = Path(__file__).resolve().parent.parent
if str(_BACKEND) not in sys.path:
sys.path.insert(0, str(_BACKEND))
@@ -41,8 +43,7 @@ def _isolated(monkeypatch):
yield
for thread in threading.enumerate():
if thread.name == "mlx-autorepair":
- thread.join(timeout = 5)
- assert not thread.is_alive()
+ assert join_when_started(thread, timeout = 5)
def _install_manifest_returning(monkeypatch, value):
@@ -220,7 +221,7 @@ def test_a_normal_install_still_repairs(monkeypatch):
assert mr.start_mlx_autorepair_if_needed() is True
for thread in threading.enumerate():
if thread.name == "mlx-autorepair":
- thread.join(timeout = 5)
+ join_when_started(thread, timeout = 5)
assert attempts == [1]
@@ -236,7 +237,7 @@ def test_an_older_install_with_no_recorded_mode_still_repairs(monkeypatch):
assert mr.start_mlx_autorepair_if_needed() is True
for thread in threading.enumerate():
if thread.name == "mlx-autorepair":
- thread.join(timeout = 5)
+ join_when_started(thread, timeout = 5)
assert attempts == [1]
diff --git a/studio/backend/tests/test_mlx_repair.py b/studio/backend/tests/test_mlx_repair.py
index e74baa1d7ea..12e9eea9ae4 100644
--- a/studio/backend/tests/test_mlx_repair.py
+++ b/studio/backend/tests/test_mlx_repair.py
@@ -15,6 +15,8 @@ from pathlib import Path
import pytest
+from .thread_drain import join_when_started
+
_BACKEND = Path(__file__).resolve().parent.parent
if str(_BACKEND) not in sys.path:
sys.path.insert(0, str(_BACKEND))
@@ -45,8 +47,7 @@ def _reset_attempt_guard(monkeypatch):
# against the next test's globals.
for thread in threading.enumerate():
if thread.name == "mlx-autorepair":
- thread.join(timeout = 5)
- assert not thread.is_alive(), (
+ assert join_when_started(thread, timeout = 5), (
"an mlx-autorepair worker outlived its test; once these stubs are "
"restored it runs the real repair and detection against another test"
)
@@ -402,7 +403,7 @@ def test_apple_silicon_missing_mlx_starts_repair_and_redetects(monkeypatch):
# Join the daemon thread deterministically.
for thread in threading.enumerate():
if thread.name == "mlx-autorepair":
- thread.join(timeout = 5)
+ join_when_started(thread, timeout = 5)
assert repaired["called"] is True
assert redetected["called"] is True
@@ -596,8 +597,7 @@ def _recorded_announcements(monkeypatch):
def _join_the_repair_worker():
for thread in threading.enumerate():
if thread.name == "mlx-autorepair":
- thread.join(timeout = 5)
- assert not thread.is_alive()
+ assert join_when_started(thread, timeout = 5)
def test_a_stack_that_measures_usable_overturns_the_verdict(monkeypatch):
diff --git a/studio/backend/tests/test_sandbox_files_and_storage_roots.py b/studio/backend/tests/test_sandbox_files_and_storage_roots.py
index 027b1b3f8b0..1650364f008 100644
--- a/studio/backend/tests/test_sandbox_files_and_storage_roots.py
+++ b/studio/backend/tests/test_sandbox_files_and_storage_roots.py
@@ -25,6 +25,8 @@ from pathlib import Path
import pytest
+from .thread_drain import join_when_started
+
def _shared_setup_1(monkeypatch, tmp_path):
monkeypatch.setenv("UNSLOTH_STUDIO_SANDBOX_HOME", str(tmp_path / "sb"))
@@ -346,7 +348,7 @@ def test_legacy_sandbox_is_migrated(tmp_path, monkeypatch):
# waits on the whole tree.
for thread in threading.enumerate():
if thread.name == "sandbox-migrate":
- thread.join(30)
+ join_when_started(thread, timeout = 30)
moved = wd.parent / "__LOCALID_old1234" / "results.csv"
print(f"\nmigrated to {moved}")
assert moved.is_file()
diff --git a/studio/backend/tests/test_thread_drain.py b/studio/backend/tests/test_thread_drain.py
new file mode 100644
index 00000000000..1bd34a72879
--- /dev/null
+++ b/studio/backend/tests/test_thread_drain.py
@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""`join_when_started` absorbs the not-started-yet window and nothing else.
+
+The window is real but tiny, so it cannot be hit on demand by starting threads and hoping. It
+is reproduced here directly instead: a thread that has never been started is in exactly the
+state `threading.enumerate()` can hand a drain, and `join()` on it raises the same
+`RuntimeError` for the same reason.
+
+What has to stay true is both halves. The helper must not raise on an unstarted thread, which
+is the bug it exists for, and it must still report a thread that is genuinely still running,
+which is what every caller asserts on. A helper that swallowed both would turn a hung worker
+into a green test.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+
+import pytest
+
+from .thread_drain import join_when_started
+
+
+def test_join_raises_on_its_own_for_a_thread_that_never_started():
+ """The premise. Without this the first test below could pass for the wrong reason."""
+ never = threading.Thread(target = lambda: None)
+ with pytest.raises(RuntimeError, match = "before it is started"):
+ never.join(timeout = 0.01)
+
+
+def test_an_unstarted_thread_does_not_raise_and_is_not_called_drained():
+ """Absorbing the exception must not turn into claiming the thread is done.
+
+ A thread that `join()` still refuses at the deadline has not run yet and, if it came out
+ of `threading.enumerate()`, still will. `is_alive()` cannot tell that apart from finished,
+ since it is False on both sides, so the answer here is taken from `join()` rather than
+ from `is_alive()`. Callers restore their monkeypatches on the strength of this answer.
+ """
+ never = threading.Thread(target = lambda: None)
+ started = time.monotonic()
+ assert join_when_started(never, timeout = 0.05) is False
+ # It gives up at the deadline rather than waiting for a thread that will never run.
+ assert time.monotonic() - started < 5
+
+
+def test_a_thread_that_runs_is_waited_for():
+ done = threading.Event()
+
+ def work():
+ time.sleep(0.05)
+ done.set()
+
+ worker = threading.Thread(target = work)
+ worker.start()
+ assert join_when_started(worker, timeout = 5) is True
+ assert done.is_set(), "it returned True while the work had not finished"
+ assert not worker.is_alive()
+
+
+def test_a_thread_still_running_at_the_deadline_is_reported():
+ """The half that must NOT be absorbed: callers assert on this to catch a leaked worker."""
+ release = threading.Event()
+ worker = threading.Thread(target = lambda: release.wait(timeout = 30))
+ worker.start()
+ try:
+ assert join_when_started(worker, timeout = 0.05) is False
+ finally:
+ release.set()
+ worker.join(timeout = 5)
+
+
+def test_joining_the_current_thread_is_refused_rather_than_retried():
+ """join() raises RuntimeError for this too, and it is not a window that closes.
+
+ Retrying it would spend the whole timeout and then report the caller as still alive, which
+ is true and useless. Fail loudly instead.
+ """
+ with pytest.raises(RuntimeError, match = "current thread"):
+ join_when_started(threading.current_thread(), timeout = 30)
diff --git a/studio/backend/tests/thread_drain.py b/studio/backend/tests/thread_drain.py
new file mode 100644
index 00000000000..8bfbc94a510
--- /dev/null
+++ b/studio/backend/tests/thread_drain.py
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Join a thread found through ``threading.enumerate()`` without racing its start.
+
+``Thread.join()`` raises ``RuntimeError: cannot join thread before it is started`` when the
+thread has not reached the point where CPython considers it running. ``threading.enumerate()``
+returns threads in exactly that state: ``start()`` puts a thread in ``_limbo`` and it stays
+enumerable-but-unjoinable until the interpreter schedules it. The window is short, which is why
+a test that drains workers this way passes for months and then fails once on a loaded runner:
+
+ tests/test_diffusion_backend.py:2509: in test_begin_load_rejects_concurrent
+ thread.join(timeout = 5)
+ RuntimeError: cannot join thread before it is started
+
+Filtering does not close the window. ``Thread.name`` is set at construction, so a name match says
+nothing about whether the thread has started. ``Thread.ident`` is assigned in ``_bootstrap_inner``
+BEFORE the ``_started`` event that ``join()`` waits on, so an ident check can pass and the very
+next line still raise. ``is_alive()`` is False both before a thread starts and after it finishes,
+so skipping on it silently drops the worker the drain existed to wait for.
+
+What is reliable is the exception itself: it means "not started YET", and a thread that has been
+started always gets there. So retry until the thread joins or the caller's deadline passes.
+
+Where a test owns the worker outright, recording it from inside the thread is better than
+enumerating at all, since a thread running its own target is past the window by definition. This
+helper is for the drains that legitimately cannot name their thread in advance.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+
+
+def join_when_started(thread: threading.Thread, timeout: float = 5.0) -> bool:
+ """Join ``thread``, waiting out the not-started-yet window. True if it finished.
+
+ Never raises for an unstarted thread: that is the condition this exists to absorb. It does
+ not report one as drained either. A thread still in `_limbo` at the deadline has not run
+ yet and still will, so True is reserved for a thread that actually finished.
+ """
+ # join() raises RuntimeError for two unrelated reasons, and only one of them is a window
+ # that closes. Joining yourself never becomes possible, so retrying it would burn the whole
+ # timeout and then report a thread that is trivially still alive.
+ if thread is threading.current_thread():
+ raise RuntimeError("cannot join current thread")
+ deadline = time.monotonic() + timeout
+ while True:
+ remaining = deadline - time.monotonic()
+ try:
+ thread.join(timeout = max(remaining, 0.0))
+ except RuntimeError:
+ if remaining <= 0:
+ # Still not started at the deadline, so it is UNDRAINED, and is_alive() must
+ # not be consulted here: it is False before a thread starts exactly as it is
+ # after one finishes, so asking it would report a worker that is still
+ # scheduled and will run later as one that is already done. Callers restore
+ # their monkeypatches on the strength of this answer, which is how that
+ # worker would end up running against the next test.
+ return False
+ time.sleep(0.005)
+ continue
+ return not thread.is_alive()