Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studio: dump the backend's own thread stacks while a stall is in progress (#9715)
he pangcc5a85b84421
diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.ymlindex 73123922ae5..01a0df2bd3e 100644--- a/.github/workflows/studio-mac-ui-smoke.yml+++ b/.github/workflows/studio-mac-ui-smoke.yml@@ -165,6 +165,8 @@ jobs:GGUF_VARIANT: UD-Q4_K_XLGGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.ggufSTUDIO_PORT: '18896'+ # Dump thread stacks on a backend stall (#9712).+ UNSLOTH_STUDIO_STALL_WATCHDOG: '1'HF_HOME: ${{ github.workspace }}/hf-cache# GitHub's macOS runners expose a PARAVIRTUAL Metal device, and Unsloth refuses to# offload to it because paravirtual Apple GPUs return corrupt output. The absorbeddiff --git a/studio/backend/main.py b/studio/backend/main.pyindex c93c754f839..2cefc5586f8 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -894,6 +894,12 @@ async def lifespan(app: FastAPI):from core.inference.key_exchange import init_key_pairinit_key_pair()++ # Stall thread-dump watchdog (#9712), only with UNSLOTH_STUDIO_STALL_WATCHDOG=1.+ from utils.stall_watchdog import stand_down_for_the_warm, start_stall_watchdog++ start_stall_watchdog(asyncio.get_running_loop(), suppress = stand_down_for_the_warm)+_lifespan_log.info("lifespan pre-auth setup completed in %.1fms",(_time.perf_counter() - _lifespan_started) * 1000,@@ -956,6 +962,11 @@ async def lifespan(app: FastAPI):# Before any shutdown await: a warm finishing during one would still read the lifespan as current._stop_post_warm_thread()+ # Before teardown blocks the loop, or shutdown dumps as a stall.+ from utils.stall_watchdog import stop_stall_watchdog++ stop_stall_watchdog()+# Retire the coordinated warm at shutdown entry too. run_lifespan_shutdown() repeats this after# cleanup, but its awaits would otherwise let startup imports continue for a stopped lifespan._invalidate_detection = getattr(_hw_module, "invalidate_detection", None)diff --git a/studio/backend/tests/test_stall_watchdog.py b/studio/backend/tests/test_stall_watchdog.pynew file mode 100644index 00000000000..08f22efe260--- /dev/null+++ b/studio/backend/tests/test_stall_watchdog.py@@ -0,0 +1,340 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A stalled event loop gets its thread stacks dumped while still stalled (#9712).++The GIL-held cases hold the GIL for real (busy loop under a long switch interval), since+faulthandler's C thread firing without the GIL is what the design rests on. CPU-only.+"""++from __future__ import annotations++import asyncio+import sys+import threading+import time+from pathlib import Path++import pytest++from utils.stall_watchdog import (+ StallWatchdog,+ stand_down_for_the_warm,+ start_stall_watchdog,+ stop_stall_watchdog,+)++_BACKEND_DIR = Path(__file__).resolve().parent.parent+++def loop_in_thread():+ loop = asyncio.new_event_loop()+ thread = threading.Thread(target = loop.run_forever, daemon = True, name = "test-loop")+ thread.start()+ yield loop+ loop.call_soon_threadsafe(loop.stop)+ thread.join(timeout = 5)+ # Drain queued probes, else never-awaited coroutine warnings.+ loop.run_until_complete(asyncio.sleep(0.1))+ loop.close()+++def _wait_for(predicate, timeout_s: float = 10.0) -> bool:+ deadline = time.monotonic() + timeout_s+ while time.monotonic() < deadline:+ if predicate():+ return True+ time.sleep(0.02)+ return predicate()+++def test_a_blocked_loop_dumps_after_consecutive_slow_probes(loop_in_thread, tmp_path):+ """The GIL-free shape: something blocking sits on the loop, Python threads still+ run, so the slow-probe counter reaches its threshold and dumps. The dump must+ name the blocking frame, otherwise it answers nothing."""+ dump_path = tmp_path / "dump.txt"++ def blocked_in_a_syscall():+ time.sleep(1.2)++ with dump_path.open("w") as dump_file:+ watchdog = StallWatchdog(+ loop_in_thread,+ dump_file = dump_file,+ beat_interval_s = 0.05,+ probe_slow_s = 0.05,+ slow_probes_before_dump = 3,+ dead_man_timeout_s = 30.0,+ dump_cooldown_s = 0.0,+ )+ watchdog.start()+ try:+ loop_in_thread.call_soon_threadsafe(blocked_in_a_syscall)+ assert _wait_for(+ lambda: "stall watchdog:" in dump_path.read_text(errors = "replace")+ ), "no dump was written while the loop was blocked"+ finally:+ watchdog.stop()++ text = dump_path.read_text(errors = "replace")+ assert "consecutive slow probes" in text+ assert (+ "blocked_in_a_syscall" in text+ ), "the dump does not show the frame the loop is stuck in; it cannot answer what #9712 asks"+++def test_a_gil_holding_stall_is_dumped_mid_stall(loop_in_thread, tmp_path):+ """The #9712 shape: the GIL is held, so the watchdog thread itself is frozen and+ slow-probe counting observes nothing until the stall is over. Only the dead+ man's switch can capture this, firing from C while no Python thread runs."""+ dump_path = tmp_path / "dump.txt"+ released = threading.Event()++ def holds_the_gil():+ old = sys.getswitchinterval()+ sys.setswitchinterval(10.0)+ try:+ deadline = time.monotonic() + 1.2+ while time.monotonic() < deadline:+ pass+ finally:+ sys.setswitchinterval(old)+ released.set()++ with dump_path.open("w") as dump_file:+ watchdog = StallWatchdog(+ loop_in_thread,+ dump_file = dump_file,+ beat_interval_s = 0.05,+ probe_slow_s = 0.05,+ slow_probes_before_dump = 1000, # the Python path must not be the one firing+ dead_man_timeout_s = 0.4,+ dump_cooldown_s = 0.0,+ )+ watchdog.start()+ try:+ assert _wait_for(lambda: watchdog._dead_man_armed, timeout_s = 5.0)+ loop_in_thread.call_soon_threadsafe(holds_the_gil)+ assert released.wait(timeout = 30.0)+ assert _wait_for(lambda: "Timeout" in dump_path.read_text(errors = "replace"))+ finally:+ watchdog.stop()++ text = dump_path.read_text(errors = "replace")+ assert "holds_the_gil" in text, (+ "the mid-stall dump does not show the GIL holder's frame; identifying it is "+ "the point of the dead man's switch"+ )+++def test_repeated_gil_stalls_dump_once_per_cooldown(loop_in_thread, tmp_path):+ """The switch fires on its own once armed, so the cooldown has to gate the+ arming: a host that stalls chronically must leave one dump, and silence, until+ the window passes."""+ dump_path = tmp_path / "dump.txt"++ def hold_gil_for(seconds: float):+ released = threading.Event()++ def holder():+ old = sys.getswitchinterval()+ sys.setswitchinterval(10.0)+ try:+ deadline = time.monotonic() + seconds+ while time.monotonic() < deadline:+ pass+ finally:+ sys.setswitchinterval(old)+ released.set()++ loop_in_thread.call_soon_threadsafe(holder)+ assert released.wait(timeout = 30.0)++ with dump_path.open("w") as dump_file:+ watchdog = StallWatchdog(+ loop_in_thread,+ dump_file = dump_file,+ beat_interval_s = 0.05,+ probe_slow_s = 0.05,+ slow_probes_before_dump = 1000,+ dead_man_timeout_s = 0.3,+ dump_cooldown_s = 60.0,+ )+ watchdog.start()+ try:+ assert _wait_for(lambda: watchdog._dead_man_armed, timeout_s = 5.0)+ hold_gil_for(0.8)+ assert _wait_for(lambda: "Timeout" in dump_path.read_text(errors = "replace"))+ assert _wait_for(lambda: watchdog._last_dump is not None, timeout_s = 5.0)+ hold_gil_for(0.8)+ time.sleep(0.3)+ finally:+ watchdog.stop()++ text = dump_path.read_text(errors = "replace")+ assert text.count("Timeout") == 1, (+ "a second stall inside the cooldown window dumped again; a chronically "+ "stalling host would fill its log with stacks"+ )+++def test_a_python_dump_disarms_the_switch_for_the_cooldown(loop_in_thread, tmp_path):+ """The two paths share the cooldown. A slow-probe dump leaves the switch armed+ from earlier in the same beat unless it is taken down with it, and a GIL freeze+ right after would land a second dump inside the window."""+ dump_path = tmp_path / "dump.txt"++ def blocks_the_loop():+ time.sleep(0.5)++ def holds_the_gil():+ old = sys.getswitchinterval()+ sys.setswitchinterval(10.0)+ try:+ deadline = time.monotonic() + 0.8+ while time.monotonic() < deadline:+ pass+ finally:+ sys.setswitchinterval(old)++ with dump_path.open("w") as dump_file:+ watchdog = StallWatchdog(+ loop_in_thread,+ dump_file = dump_file,+ beat_interval_s = 0.05,+ probe_slow_s = 0.05,+ slow_probes_before_dump = 2,+ dead_man_timeout_s = 0.3,+ dump_cooldown_s = 60.0,+ )+ watchdog.start()+ try:+ loop_in_thread.call_soon_threadsafe(blocks_the_loop)+ assert _wait_for(lambda: "stall watchdog:" in dump_path.read_text(errors = "replace"))+ loop_in_thread.call_soon_threadsafe(holds_the_gil)+ time.sleep(1.4)+ finally:+ watchdog.stop()++ text = dump_path.read_text(errors = "replace")+ assert "Timeout" not in text, (+ "the dead man's switch fired inside the cooldown a slow-probe dump started; "+ "one stall episode left two dumps"+ )+++def test_stands_down_until_the_warm_is_over(monkeypatch):+ """The dead man's switch cannot be disarmed once the warm holds the GIL, so the+ gate has to be closed before the warm starts, and stay closed until it is over.+ Gating on 'warm running' left the window between the watchdog's first beat and+ start_background_warm(), and every cold start dumped over the torch import."""+ import utils.stall_watchdog as sw+ import utils.torch_warmup as tw++ def status(started: bool, finished: bool, alive: bool):+ monkeypatch.setattr(+ tw,+ "warm_status",+ lambda: {"started": started, "finished": finished, "alive": alive, "stages": {}},+ )++ monkeypatch.delenv(tw.DISABLE_ENV_VAR, raising = False)+ status(started = False, finished = False, alive = False)+ assert sw.stand_down_for_the_warm() is True, (+ "not standing down before the warm starts is the arming race: the switch "+ "armed in that window dumps over `import torch` on every cold start"+ )+ status(started = True, finished = False, alive = True)+ assert sw.stand_down_for_the_warm() is True+ status(started = True, finished = True, alive = False)+ assert sw.stand_down_for_the_warm() is False+ # A warm that died mid-stage is not coming back.+ status(started = True, finished = False, alive = False)+ assert sw.stand_down_for_the_warm() is False++ monkeypatch.setenv(tw.DISABLE_ENV_VAR, "1")+ status(started = False, finished = False, alive = False)+ assert sw.stand_down_for_the_warm() is False, (+ "with the warm switched off no warm is coming; standing down anyway keeps "+ "the watchdog disarmed for the whole session"+ )+++def test_no_dump_while_suppressed(loop_in_thread, tmp_path):+ """The warm's `import torch` is a legitimate long GIL hold with a known frame.+ While the suppress callable says so, neither capture path may fire."""+ dump_path = tmp_path / "dump.txt"++ with dump_path.open("w") as dump_file:+ watchdog = StallWatchdog(+ loop_in_thread,+ suppress = lambda: True,+ dump_file = dump_file,+ beat_interval_s = 0.05,+ probe_slow_s = 0.05,+ slow_probes_before_dump = 1,+ dead_man_timeout_s = 0.2,+ dump_cooldown_s = 0.0,+ )+ watchdog.start()+ try:+ loop_in_thread.call_soon_threadsafe(time.sleep, 0.8)+ time.sleep(1.2)+ finally:+ watchdog.stop()++ assert dump_path.read_text(errors = "replace") == "", (+ "the watchdog dumped during a suppressed window; every warm import would "+ "land a spurious dump in the log"+ )+++def test_off_unless_the_env_opts_in(loop_in_thread, monkeypatch):+ """Diagnostic tooling: a desktop run must carry neither the thread nor the+ faulthandler timer, so nothing starts without the env the CI workflow sets."""+ monkeypatch.delenv("UNSLOTH_STUDIO_STALL_WATCHDOG", raising = False)+ assert start_stall_watchdog(loop_in_thread) is None+ assert not any(t.name == "stall-watchdog" for t in threading.enumerate())+++def test_start_and_stop_manage_one_process_wide_instance(loop_in_thread, monkeypatch):+ monkeypatch.setenv("UNSLOTH_STUDIO_STALL_WATCHDOG", "1")+ first = start_stall_watchdog(loop_in_thread)+ try:+ assert first is not None+ second = start_stall_watchdog(loop_in_thread)+ assert second is not None+ assert _wait_for(+ lambda: not (first._thread and first._thread.is_alive()), timeout_s = 5.0+ ), "starting a second watchdog left the first one's thread running"+ finally:+ stop_stall_watchdog()+ assert _wait_for(+ lambda: not any(t.name == "stall-watchdog" for t in threading.enumerate()),+ timeout_s = 5.0,+ )+++def test_the_lifespan_wires_it_in():+ """Cross-file guard: the module only does anything because main.py starts it at+ startup and retires it at shutdown entry, and either side can be edited alone."""+ source = (_BACKEND_DIR / "main.py").read_text(encoding = "utf-8")+ assert (+ "start_stall_watchdog(asyncio.get_running_loop()" in source+ ), "main.py no longer starts the stall watchdog; the next #9712 stall leaves no dump behind"+ assert "suppress = stand_down_for_the_warm" in source, (+ "the watchdog no longer stands down for the warm; every cold start would "+ "dump over the torch import"+ )+ assert "stop_stall_watchdog()" in source, (+ "main.py never retires the watchdog, so shutdown's deliberate blocking "+ "reads as a stall and dumps into the exit log"+ )++ workflow = _BACKEND_DIR.parent.parent / ".github" / "workflows" / "studio-mac-ui-smoke.yml"+ assert workflow.is_file(), f"{workflow} moved; update this guard"+ assert "UNSLOTH_STUDIO_STALL_WATCHDOG" in workflow.read_text(encoding = "utf-8"), (+ "the watchdog is opt-in and the mac smoke workflow no longer opts in; the "+ "one place #9712 stalls have been seen runs without it"+ )diff --git a/studio/backend/utils/stall_watchdog.py b/studio/backend/utils/stall_watchdog.pynew file mode 100644index 00000000000..c96d3beb2b6--- /dev/null+++ b/studio/backend/utils/stall_watchdog.py@@ -0,0 +1,246 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Dump every thread's stack from inside the backend while its event loop is stalled (#9712).++Two capture paths. Loop stuck with the GIL free: the watchdog thread counts consecutive slow+no-op probes and dumps from Python. GIL held: no Python thread runs, so each beat re-arms+``faulthandler.dump_traceback_later`` as a dead man's switch; its C timer thread dumps without+the GIL. Off unless UNSLOTH_STUDIO_STALL_WATCHDOG=1. Dumps go to the stderr fd (below run.py's+tee). Assumes it is the only user of faulthandler's process-global delayed-dump timer.+"""++from __future__ import annotations++import asyncio+import faulthandler+import os+import threading+import time+from concurrent.futures import TimeoutError as FutureTimeoutError+from typing import Callable, Optional, TextIO++import structlog++logger = structlog.get_logger(__name__)++ENABLE_ENV_VAR = "UNSLOTH_STUDIO_STALL_WATCHDOG"++BEAT_INTERVAL_S = 2.5+# Healthy mac smoke runs: ~50ms worst latency, single-probe outliers to ~3.4s.+PROBE_SLOW_S = 1.0+# 3 slow beats = 6-8.5s unresponsive: dumps before the shortest recorded stall (10.03s) ends.+SLOW_PROBES_BEFORE_DUMP = 3+DEAD_MAN_TIMEOUT_S = 8.0+# Shared by both capture paths.+DUMP_COOLDOWN_S = 600.0+++def stand_down_for_the_warm() -> bool:+ """True from process start until the warm is over: a switch armed before the warm's+ GIL-holding ``import torch`` cannot be disarmed and would dump a known stall."""+ from utils.torch_warmup import DISABLE_ENV_VAR as _WARM_DISABLED+ from utils.torch_warmup import warm_status++ status = warm_status()+ if status["started"]:+ return bool(status["alive"] and not status["finished"])+ return os.environ.get(_WARM_DISABLED) != "1"+++async def _noop() -> None:+ return None+++class StallWatchdog:+ """One daemon thread beating against the event loop. start() / stop()."""++ def __init__(+ self,+ loop: asyncio.AbstractEventLoop,+ *,+ suppress: Optional[Callable[[], bool]] = None,+ dump_file: Optional[TextIO] = None,+ beat_interval_s: float = BEAT_INTERVAL_S,+ probe_slow_s: float = PROBE_SLOW_S,+ slow_probes_before_dump: int = SLOW_PROBES_BEFORE_DUMP,+ dead_man_timeout_s: float = DEAD_MAN_TIMEOUT_S,+ dump_cooldown_s: float = DUMP_COOLDOWN_S,+ ) -> None:+ import sys++ self._loop = loop+ self._suppress = suppress or (lambda: False)+ self._dump_file = dump_file if dump_file is not None else sys.stderr+ self._beat_interval_s = beat_interval_s+ self._probe_slow_s = probe_slow_s+ self._slow_probes_before_dump = slow_probes_before_dump+ self._dead_man_timeout_s = dead_man_timeout_s+ self._dump_cooldown_s = dump_cooldown_s++ self._stop_event = threading.Event()+ self._thread: Optional[threading.Thread] = None+ self._slow_streak = 0+ self._stall_started: Optional[float] = None+ self._last_dump: Optional[float] = None+ self._dead_man_armed = False+ self._arm_failure_logged = False++ def start(self) -> None:+ self._thread = threading.Thread(+ target = self._run,+ daemon = True,+ name = "stall-watchdog",+ )+ self._thread.start()++ def stop(self) -> None:+ self._stop_event.set()+ self._cancel_dead_man()+ thread = self._thread+ if thread is not None:+ thread.join(timeout = 2.0)++ def _run(self) -> None:+ last_beat = time.monotonic()+ while not self._stop_event.is_set():+ beat_started = time.monotonic()+ # Beat gap past the switch timeout: the switch fired; start the cooldown.+ gap = beat_started - last_beat+ if self._dead_man_armed and gap > self._dead_man_timeout_s:+ self._last_dump = beat_started+ self._dead_man_armed = False+ logger.warning(+ "stall watchdog was itself blocked for %.1fs; if the GIL was held, "+ "faulthandler wrote a thread dump to stderr (system sleep also lands here)",+ gap,+ )+ last_beat = beat_started++ if self._suppress_now():+ self._stop_event.wait(min(self._beat_interval_s, 0.5))+ continue++ self._arm_dead_man()+ self._probe(beat_started)++ elapsed = time.monotonic() - beat_started+ self._stop_event.wait(max(0.0, self._beat_interval_s - elapsed))+ self._cancel_dead_man()++ def _suppress_now(self) -> bool:+ try:+ suppressed = bool(self._suppress())+ except Exception:+ suppressed = False+ if suppressed:+ self._cancel_dead_man()+ self._slow_streak = 0+ self._stall_started = None+ return suppressed++ def _probe(self, beat_started: float) -> None:+ try:+ future = asyncio.run_coroutine_threadsafe(_noop(), self._loop)+ except RuntimeError:+ self._stop_event.wait(self._beat_interval_s)+ return+ try:+ future.result(timeout = self._probe_slow_s)+ except FutureTimeoutError:+ # Not cancelled: that leaves a never-awaited coroutine warning.+ self._slow_streak += 1+ if self._stall_started is None:+ self._stall_started = beat_started+ if self._slow_streak >= self._slow_probes_before_dump:+ self._dump_from_python()+ return+ except Exception:+ pass+ if self._slow_streak:+ stalled_for = time.monotonic() - (self._stall_started or beat_started)+ logger.warning(+ "event loop answering again after %.1fs (%d consecutive slow probes)",+ stalled_for,+ self._slow_streak,+ )+ self._slow_streak = 0+ self._stall_started = None++ def _in_cooldown(self) -> bool:+ return (+ self._last_dump is not None+ and time.monotonic() - self._last_dump < self._dump_cooldown_s+ )++ def _arm_dead_man(self) -> None:+ if self._in_cooldown():+ self._cancel_dead_man()+ return+ try:+ faulthandler.dump_traceback_later(+ self._dead_man_timeout_s,+ repeat = False,+ file = self._dump_file,+ exit = False,+ )+ self._dead_man_armed = True+ except Exception as exc:+ self._dead_man_armed = False+ if not self._arm_failure_logged:+ self._arm_failure_logged = True+ logger.warning(+ "stall watchdog cannot arm faulthandler (%s); GIL-held stalls "+ "will not be dumped, only slow-probe ones",+ exc,+ )++ def _cancel_dead_man(self) -> None:+ if self._dead_man_armed:+ faulthandler.cancel_dump_traceback_later()+ self._dead_man_armed = False++ def _dump_from_python(self) -> None:+ if self._in_cooldown():+ return+ now = time.monotonic()+ self._last_dump = now+ # Else this beat's switch could fire a second dump inside the cooldown.+ self._cancel_dead_man()+ stalled_for = now - (self._stall_started or now)+ try:+ self._dump_file.write(+ f"\nstall watchdog: event loop unresponsive for {stalled_for:.1f}s "+ f"({self._slow_streak} consecutive slow probes), dumping all threads\n"+ )+ self._dump_file.flush()+ faulthandler.dump_traceback(file = self._dump_file, all_threads = True)+ except Exception as exc:+ logger.warning("stall watchdog could not write a thread dump: %s", exc)+++_watchdog: Optional[StallWatchdog] = None+_watchdog_lock = threading.Lock()+++def start_stall_watchdog(+ loop: asyncio.AbstractEventLoop, *, suppress: Optional[Callable[[], bool]] = None+) -> Optional[StallWatchdog]:+ """Start the process-wide watchdog. Returns None unless the env opts in."""+ if os.environ.get(ENABLE_ENV_VAR) != "1":+ return None+ global _watchdog+ with _watchdog_lock:+ if _watchdog is not None:+ _watchdog.stop()+ _watchdog = StallWatchdog(loop, suppress = suppress)+ _watchdog.start()+ return _watchdog+++def stop_stall_watchdog() -> None:+ global _watchdog+ with _watchdog_lock:+ if _watchdog is not None:+ _watchdog.stop()+ _watchdog = None