Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studio: dump the backend's own thread stacks while a stall is in progress (#9715)

he pangcc5a85b84421
diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml
index 73123922ae5..01a0df2bd3e 100644
--- a/.github/workflows/studio-mac-ui-smoke.yml
+++ b/.github/workflows/studio-mac-ui-smoke.yml
@@ -165,6 +165,8 @@ jobs:
GGUF_VARIANT: UD-Q4_K_XL
GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
STUDIO_PORT: '18896'
+ # Dump thread stacks on a backend stall (#9712).
+ UNSLOTH_STUDIO_STALL_WATCHDOG: '1'
HF_HOME: ${{ github.workspace }}/hf-cache
# GitHub's macOS runners expose a PARAVIRTUAL Metal device, and Unsloth refuses to
# offload to it because paravirtual Apple GPUs return corrupt output. The absorbed
diff --git a/studio/backend/main.py b/studio/backend/main.py
index c93c754f839..2cefc5586f8 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -894,6 +894,12 @@ async def lifespan(app: FastAPI):
from core.inference.key_exchange import init_key_pair
init_key_pair()
+
+ # Stall thread-dump watchdog (#9712), only with UNSLOTH_STUDIO_STALL_WATCHDOG=1.
+ from utils.stall_watchdog import stand_down_for_the_warm, start_stall_watchdog
+
+ start_stall_watchdog(asyncio.get_running_loop(), suppress = stand_down_for_the_warm)
+
_lifespan_log.info(
"lifespan pre-auth setup completed in %.1fms",
(_time.perf_counter() - _lifespan_started) * 1000,
@@ -956,6 +962,11 @@ async def lifespan(app: FastAPI):
# Before any shutdown await: a warm finishing during one would still read the lifespan as current.
_stop_post_warm_thread()
+ # Before teardown blocks the loop, or shutdown dumps as a stall.
+ from utils.stall_watchdog import stop_stall_watchdog
+
+ stop_stall_watchdog()
+
# Retire the coordinated warm at shutdown entry too. run_lifespan_shutdown() repeats this after
# cleanup, but its awaits would otherwise let startup imports continue for a stopped lifespan.
_invalidate_detection = getattr(_hw_module, "invalidate_detection", None)
diff --git a/studio/backend/tests/test_stall_watchdog.py b/studio/backend/tests/test_stall_watchdog.py
new file mode 100644
index 00000000000..08f22efe260
--- /dev/null
+++ b/studio/backend/tests/test_stall_watchdog.py
@@ -0,0 +1,340 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A stalled event loop gets its thread stacks dumped while still stalled (#9712).
+
+The GIL-held cases hold the GIL for real (busy loop under a long switch interval), since
+faulthandler's C thread firing without the GIL is what the design rests on. CPU-only.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import sys
+import threading
+import time
+from pathlib import Path
+
+import pytest
+
+from utils.stall_watchdog import (
+ StallWatchdog,
+ stand_down_for_the_warm,
+ start_stall_watchdog,
+ stop_stall_watchdog,
+)
+
+_BACKEND_DIR = Path(__file__).resolve().parent.parent
+
+
+def loop_in_thread():
+ loop = asyncio.new_event_loop()
+ thread = threading.Thread(target = loop.run_forever, daemon = True, name = "test-loop")
+ thread.start()
+ yield loop
+ loop.call_soon_threadsafe(loop.stop)
+ thread.join(timeout = 5)
+ # Drain queued probes, else never-awaited coroutine warnings.
+ loop.run_until_complete(asyncio.sleep(0.1))
+ loop.close()
+
+
+def _wait_for(predicate, timeout_s: float = 10.0) -> bool:
+ deadline = time.monotonic() + timeout_s
+ while time.monotonic() < deadline:
+ if predicate():
+ return True
+ time.sleep(0.02)
+ return predicate()
+
+
+def test_a_blocked_loop_dumps_after_consecutive_slow_probes(loop_in_thread, tmp_path):
+ """The GIL-free shape: something blocking sits on the loop, Python threads still
+ run, so the slow-probe counter reaches its threshold and dumps. The dump must
+ name the blocking frame, otherwise it answers nothing."""
+ dump_path = tmp_path / "dump.txt"
+
+ def blocked_in_a_syscall():
+ time.sleep(1.2)
+
+ with dump_path.open("w") as dump_file:
+ watchdog = StallWatchdog(
+ loop_in_thread,
+ dump_file = dump_file,
+ beat_interval_s = 0.05,
+ probe_slow_s = 0.05,
+ slow_probes_before_dump = 3,
+ dead_man_timeout_s = 30.0,
+ dump_cooldown_s = 0.0,
+ )
+ watchdog.start()
+ try:
+ loop_in_thread.call_soon_threadsafe(blocked_in_a_syscall)
+ assert _wait_for(
+ lambda: "stall watchdog:" in dump_path.read_text(errors = "replace")
+ ), "no dump was written while the loop was blocked"
+ finally:
+ watchdog.stop()
+
+ text = dump_path.read_text(errors = "replace")
+ assert "consecutive slow probes" in text
+ assert (
+ "blocked_in_a_syscall" in text
+ ), "the dump does not show the frame the loop is stuck in; it cannot answer what #9712 asks"
+
+
+def test_a_gil_holding_stall_is_dumped_mid_stall(loop_in_thread, tmp_path):
+ """The #9712 shape: the GIL is held, so the watchdog thread itself is frozen and
+ slow-probe counting observes nothing until the stall is over. Only the dead
+ man's switch can capture this, firing from C while no Python thread runs."""
+ dump_path = tmp_path / "dump.txt"
+ released = threading.Event()
+
+ def holds_the_gil():
+ old = sys.getswitchinterval()
+ sys.setswitchinterval(10.0)
+ try:
+ deadline = time.monotonic() + 1.2
+ while time.monotonic() < deadline:
+ pass
+ finally:
+ sys.setswitchinterval(old)
+ released.set()
+
+ with dump_path.open("w") as dump_file:
+ watchdog = StallWatchdog(
+ loop_in_thread,
+ dump_file = dump_file,
+ beat_interval_s = 0.05,
+ probe_slow_s = 0.05,
+ slow_probes_before_dump = 1000, # the Python path must not be the one firing
+ dead_man_timeout_s = 0.4,
+ dump_cooldown_s = 0.0,
+ )
+ watchdog.start()
+ try:
+ assert _wait_for(lambda: watchdog._dead_man_armed, timeout_s = 5.0)
+ loop_in_thread.call_soon_threadsafe(holds_the_gil)
+ assert released.wait(timeout = 30.0)
+ assert _wait_for(lambda: "Timeout" in dump_path.read_text(errors = "replace"))
+ finally:
+ watchdog.stop()
+
+ text = dump_path.read_text(errors = "replace")
+ assert "holds_the_gil" in text, (
+ "the mid-stall dump does not show the GIL holder's frame; identifying it is "
+ "the point of the dead man's switch"
+ )
+
+
+def test_repeated_gil_stalls_dump_once_per_cooldown(loop_in_thread, tmp_path):
+ """The switch fires on its own once armed, so the cooldown has to gate the
+ arming: a host that stalls chronically must leave one dump, and silence, until
+ the window passes."""
+ dump_path = tmp_path / "dump.txt"
+
+ def hold_gil_for(seconds: float):
+ released = threading.Event()
+
+ def holder():
+ old = sys.getswitchinterval()
+ sys.setswitchinterval(10.0)
+ try:
+ deadline = time.monotonic() + seconds
+ while time.monotonic() < deadline:
+ pass
+ finally:
+ sys.setswitchinterval(old)
+ released.set()
+
+ loop_in_thread.call_soon_threadsafe(holder)
+ assert released.wait(timeout = 30.0)
+
+ with dump_path.open("w") as dump_file:
+ watchdog = StallWatchdog(
+ loop_in_thread,
+ dump_file = dump_file,
+ beat_interval_s = 0.05,
+ probe_slow_s = 0.05,
+ slow_probes_before_dump = 1000,
+ dead_man_timeout_s = 0.3,
+ dump_cooldown_s = 60.0,
+ )
+ watchdog.start()
+ try:
+ assert _wait_for(lambda: watchdog._dead_man_armed, timeout_s = 5.0)
+ hold_gil_for(0.8)
+ assert _wait_for(lambda: "Timeout" in dump_path.read_text(errors = "replace"))
+ assert _wait_for(lambda: watchdog._last_dump is not None, timeout_s = 5.0)
+ hold_gil_for(0.8)
+ time.sleep(0.3)
+ finally:
+ watchdog.stop()
+
+ text = dump_path.read_text(errors = "replace")
+ assert text.count("Timeout") == 1, (
+ "a second stall inside the cooldown window dumped again; a chronically "
+ "stalling host would fill its log with stacks"
+ )
+
+
+def test_a_python_dump_disarms_the_switch_for_the_cooldown(loop_in_thread, tmp_path):
+ """The two paths share the cooldown. A slow-probe dump leaves the switch armed
+ from earlier in the same beat unless it is taken down with it, and a GIL freeze
+ right after would land a second dump inside the window."""
+ dump_path = tmp_path / "dump.txt"
+
+ def blocks_the_loop():
+ time.sleep(0.5)
+
+ def holds_the_gil():
+ old = sys.getswitchinterval()
+ sys.setswitchinterval(10.0)
+ try:
+ deadline = time.monotonic() + 0.8
+ while time.monotonic() < deadline:
+ pass
+ finally:
+ sys.setswitchinterval(old)
+
+ with dump_path.open("w") as dump_file:
+ watchdog = StallWatchdog(
+ loop_in_thread,
+ dump_file = dump_file,
+ beat_interval_s = 0.05,
+ probe_slow_s = 0.05,
+ slow_probes_before_dump = 2,
+ dead_man_timeout_s = 0.3,
+ dump_cooldown_s = 60.0,
+ )
+ watchdog.start()
+ try:
+ loop_in_thread.call_soon_threadsafe(blocks_the_loop)
+ assert _wait_for(lambda: "stall watchdog:" in dump_path.read_text(errors = "replace"))
+ loop_in_thread.call_soon_threadsafe(holds_the_gil)
+ time.sleep(1.4)
+ finally:
+ watchdog.stop()
+
+ text = dump_path.read_text(errors = "replace")
+ assert "Timeout" not in text, (
+ "the dead man's switch fired inside the cooldown a slow-probe dump started; "
+ "one stall episode left two dumps"
+ )
+
+
+def test_stands_down_until_the_warm_is_over(monkeypatch):
+ """The dead man's switch cannot be disarmed once the warm holds the GIL, so the
+ gate has to be closed before the warm starts, and stay closed until it is over.
+ Gating on 'warm running' left the window between the watchdog's first beat and
+ start_background_warm(), and every cold start dumped over the torch import."""
+ import utils.stall_watchdog as sw
+ import utils.torch_warmup as tw
+
+ def status(started: bool, finished: bool, alive: bool):
+ monkeypatch.setattr(
+ tw,
+ "warm_status",
+ lambda: {"started": started, "finished": finished, "alive": alive, "stages": {}},
+ )
+
+ monkeypatch.delenv(tw.DISABLE_ENV_VAR, raising = False)
+ status(started = False, finished = False, alive = False)
+ assert sw.stand_down_for_the_warm() is True, (
+ "not standing down before the warm starts is the arming race: the switch "
+ "armed in that window dumps over `import torch` on every cold start"
+ )
+ status(started = True, finished = False, alive = True)
+ assert sw.stand_down_for_the_warm() is True
+ status(started = True, finished = True, alive = False)
+ assert sw.stand_down_for_the_warm() is False
+ # A warm that died mid-stage is not coming back.
+ status(started = True, finished = False, alive = False)
+ assert sw.stand_down_for_the_warm() is False
+
+ monkeypatch.setenv(tw.DISABLE_ENV_VAR, "1")
+ status(started = False, finished = False, alive = False)
+ assert sw.stand_down_for_the_warm() is False, (
+ "with the warm switched off no warm is coming; standing down anyway keeps "
+ "the watchdog disarmed for the whole session"
+ )
+
+
+def test_no_dump_while_suppressed(loop_in_thread, tmp_path):
+ """The warm's `import torch` is a legitimate long GIL hold with a known frame.
+ While the suppress callable says so, neither capture path may fire."""
+ dump_path = tmp_path / "dump.txt"
+
+ with dump_path.open("w") as dump_file:
+ watchdog = StallWatchdog(
+ loop_in_thread,
+ suppress = lambda: True,
+ dump_file = dump_file,
+ beat_interval_s = 0.05,
+ probe_slow_s = 0.05,
+ slow_probes_before_dump = 1,
+ dead_man_timeout_s = 0.2,
+ dump_cooldown_s = 0.0,
+ )
+ watchdog.start()
+ try:
+ loop_in_thread.call_soon_threadsafe(time.sleep, 0.8)
+ time.sleep(1.2)
+ finally:
+ watchdog.stop()
+
+ assert dump_path.read_text(errors = "replace") == "", (
+ "the watchdog dumped during a suppressed window; every warm import would "
+ "land a spurious dump in the log"
+ )
+
+
+def test_off_unless_the_env_opts_in(loop_in_thread, monkeypatch):
+ """Diagnostic tooling: a desktop run must carry neither the thread nor the
+ faulthandler timer, so nothing starts without the env the CI workflow sets."""
+ monkeypatch.delenv("UNSLOTH_STUDIO_STALL_WATCHDOG", raising = False)
+ assert start_stall_watchdog(loop_in_thread) is None
+ assert not any(t.name == "stall-watchdog" for t in threading.enumerate())
+
+
+def test_start_and_stop_manage_one_process_wide_instance(loop_in_thread, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_STUDIO_STALL_WATCHDOG", "1")
+ first = start_stall_watchdog(loop_in_thread)
+ try:
+ assert first is not None
+ second = start_stall_watchdog(loop_in_thread)
+ assert second is not None
+ assert _wait_for(
+ lambda: not (first._thread and first._thread.is_alive()), timeout_s = 5.0
+ ), "starting a second watchdog left the first one's thread running"
+ finally:
+ stop_stall_watchdog()
+ assert _wait_for(
+ lambda: not any(t.name == "stall-watchdog" for t in threading.enumerate()),
+ timeout_s = 5.0,
+ )
+
+
+def test_the_lifespan_wires_it_in():
+ """Cross-file guard: the module only does anything because main.py starts it at
+ startup and retires it at shutdown entry, and either side can be edited alone."""
+ source = (_BACKEND_DIR / "main.py").read_text(encoding = "utf-8")
+ assert (
+ "start_stall_watchdog(asyncio.get_running_loop()" in source
+ ), "main.py no longer starts the stall watchdog; the next #9712 stall leaves no dump behind"
+ assert "suppress = stand_down_for_the_warm" in source, (
+ "the watchdog no longer stands down for the warm; every cold start would "
+ "dump over the torch import"
+ )
+ assert "stop_stall_watchdog()" in source, (
+ "main.py never retires the watchdog, so shutdown's deliberate blocking "
+ "reads as a stall and dumps into the exit log"
+ )
+
+ workflow = _BACKEND_DIR.parent.parent / ".github" / "workflows" / "studio-mac-ui-smoke.yml"
+ assert workflow.is_file(), f"{workflow} moved; update this guard"
+ assert "UNSLOTH_STUDIO_STALL_WATCHDOG" in workflow.read_text(encoding = "utf-8"), (
+ "the watchdog is opt-in and the mac smoke workflow no longer opts in; the "
+ "one place #9712 stalls have been seen runs without it"
+ )
diff --git a/studio/backend/utils/stall_watchdog.py b/studio/backend/utils/stall_watchdog.py
new file mode 100644
index 00000000000..c96d3beb2b6
--- /dev/null
+++ b/studio/backend/utils/stall_watchdog.py
@@ -0,0 +1,246 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Dump every thread's stack from inside the backend while its event loop is stalled (#9712).
+
+Two capture paths. Loop stuck with the GIL free: the watchdog thread counts consecutive slow
+no-op probes and dumps from Python. GIL held: no Python thread runs, so each beat re-arms
+``faulthandler.dump_traceback_later`` as a dead man's switch; its C timer thread dumps without
+the GIL. Off unless UNSLOTH_STUDIO_STALL_WATCHDOG=1. Dumps go to the stderr fd (below run.py's
+tee). Assumes it is the only user of faulthandler's process-global delayed-dump timer.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import faulthandler
+import os
+import threading
+import time
+from concurrent.futures import TimeoutError as FutureTimeoutError
+from typing import Callable, Optional, TextIO
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+ENABLE_ENV_VAR = "UNSLOTH_STUDIO_STALL_WATCHDOG"
+
+BEAT_INTERVAL_S = 2.5
+# Healthy mac smoke runs: ~50ms worst latency, single-probe outliers to ~3.4s.
+PROBE_SLOW_S = 1.0
+# 3 slow beats = 6-8.5s unresponsive: dumps before the shortest recorded stall (10.03s) ends.
+SLOW_PROBES_BEFORE_DUMP = 3
+DEAD_MAN_TIMEOUT_S = 8.0
+# Shared by both capture paths.
+DUMP_COOLDOWN_S = 600.0
+
+
+def stand_down_for_the_warm() -> bool:
+ """True from process start until the warm is over: a switch armed before the warm's
+ GIL-holding ``import torch`` cannot be disarmed and would dump a known stall."""
+ from utils.torch_warmup import DISABLE_ENV_VAR as _WARM_DISABLED
+ from utils.torch_warmup import warm_status
+
+ status = warm_status()
+ if status["started"]:
+ return bool(status["alive"] and not status["finished"])
+ return os.environ.get(_WARM_DISABLED) != "1"
+
+
+async def _noop() -> None:
+ return None
+
+
+class StallWatchdog:
+ """One daemon thread beating against the event loop. start() / stop()."""
+
+ def __init__(
+ self,
+ loop: asyncio.AbstractEventLoop,
+ *,
+ suppress: Optional[Callable[[], bool]] = None,
+ dump_file: Optional[TextIO] = None,
+ beat_interval_s: float = BEAT_INTERVAL_S,
+ probe_slow_s: float = PROBE_SLOW_S,
+ slow_probes_before_dump: int = SLOW_PROBES_BEFORE_DUMP,
+ dead_man_timeout_s: float = DEAD_MAN_TIMEOUT_S,
+ dump_cooldown_s: float = DUMP_COOLDOWN_S,
+ ) -> None:
+ import sys
+
+ self._loop = loop
+ self._suppress = suppress or (lambda: False)
+ self._dump_file = dump_file if dump_file is not None else sys.stderr
+ self._beat_interval_s = beat_interval_s
+ self._probe_slow_s = probe_slow_s
+ self._slow_probes_before_dump = slow_probes_before_dump
+ self._dead_man_timeout_s = dead_man_timeout_s
+ self._dump_cooldown_s = dump_cooldown_s
+
+ self._stop_event = threading.Event()
+ self._thread: Optional[threading.Thread] = None
+ self._slow_streak = 0
+ self._stall_started: Optional[float] = None
+ self._last_dump: Optional[float] = None
+ self._dead_man_armed = False
+ self._arm_failure_logged = False
+
+ def start(self) -> None:
+ self._thread = threading.Thread(
+ target = self._run,
+ daemon = True,
+ name = "stall-watchdog",
+ )
+ self._thread.start()
+
+ def stop(self) -> None:
+ self._stop_event.set()
+ self._cancel_dead_man()
+ thread = self._thread
+ if thread is not None:
+ thread.join(timeout = 2.0)
+
+ def _run(self) -> None:
+ last_beat = time.monotonic()
+ while not self._stop_event.is_set():
+ beat_started = time.monotonic()
+ # Beat gap past the switch timeout: the switch fired; start the cooldown.
+ gap = beat_started - last_beat
+ if self._dead_man_armed and gap > self._dead_man_timeout_s:
+ self._last_dump = beat_started
+ self._dead_man_armed = False
+ logger.warning(
+ "stall watchdog was itself blocked for %.1fs; if the GIL was held, "
+ "faulthandler wrote a thread dump to stderr (system sleep also lands here)",
+ gap,
+ )
+ last_beat = beat_started
+
+ if self._suppress_now():
+ self._stop_event.wait(min(self._beat_interval_s, 0.5))
+ continue
+
+ self._arm_dead_man()
+ self._probe(beat_started)
+
+ elapsed = time.monotonic() - beat_started
+ self._stop_event.wait(max(0.0, self._beat_interval_s - elapsed))
+ self._cancel_dead_man()
+
+ def _suppress_now(self) -> bool:
+ try:
+ suppressed = bool(self._suppress())
+ except Exception:
+ suppressed = False
+ if suppressed:
+ self._cancel_dead_man()
+ self._slow_streak = 0
+ self._stall_started = None
+ return suppressed
+
+ def _probe(self, beat_started: float) -> None:
+ try:
+ future = asyncio.run_coroutine_threadsafe(_noop(), self._loop)
+ except RuntimeError:
+ self._stop_event.wait(self._beat_interval_s)
+ return
+ try:
+ future.result(timeout = self._probe_slow_s)
+ except FutureTimeoutError:
+ # Not cancelled: that leaves a never-awaited coroutine warning.
+ self._slow_streak += 1
+ if self._stall_started is None:
+ self._stall_started = beat_started
+ if self._slow_streak >= self._slow_probes_before_dump:
+ self._dump_from_python()
+ return
+ except Exception:
+ pass
+ if self._slow_streak:
+ stalled_for = time.monotonic() - (self._stall_started or beat_started)
+ logger.warning(
+ "event loop answering again after %.1fs (%d consecutive slow probes)",
+ stalled_for,
+ self._slow_streak,
+ )
+ self._slow_streak = 0
+ self._stall_started = None
+
+ def _in_cooldown(self) -> bool:
+ return (
+ self._last_dump is not None
+ and time.monotonic() - self._last_dump < self._dump_cooldown_s
+ )
+
+ def _arm_dead_man(self) -> None:
+ if self._in_cooldown():
+ self._cancel_dead_man()
+ return
+ try:
+ faulthandler.dump_traceback_later(
+ self._dead_man_timeout_s,
+ repeat = False,
+ file = self._dump_file,
+ exit = False,
+ )
+ self._dead_man_armed = True
+ except Exception as exc:
+ self._dead_man_armed = False
+ if not self._arm_failure_logged:
+ self._arm_failure_logged = True
+ logger.warning(
+ "stall watchdog cannot arm faulthandler (%s); GIL-held stalls "
+ "will not be dumped, only slow-probe ones",
+ exc,
+ )
+
+ def _cancel_dead_man(self) -> None:
+ if self._dead_man_armed:
+ faulthandler.cancel_dump_traceback_later()
+ self._dead_man_armed = False
+
+ def _dump_from_python(self) -> None:
+ if self._in_cooldown():
+ return
+ now = time.monotonic()
+ self._last_dump = now
+ # Else this beat's switch could fire a second dump inside the cooldown.
+ self._cancel_dead_man()
+ stalled_for = now - (self._stall_started or now)
+ try:
+ self._dump_file.write(
+ f"\nstall watchdog: event loop unresponsive for {stalled_for:.1f}s "
+ f"({self._slow_streak} consecutive slow probes), dumping all threads\n"
+ )
+ self._dump_file.flush()
+ faulthandler.dump_traceback(file = self._dump_file, all_threads = True)
+ except Exception as exc:
+ logger.warning("stall watchdog could not write a thread dump: %s", exc)
+
+
+_watchdog: Optional[StallWatchdog] = None
+_watchdog_lock = threading.Lock()
+
+
+def start_stall_watchdog(
+ loop: asyncio.AbstractEventLoop, *, suppress: Optional[Callable[[], bool]] = None
+) -> Optional[StallWatchdog]:
+ """Start the process-wide watchdog. Returns None unless the env opts in."""
+ if os.environ.get(ENABLE_ENV_VAR) != "1":
+ return None
+ global _watchdog
+ with _watchdog_lock:
+ if _watchdog is not None:
+ _watchdog.stop()
+ _watchdog = StallWatchdog(loop, suppress = suppress)
+ _watchdog.start()
+ return _watchdog
+
+
+def stop_stall_watchdog() -> None:
+ global _watchdog
+ with _watchdog_lock:
+ if _watchdog is not None:
+ _watchdog.stop()
+ _watchdog = None