Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stall the signalled launchers in slices, so a signal at READY is not deferred (#11661)

Daniel Hanff29fb48c713
diff --git a/tests/kaggle/test_launch_cleanup.py b/tests/kaggle/test_launch_cleanup.py
index 4f807fd10ae..33568b11d36 100644
--- a/tests/kaggle/test_launch_cleanup.py
+++ b/tests/kaggle/test_launch_cleanup.py
@@ -137,6 +137,18 @@ _DEATH_BUDGET_SEC = 120
# ordinary path and exit inside the wait, passing every deletion test.
_STALL_SEC = 900
+# The stall itself, in half-second slices rather than one time.sleep(_STALL_SEC). CPython runs a Python signal handler
+# only after the sleeping syscall returns, and a signal that lands between the interpreter's last check and the
+# clock_nanosleep call does not interrupt it: the handler then waits out the whole sleep. Measured with a bare script
+# that installs a SIGTERM handler, prints READY and sleeps, signalled the moment READY is read: 3 of 1500 one-long-sleep
+# children were still asleep 5s later with SIGTERM caught and nothing pending or blocked, and 0 of 1500 slicing ones.
+# This file signals exactly at READY, so it hit that window about once in 300 and reported a 120s hang. Slices still
+# add up to _STALL_SEC, so a launcher that swallows its signal still outlasts the budget.
+_STALL_SLICE_SEC = 0.5
+_STALL = (
+ f"for _slice in range({int(_STALL_SEC / _STALL_SLICE_SEC)}): time.sleep({_STALL_SLICE_SEC})"
+)
+
def _await_ready(proc: subprocess.Popen) -> None:
"""Wait for the runner to say it is where the test wants it, past whatever
@@ -509,7 +521,7 @@ def _waiting_launcher(outdir: Path) -> str:
" launch.push = _push",
" def _wait(*a, **kw):",
" print('READY', flush=True)",
- " time.sleep(%d)" % _STALL_SEC,
+ " " + _STALL,
" launch.wait = _wait",
" sys.argv = ['launch.py', '--notebook', 'a.ipynb', '--user', 'me',",
f" '--outdir', {str(outdir)!r}]",
@@ -532,7 +544,7 @@ def _flooding_launcher(outdir: Path) -> str:
return (
_waiting_launcher(outdir)
.replace(
- " time.sleep(%d)" % _STALL_SEC,
+ " " + _STALL,
"\n".join(
[
" while True:",
@@ -647,6 +659,26 @@ def test_the_stall_outlasts_the_death_budget():
)
+def test_the_stall_is_sliced_so_a_signal_at_ready_is_not_deferred():
+ """Every stub stalls through `_STALL`, whose slices bound how long CPython can defer a handler.
+
+ A single long sleep turned a signal landing just before the syscall into a wait for the whole stall,
+ which is how this file reported "still alive 120s after its signal" with the handler never having run.
+ """
+ ns = {"time": __import__("types").SimpleNamespace(sleep = lambda s: slept.append(s))}
+ slept: list[float] = []
+ exec(_STALL, ns)
+ assert max(slept) <= 1.0, f"a {max(slept)}s slice can hold a pending handler that long"
+ assert sum(slept) == pytest.approx(
+ _STALL_SEC
+ ), "the slices must still add up to the whole stall"
+ source = Path(__file__).read_text(encoding = "utf-8")
+ long_sleeps = source.count("time.sleep(%d)" + chr(34) + " % _STALL_SEC") + source.count(
+ "time.sleep({" + "_STALL_SEC})"
+ )
+ assert long_sleeps == 0, "a stub went back to one long sleep; stall through _STALL instead"
+
+
def test_the_handler_survives_its_own_logging_failing(tmp_path):
"""The reentrancy that a contended runner produced, made deterministic.
@@ -774,7 +806,7 @@ def test_a_reentrant_log_inside_the_delete_retries_does_not_abandon_them(tmp_pat
launch._inflight_add("me/k-1")
launch._install_release_handlers(release)
print("READY", flush=True)
- time.sleep({_STALL_SEC})
+ {_STALL}
"""),
encoding = "utf-8",
)
@@ -939,7 +971,7 @@ def test_kill_9_leaves_it_for_the_sweep(tmp_path):
f"""
launch._inflight_add("me/k-9")
print("READY", flush=True)
- time.sleep({_STALL_SEC})
+ {_STALL}
""",
)
try:
@@ -1067,7 +1099,7 @@ def test_a_kernel_pushed_before_the_signal_is_still_deleted(tmp_path):
" if len(calls) == 1:",
" return {'ok': True, 'slug': slug, 'attempts': attempted}",
" print('READY', flush=True)",
- " time.sleep(%d)" % _STALL_SEC,
+ " " + _STALL,
" launch.push = _push",
" sys.argv = ['launch.py', '--notebook', 'a.ipynb', '--notebook', 'b.ipynb',",
f" '--user', 'me', '--outdir', {str(tmp_path / 'out')!r}]",