Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Derive the chat UI wall-clock watchdog from the turn timeout (#9025)

Daniel Han2c3873d36beb
diff --git a/.github/scripts/run-studio-permission-browser.sh b/.github/scripts/run-studio-permission-browser.sh
index f9e15f9d3d7..cb8e085c7c8 100755
--- a/.github/scripts/run-studio-permission-browser.sh
+++ b/.github/scripts/run-studio-permission-browser.sh
@@ -94,7 +94,11 @@ export STUDIO_OLD_PW="$old_password"
export STUDIO_NEW_PW="$new_password"
export STUDIO_UI_STRICT=1
export STUDIO_UI_PERMISSION_ONLY=1
+# 240s is a per-browser cap, and three browsers run in one step. The wall is the budget
+# between two progress reports now, so it no longer bounds a whole invocation on its own:
+# the total is what holds this to four minutes per browser.
export STUDIO_UI_WALL_TIMEOUT_S=240
+export STUDIO_UI_TOTAL_TIMEOUT_S=240
export STUDIO_PLAYWRIGHT_BROWSER="$browser"
export PW_ART_DIR="$artifact_dir"
if [ -n "$channel" ]; then
diff --git a/tests/kaggle/studio_gpu/run_studio_gpu.py b/tests/kaggle/studio_gpu/run_studio_gpu.py
index 045675e3f02..fe690847ab4 100644
--- a/tests/kaggle/studio_gpu/run_studio_gpu.py
+++ b/tests/kaggle/studio_gpu/run_studio_gpu.py
@@ -127,6 +127,14 @@ MIN_FREE_GB = 25.0
# once per session before anything needs it. Generous, because the cost of
# being too tight is a red that reads like a selection bug.
LLAMA_CPP_INSTALL_TIMEOUT_S = 900.0
+# STUDIO_UI_WALL_TIMEOUT_S measures silence between progress reports, so it cannot size a
+# backstop: the driver exits one budget after its LAST step, and any fixed number here is a
+# guess a still-progressing run can beat. So hand the driver a total, which no kick moves,
+# and the backstop becomes a sum. Six times the lane's ~10 min healthy pass, inside its
+# 120 min job. Losing the race costs the driver's traceback: SIGKILL leaves
+# TimeoutExpired holding no stderr.
+UI_DRIVER_TOTAL_TIMEOUT_S = 3600.0
+UI_DRIVER_PROC_TIMEOUT_S = UI_DRIVER_TOTAL_TIMEOUT_S + 300.0
# How long to let VRAM fall after an unload before calling it the baseline.
# 12 x 2.5s bounds the wait at 30s, which is well past the ~3s a llama-server
# takes to exit on the models this harness loads, without stalling the run if
@@ -3094,6 +3102,7 @@ class Payload:
# a cached 270M model. Here the model is larger and the box is
# busier.
"STUDIO_UI_WALL_TIMEOUT_S": str(self.args.ui_wall_timeout),
+ "STUDIO_UI_TOTAL_TIMEOUT_S": str(UI_DRIVER_TOTAL_TIMEOUT_S),
"STUDIO_UI_LOAD_TIMEOUT_MS": "600000",
"STUDIO_UI_TURN_TIMEOUT_MS": "180000",
"PYTHONUNBUFFERED": "1",
@@ -3108,7 +3117,7 @@ class Payload:
env = env,
capture_output = True,
text = True,
- timeout = self.args.ui_wall_timeout + 300,
+ timeout = UI_DRIVER_PROC_TIMEOUT_S,
)
rc, out, err = proc.returncode, proc.stdout, proc.stderr
except subprocess.TimeoutExpired as exc:
diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.py
index 4013b72a218..58c8019b190 100644
--- a/tests/kaggle/test_studio_gpu_harness.py
+++ b/tests/kaggle/test_studio_gpu_harness.py
@@ -1252,6 +1252,30 @@ def test_the_ui_driver_gets_a_freshly_seeded_account():
), "the password must be read after the restart, or it is the old one"
+def test_the_driver_subprocess_timeout_does_not_track_the_ui_wall_budget():
+ """The parent must not out-race the watchdog it is a backstop for.
+
+ `ui_wall_timeout + 300` was the looser of the two only while that budget bounded the
+ whole run. It bounds silence now, so the backstop has to come from the total instead.
+
+ Asserted on the source because reaching the call needs a live server."""
+ source = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")
+ body = source[source.index("def assert_chat_ui") :]
+ body = body[: body.index("\n def ")] if "\n def " in body else body
+ call = body[body.index("subprocess.run(") :]
+ call = call[: call.index("\n )")]
+ assert "UI_DRIVER_PROC_TIMEOUT_S" in call, call
+ assert "ui_wall_timeout" not in call, call
+
+ module = _load_payload()
+ # A sum, not a guess: the driver is handed a total no progress report can move.
+ assert "STUDIO_UI_TOTAL_TIMEOUT_S" in body, body
+ assert module.UI_DRIVER_PROC_TIMEOUT_S > module.UI_DRIVER_TOTAL_TIMEOUT_S
+ # Six times the ~10 minute healthy pass, and inside the lane's 120 minute job.
+ assert module.UI_DRIVER_TOTAL_TIMEOUT_S >= 6 * 600
+ assert module.UI_DRIVER_PROC_TIMEOUT_S < 120 * 60
+
+
# The llama.cpp install step.
# Four hardware runs reported install_kind=None and failed the export assertion for it, because nothing had ever
# installed a llama.cpp under STUDIO_HOME.
diff --git a/tests/studio/_playwright_robust.py b/tests/studio/_playwright_robust.py
index 67fbe9b6ebf..ddc17a2e78f 100644
--- a/tests/studio/_playwright_robust.py
+++ b/tests/studio/_playwright_robust.py
@@ -744,23 +744,89 @@ def evaluate_fetch(
return last or {"status": 0, "body": None, "error": "no attempt made"}
-# Wall-clock watchdog.
-# A browser wedge (CPU-pinned JS, silent renderer crash, asyncio deadlock) can still hang the script. A daemon Timer
-# calls os._exit(2) after deadline_s; exit code 2 lets the workflow's `set -e` propagate. Pick deadline_s above the
-# slowest healthy run (macos-14 cold cache ~7-9 min) but under the 30-min cap.
+class _WallClockWatchdog:
+ """A `deadline_s` budget that `kick()` restarts; never kicking is the absolute wall
+ a `threading.Timer` gave. Timer has no reschedule, hence a thread over a live deadline.
+ """
+
+ def __init__(
+ self,
+ deadline_s: float,
+ on_expiry: Callable[[], None],
+ total_deadline_s: float | None = None,
+ ) -> None:
+ self._budget_s = float(deadline_s)
+ self._on_expiry = on_expiry
+ self._lock = threading.Lock()
+ started = time.monotonic()
+ # A kick moves the deadline forever, so a caller sizing an outer bound has nothing
+ # to size against. `total_deadline_s` is a ceiling no kick moves, which makes that
+ # bound a sum. Off by default: a ceiling is what cuts a wait off mid-flight.
+ self._ceiling = started + float(total_deadline_s) if total_deadline_s else None
+ self._deadline = self._clamp(started + self._budget_s)
+ self._cancelled = threading.Event()
+ self._thread = threading.Thread(target = self._run, daemon = True)
+ self.kicked = False
+
+ def at_ceiling(self) -> bool:
+ """Did the total cap, rather than the per-wait budget, decide the deadline?"""
+ with self._lock:
+ return self._ceiling is not None and self._deadline >= self._ceiling
+
+ def _clamp(self, deadline: float) -> float:
+ return deadline if self._ceiling is None else min(deadline, self._ceiling)
+
+ def start(self) -> "_WallClockWatchdog":
+ self._thread.start()
+ return self
+
+ def kick(self) -> None:
+ """Progress was made: restart the budget."""
+ with self._lock:
+ self.kicked = True
+ self._deadline = self._clamp(time.monotonic() + self._budget_s)
+
+ def cancel(self) -> None:
+ self._cancelled.set()
+
+ def _run(self) -> None:
+ while True:
+ with self._lock:
+ remaining = self._deadline - time.monotonic()
+ if remaining <= 0:
+ if not self._cancelled.is_set():
+ self._on_expiry()
+ return
+ # Capped so a mid-sleep kick is seen; the deadline is re-read, not trusted.
+ if self._cancelled.wait(min(remaining, 1.0)):
+ return
+
+
+# For a wedge no per-action timeout can bound. Exit 2 propagates through the workflow's `set -e`. Keep deadline_s
+# above the longest single wait, or that wait is cut off before it can name itself.
def install_wall_clock_watchdog(
deadline_s: float,
*,
label: str = "playwright",
info: Callable[[str], None] | None = None,
-) -> threading.Timer:
- """Start a daemon Timer that hard-exits the process at `deadline_s`; returned
- so the caller can `.cancel()` on clean exit (daemonised, dies with process)."""
+ total_deadline_s: float | None = None,
+) -> _WallClockWatchdog:
+ """Hard-exit `deadline_s` after the last `kick()`, or at `total_deadline_s` from arming
+ if that comes first; returned so the caller can `.cancel()`."""
def _kaboom() -> None:
+ # A caller that kicks is measuring inactivity, one that does not is measuring the
+ # whole run. Saying "no step" to a script that never reports one sends its reader
+ # looking for a step that was never going to come.
+ if total_deadline_s and watchdog.at_ceiling():
+ spent = f"hit the {total_deadline_s:.0f}s total cap"
+ elif watchdog.kicked:
+ spent = f"{deadline_s:.0f}s with no step reported"
+ else:
+ spent = f"hit {deadline_s:.0f}s wall-clock deadline"
msg = (
- f"[{label}] WATCHDOG: hit {deadline_s:.0f}s wall-clock "
- f"deadline; forcing exit(2). The script wedged somewhere "
+ f"[{label}] WATCHDOG: {spent}; "
+ f"forcing exit(2). The script wedged somewhere "
f"the per-action timeouts could not bound. Inspect the "
f"most recent step printed above to localise."
)
@@ -769,14 +835,24 @@ def install_wall_clock_watchdog(
sys.stderr.flush()
except Exception:
pass
+ # The last step printed is where the script ENTERED, not where it blocked. Under the
+ # sync API the main thread stops at the driver loop: driver vs our code, no finer.
+ try:
+ import faulthandler
+ faulthandler.dump_traceback(file = sys.stderr, all_threads = True)
+ sys.stderr.flush()
+ except Exception:
+ pass
os._exit(2)
- timer = threading.Timer(deadline_s, _kaboom)
- timer.daemon = True
- timer.start()
+ # Bound before started: at deadline_s <= 0 the thread reaches _kaboom during start(),
+ # and a _kaboom that closed over an unbound name dies of NameError in that thread
+ # instead of exiting, leaving the run with no watchdog at all.
+ watchdog = _WallClockWatchdog(deadline_s, _kaboom, total_deadline_s)
+ watchdog.start()
if info is not None:
- info(f"watchdog armed: hard-exit at {deadline_s:.0f}s")
- return timer
+ info(f"watchdog armed: hard-exit {deadline_s:.0f}s after the last step")
+ return watchdog
def click_forced(
diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py
index e1ab0bf8456..2a71fe980ac 100644
--- a/tests/studio/playwright_chat_ui.py
+++ b/tests/studio/playwright_chat_ui.py
@@ -54,9 +54,6 @@ TURN_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_TURN_TIMEOUT_MS", "180000"))
# cannot close the gap, and paid once per run.
RAPID_FIRST_TURN_HOLD_S = 3.0
-# Wall-clock cap for the whole script (healthy run is 5-9 min).
-WALL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_WALL_TIMEOUT_S", "720"))
-
PERMISSION_ONLY = os.environ.get("STUDIO_UI_PERMISSION_ONLY", "0") == "1"
# Default stays Chromium for CI. Local runs can select firefox/webkit or a Chromium channel such as chrome/msedge.
@@ -72,15 +69,49 @@ CPU_THROTTLE = float(os.environ.get("STUDIO_UI_CPU_THROTTLE", "0") or 0)
FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_FETCH_TIMEOUT_MS", "30000"))
LOAD_FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_LOAD_TIMEOUT_MS", "180000"))
+# Budget for ONE wait, restarted by `wall_kick()`. It must outlast the longest single wait
+# or it hard-exits mid-wait and the run says only "wedged somewhere". All three candidates
+# are env vars, so take the max of all three rather than whichever wins at today's values:
+# the rapid-submit settle at 2x the turn timeout (1080s where studio-mac-ui-smoke.yml sets
+# 540000, against the 720s this was pinned at), the load fetch (600s on the Kaggle lane),
+# and the ordinary fetch. Not a total: `send_and_wait` budgets 4x the turn timeout across
+# seven turns. Linux, raising none of them, keeps its 720s floor.
+_WALL_FLOOR_S = 720.0
+_LONGEST_WAIT_S = max(
+ (TURN_TIMEOUT_MS / 1000) * 2,
+ LOAD_FETCH_TIMEOUT_MS / 1000,
+ FETCH_TIMEOUT_MS / 1000,
+)
+WALL_TIMEOUT_S = float(
+ os.environ.get(
+ "STUDIO_UI_WALL_TIMEOUT_S",
+ max(_WALL_FLOOR_S, _LONGEST_WAIT_S + 120),
+ )
+)
+# Off by default, so the run has no total; see `_WallClockWatchdog`. Set by the callers
+# that must size an outer bound around this process: tests/kaggle/studio_gpu and
+# .github/scripts/run-studio-permission-browser.sh.
+TOTAL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_TOTAL_TIMEOUT_S", "0")) or None
+
_n = [0]
+_watchdog = None # armed below; everything above it runs before there is one
+
+
+def wall_kick():
+ """Restart the budget. Call after a bounded wait, or the next inherits its leftover."""
+ if _watchdog is not None:
+ _watchdog.kick()
+
def step(s):
print(f"[ui] STEP {s}", flush = True)
+ wall_kick()
def info(s):
print(f"[ui] {s}", flush = True)
+ wall_kick()
def fail(m):
@@ -674,6 +705,7 @@ with sync_playwright() as p:
WALL_TIMEOUT_S,
label = "ui",
info = info,
+ total_deadline_s = TOTAL_TIMEOUT_S,
)
# Pre-flight: macos-14 can surface a 200 /api/health while the auth DB is still migrating;
# this 30s probe catches that gap before we sink 60s into a change-password timeout.
@@ -766,6 +798,9 @@ with sync_playwright() as p:
# page/reload between tries so a mid-try rerender doesn't poison the next.
form_err: Exception | None = None
for _form_attempt in range(3):
+ # Forward progress that reports itself with bare print(), so nothing else here
+ # resets the budget: two failed attempts spend the whole Linux 720s.
+ wall_kick()
try:
page.goto(f"{BASE}/change-password", wait_until = "domcontentloaded", timeout = 60_000)
try:
@@ -855,6 +890,7 @@ with sync_playwright() as p:
composer = page.locator('textarea[aria-label="Message input"]')
last_err: Exception | None = None
for _attempt in range(2):
+ wall_kick() # as in the change-password loop: the retry logs with bare print()
try:
composer.wait_for(state = "visible", timeout = 60_000)
last_err = None
@@ -1118,6 +1154,7 @@ with sync_playwright() as p:
state = "attached",
timeout = TURN_TIMEOUT_MS,
)
+ wall_kick()
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
@@ -1131,6 +1168,7 @@ with sync_playwright() as p:
state = "detached",
timeout = TURN_TIMEOUT_MS,
)
+ wall_kick()
# 2. Snapshot total bubble count before send; we wait for it to grow by exactly 1. We do NOT require
# non-empty text: an empty assistant response is legitimate (gemma-3-270m does this at temp 0), and the
@@ -1165,6 +1203,7 @@ with sync_playwright() as p:
arg = bubbles_before + 1,
timeout = TURN_TIMEOUT_MS,
)
+ wall_kick()
# 4. Wait for this turn's streaming to finish. Stop may never appear (gemma-3-270m can finish before it
# paints), so its appearance is best-effort; then wait for it to detach.
@@ -1176,6 +1215,7 @@ with sync_playwright() as p:
)
except Exception:
pass
+ wall_kick()
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
@@ -1362,6 +1402,8 @@ with sync_playwright() as p:
# Settle before the five-turn sequence below: two bubbles, nothing streaming, nothing queued.
# What the turns SAID is not checked here and never was;
# the queue behaviour this step exists to prove is `state.queueSeen` above.
+ # The longest single wait in the script, so it starts on a full budget.
+ wall_kick()
page.wait_for_function(
"""(want) => {
const replies = Array.from(
diff --git a/tests/studio/test_playwright_wall_clock_watchdog.py b/tests/studio/test_playwright_wall_clock_watchdog.py
new file mode 100644
index 00000000000..1b606658470
--- /dev/null
+++ b/tests/studio/test_playwright_wall_clock_watchdog.py
@@ -0,0 +1,213 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The watchdog may not hard-exit a wait still inside its own timeout, or the run reports
+only "wedged somewhere". So the budget must restart on `wall_kick()`, and no two
+turn-scaled waits may run without a kick between them. `playwright_chat_ui.py` drives
+Playwright at import, so its half is read from source; the watchdog is run for real.
+"""
+
+from __future__ import annotations
+
+import ast
+import contextlib
+import io
+import sys
+import threading
+import time
+from pathlib import Path
+from unittest import mock
+
+STUDIO_TESTS = Path(__file__).resolve().parent
+sys.path.insert(0, str(STUDIO_TESTS))
+
+import _playwright_robust as robust # noqa: E402
+from _playwright_robust import _WallClockWatchdog # noqa: E402
+
+CHAT_UI = STUDIO_TESTS / "playwright_chat_ui.py"
+CHAT_UI_SRC = CHAT_UI.read_text(encoding = "utf-8")
+CHAT_UI_TREE = ast.parse(CHAT_UI_SRC)
+
+
+def _fired(
+ budget_s,
+ *,
+ kicks = (),
+ cancel_after = None,
+ run_for = None,
+):
+ """Run a watchdog for `run_for` seconds, kicking at each offset in `kicks`."""
+ fired = threading.Event()
+ watchdog = _WallClockWatchdog(budget_s, fired.set).start()
+ started = time.monotonic()
+ try:
+ for at in kicks:
+ time.sleep(max(0.0, started + at - time.monotonic()))
+ watchdog.kick()
+ if cancel_after is not None:
+ time.sleep(max(0.0, started + cancel_after - time.monotonic()))
+ watchdog.cancel()
+ time.sleep(max(0.0, started + run_for - time.monotonic()))
+ finally:
+ watchdog.cancel()
+ return fired.is_set()
+
+
+# Every deadline in these three has 0.6s of slack, the last one included: a kick at 0.8s
+# moves expiry to 1.8s and observation stops at 1.2s, so only a 600ms overshoot could
+# decide the result rather than the watchdog.
+def test_an_unkicked_watchdog_still_fires_at_its_budget():
+ assert _fired(0.4, run_for = 1.6)
+
+
+def test_a_kick_restarts_the_budget():
+ """The defect: setup ran before the wait, so the wait inherited what setup left."""
+ assert not _fired(1.0, kicks = (0.4, 0.8), run_for = 1.2)
+
+
+def test_a_cancelled_watchdog_does_not_fire():
+ assert not _fired(0.8, cancel_after = 0.2, run_for = 1.4)
+
+
+def test_a_watchdog_that_expires_during_start_still_exits():
+ # `install_wall_clock_watchdog` builds the handle its own expiry callback reads, so at
+ # a deadline of 0 the thread can reach that callback before the name is bound. It then
+ # dies of NameError in the daemon thread and the run silently loses its watchdog.
+ codes = []
+ with mock.patch.object(robust.os, "_exit", codes.append):
+ robust.install_wall_clock_watchdog(0.0, label = "ui")
+ time.sleep(0.5)
+ assert codes == [2], codes
+
+
+def test_a_total_cap_is_a_ceiling_no_kick_can_move():
+ """What makes an outer bound around this process a sum instead of a guess.
+
+ Without it a caller sizing a backstop has nothing to size against: every kick moves
+ the deadline, so the exit lands at a wall-clock time the caller cannot predict."""
+ fired = threading.Event()
+ watchdog = _WallClockWatchdog(10.0, fired.set, total_deadline_s = 0.7).start()
+ started = time.monotonic()
+ try:
+ while time.monotonic() - started < 1.4:
+ time.sleep(0.05)
+ watchdog.kick() # kicking throughout must not push past the ceiling
+ finally:
+ watchdog.cancel()
+ assert fired.is_set()
+ assert watchdog.at_ceiling()
+
+
+def test_without_a_total_cap_nothing_is_at_the_ceiling():
+ watchdog = _WallClockWatchdog(10.0, lambda: None)
+ watchdog.kick()
+ assert not watchdog.at_ceiling()
+
+
+def _watchdog_message(kick):
+ """The line a real `install_wall_clock_watchdog` prints on expiry, minus the exit."""
+ watchdog = robust.install_wall_clock_watchdog(30.0, label = "ui")
+ watchdog.cancel()
+ if kick:
+ watchdog.kick()
+ buf = io.StringIO()
+ with mock.patch.object(robust.os, "_exit"), contextlib.redirect_stderr(buf):
+ watchdog._on_expiry()
+ return buf.getvalue().splitlines()[0]
+
+
+def test_the_message_names_what_actually_ran_out():
+ # The scripts that never kick are measuring the whole run, not inactivity; telling
+ # their reader to look for a step sends them after one that never existed.
+ assert "hit 30s wall-clock deadline" in _watchdog_message(kick = False)
+ assert "30s with no step reported" in _watchdog_message(kick = True)
+
+
+def _chat_ui_wall_timeout_s(
+ turn_timeout_ms,
+ load_timeout_ms = 180_000,
+ fetch_timeout_ms = 30_000,
+):
+ """Evaluate the script's own WALL_TIMEOUT_S expression at a given set of budgets."""
+ wanted = {
+ "TURN_TIMEOUT_MS",
+ "LOAD_FETCH_TIMEOUT_MS",
+ "FETCH_TIMEOUT_MS",
+ "_WALL_FLOOR_S",
+ "_LONGEST_WAIT_S",
+ "WALL_TIMEOUT_S",
+ }
+ body = [
+ node
+ for node in CHAT_UI_TREE.body
+ if isinstance(node, ast.Assign)
+ and isinstance(node.targets[0], ast.Name)
+ and node.targets[0].id in wanted
+ ]
+ assert {n.targets[0].id for n in body} == wanted, "the wall-timeout constants moved"
+ env = {
+ "STUDIO_UI_TURN_TIMEOUT_MS": str(turn_timeout_ms),
+ "STUDIO_UI_LOAD_TIMEOUT_MS": str(load_timeout_ms),
+ "STUDIO_UI_FETCH_TIMEOUT_MS": str(fetch_timeout_ms),
+ }
+ ns = {"os": type("_os", (), {"environ": env})}
+ exec(compile(ast.Module(body = body, type_ignores = []), str(CHAT_UI), "exec"), ns)
+ return ns["WALL_TIMEOUT_S"], ns["_LONGEST_WAIT_S"]
+
+
+def test_the_wall_budget_outlasts_the_longest_single_wait():
+ # 540000 is studio-mac-ui-smoke.yml's; 180000 is every other runner's default.
+ for turn_timeout_ms in (180_000, 540_000):
+ wall, longest_wait = _chat_ui_wall_timeout_s(turn_timeout_ms)
+ assert wall >= longest_wait + 120, (turn_timeout_ms, wall, longest_wait)
+
+
+def test_a_raised_fetch_budget_also_raises_the_wall():
+ # Every budget in the max is an env var, so none of them may be left out on the
+ # grounds that no lane raises it today.
+ wall, longest_wait = _chat_ui_wall_timeout_s(180_000, fetch_timeout_ms = 900_000)
+ assert longest_wait == 900.0
+ assert wall >= 900.0 + 120
+
+
+def test_a_raised_load_budget_also_raises_the_wall():
+ # The Kaggle lane sets STUDIO_UI_LOAD_TIMEOUT_MS to 600000 and leaves the turn timeout
+ # at its default, so the load fetch, not the turn, is the longest wait there.
+ wall, longest_wait = _chat_ui_wall_timeout_s(180_000, load_timeout_ms = 600_000)
+ assert longest_wait == 600.0
+ assert wall >= 600.0 + 120
+
+
+def test_the_linux_default_keeps_the_budget_it_had():
+ assert _chat_ui_wall_timeout_s(180_000)[0] == 720.0
+
+
+def _turn_scaled_wait_lines():
+ """Lines of the waits whose timeout is TURN_TIMEOUT_MS or a multiple of it."""
+ lines = []
+ for node in ast.walk(CHAT_UI_TREE):
+ if not isinstance(node, ast.Call):
+ continue
+ for kw in node.keywords:
+ if kw.arg != "timeout":
+ continue
+ names = {n.id for n in ast.walk(kw.value) if isinstance(n, ast.Name)}
+ if "TURN_TIMEOUT_MS" in names:
+ lines.append(kw.value.lineno)
+ return sorted(lines)
+
+
+def test_every_pair_of_turn_scaled_waits_is_separated_by_a_kick():
+ kicks = sorted(
+ node.lineno
+ for node in ast.walk(CHAT_UI_TREE)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Name)
+ and node.func.id == "wall_kick"
+ )
+ waits = _turn_scaled_wait_lines()
+ assert len(waits) >= 5, waits
+ for first, second in zip(waits, waits[1:]):
+ assert any(
+ first < k < second for k in kicks
+ ), f"the waits at lines {first} and {second} share one watchdog budget"
diff --git a/tests/studio/test_ui_scripts_report_signals.py b/tests/studio/test_ui_scripts_report_signals.py
index cf204dc3d98..13f1c82bb62 100644
--- a/tests/studio/test_ui_scripts_report_signals.py
+++ b/tests/studio/test_ui_scripts_report_signals.py
@@ -94,3 +94,20 @@ def test_the_guard_reads_real_files() -> None:
for name in WRAPPERS:
assert (SCRIPTS / name).is_file(), name
assert len(_body(name)) > 400, name
+
+
+def test_a_wall_budget_in_a_wrapper_comes_with_a_total():
+ """A wrapper's STUDIO_UI_WALL_TIMEOUT_S is a cap on the whole invocation.
+
+ It stopped being one when playwright_chat_ui.py started restarting that budget on
+ every step, so a run that keeps reporting progress runs past it. Three browsers share
+ one step here, so the cap is what keeps a wedged first browser from taking the other
+ two with it. STUDIO_UI_TOTAL_TIMEOUT_S is the ceiling no progress report moves."""
+ for name in WRAPPERS:
+ text = (SCRIPTS / name).read_text(encoding = "utf-8")
+ wall = re.search(r"STUDIO_UI_WALL_TIMEOUT_S=(\d+)", text)
+ if wall is None:
+ continue
+ total = re.search(r"STUDIO_UI_TOTAL_TIMEOUT_S=(\d+)", text)
+ assert total is not None, f"{name} caps the wall but not the run"
+ assert total.group(1) == wall.group(1), name