Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Derive the chat UI wall-clock watchdog from the turn timeout (#9025)
Daniel Han2c3873d36beb
diff --git a/.github/scripts/run-studio-permission-browser.sh b/.github/scripts/run-studio-permission-browser.shindex f9e15f9d3d7..cb8e085c7c8 100755--- a/.github/scripts/run-studio-permission-browser.sh+++ b/.github/scripts/run-studio-permission-browser.sh@@ -94,7 +94,11 @@ export STUDIO_OLD_PW="$old_password"export STUDIO_NEW_PW="$new_password"export STUDIO_UI_STRICT=1export STUDIO_UI_PERMISSION_ONLY=1+# 240s is a per-browser cap, and three browsers run in one step. The wall is the budget+# between two progress reports now, so it no longer bounds a whole invocation on its own:+# the total is what holds this to four minutes per browser.export STUDIO_UI_WALL_TIMEOUT_S=240+export STUDIO_UI_TOTAL_TIMEOUT_S=240export STUDIO_PLAYWRIGHT_BROWSER="$browser"export PW_ART_DIR="$artifact_dir"if [ -n "$channel" ]; thendiff --git a/tests/kaggle/studio_gpu/run_studio_gpu.py b/tests/kaggle/studio_gpu/run_studio_gpu.pyindex 045675e3f02..fe690847ab4 100644--- a/tests/kaggle/studio_gpu/run_studio_gpu.py+++ b/tests/kaggle/studio_gpu/run_studio_gpu.py@@ -127,6 +127,14 @@ MIN_FREE_GB = 25.0# once per session before anything needs it. Generous, because the cost of# being too tight is a red that reads like a selection bug.LLAMA_CPP_INSTALL_TIMEOUT_S = 900.0+# STUDIO_UI_WALL_TIMEOUT_S measures silence between progress reports, so it cannot size a+# backstop: the driver exits one budget after its LAST step, and any fixed number here is a+# guess a still-progressing run can beat. So hand the driver a total, which no kick moves,+# and the backstop becomes a sum. Six times the lane's ~10 min healthy pass, inside its+# 120 min job. Losing the race costs the driver's traceback: SIGKILL leaves+# TimeoutExpired holding no stderr.+UI_DRIVER_TOTAL_TIMEOUT_S = 3600.0+UI_DRIVER_PROC_TIMEOUT_S = UI_DRIVER_TOTAL_TIMEOUT_S + 300.0# How long to let VRAM fall after an unload before calling it the baseline.# 12 x 2.5s bounds the wait at 30s, which is well past the ~3s a llama-server# takes to exit on the models this harness loads, without stalling the run if@@ -3094,6 +3102,7 @@ class Payload:# a cached 270M model. Here the model is larger and the box is# busier."STUDIO_UI_WALL_TIMEOUT_S": str(self.args.ui_wall_timeout),+ "STUDIO_UI_TOTAL_TIMEOUT_S": str(UI_DRIVER_TOTAL_TIMEOUT_S),"STUDIO_UI_LOAD_TIMEOUT_MS": "600000","STUDIO_UI_TURN_TIMEOUT_MS": "180000","PYTHONUNBUFFERED": "1",@@ -3108,7 +3117,7 @@ class Payload:env = env,capture_output = True,text = True,- timeout = self.args.ui_wall_timeout + 300,+ timeout = UI_DRIVER_PROC_TIMEOUT_S,)rc, out, err = proc.returncode, proc.stdout, proc.stderrexcept subprocess.TimeoutExpired as exc:diff --git a/tests/kaggle/test_studio_gpu_harness.py b/tests/kaggle/test_studio_gpu_harness.pyindex 4013b72a218..58c8019b190 100644--- a/tests/kaggle/test_studio_gpu_harness.py+++ b/tests/kaggle/test_studio_gpu_harness.py@@ -1252,6 +1252,30 @@ def test_the_ui_driver_gets_a_freshly_seeded_account():), "the password must be read after the restart, or it is the old one"+def test_the_driver_subprocess_timeout_does_not_track_the_ui_wall_budget():+ """The parent must not out-race the watchdog it is a backstop for.++ `ui_wall_timeout + 300` was the looser of the two only while that budget bounded the+ whole run. It bounds silence now, so the backstop has to come from the total instead.++ Asserted on the source because reaching the call needs a live server."""+ source = (PAYLOAD_DIR / "run_studio_gpu.py").read_text(encoding = "utf-8")+ body = source[source.index("def assert_chat_ui") :]+ body = body[: body.index("\n def ")] if "\n def " in body else body+ call = body[body.index("subprocess.run(") :]+ call = call[: call.index("\n )")]+ assert "UI_DRIVER_PROC_TIMEOUT_S" in call, call+ assert "ui_wall_timeout" not in call, call++ module = _load_payload()+ # A sum, not a guess: the driver is handed a total no progress report can move.+ assert "STUDIO_UI_TOTAL_TIMEOUT_S" in body, body+ assert module.UI_DRIVER_PROC_TIMEOUT_S > module.UI_DRIVER_TOTAL_TIMEOUT_S+ # Six times the ~10 minute healthy pass, and inside the lane's 120 minute job.+ assert module.UI_DRIVER_TOTAL_TIMEOUT_S >= 6 * 600+ assert module.UI_DRIVER_PROC_TIMEOUT_S < 120 * 60++# The llama.cpp install step.# Four hardware runs reported install_kind=None and failed the export assertion for it, because nothing had ever# installed a llama.cpp under STUDIO_HOME.diff --git a/tests/studio/_playwright_robust.py b/tests/studio/_playwright_robust.pyindex 67fbe9b6ebf..ddc17a2e78f 100644--- a/tests/studio/_playwright_robust.py+++ b/tests/studio/_playwright_robust.py@@ -744,23 +744,89 @@ def evaluate_fetch(return last or {"status": 0, "body": None, "error": "no attempt made"}-# Wall-clock watchdog.-# A browser wedge (CPU-pinned JS, silent renderer crash, asyncio deadlock) can still hang the script. A daemon Timer-# calls os._exit(2) after deadline_s; exit code 2 lets the workflow's `set -e` propagate. Pick deadline_s above the-# slowest healthy run (macos-14 cold cache ~7-9 min) but under the 30-min cap.+class _WallClockWatchdog:+ """A `deadline_s` budget that `kick()` restarts; never kicking is the absolute wall+ a `threading.Timer` gave. Timer has no reschedule, hence a thread over a live deadline.+ """++ def __init__(+ self,+ deadline_s: float,+ on_expiry: Callable[[], None],+ total_deadline_s: float | None = None,+ ) -> None:+ self._budget_s = float(deadline_s)+ self._on_expiry = on_expiry+ self._lock = threading.Lock()+ started = time.monotonic()+ # A kick moves the deadline forever, so a caller sizing an outer bound has nothing+ # to size against. `total_deadline_s` is a ceiling no kick moves, which makes that+ # bound a sum. Off by default: a ceiling is what cuts a wait off mid-flight.+ self._ceiling = started + float(total_deadline_s) if total_deadline_s else None+ self._deadline = self._clamp(started + self._budget_s)+ self._cancelled = threading.Event()+ self._thread = threading.Thread(target = self._run, daemon = True)+ self.kicked = False++ def at_ceiling(self) -> bool:+ """Did the total cap, rather than the per-wait budget, decide the deadline?"""+ with self._lock:+ return self._ceiling is not None and self._deadline >= self._ceiling++ def _clamp(self, deadline: float) -> float:+ return deadline if self._ceiling is None else min(deadline, self._ceiling)++ def start(self) -> "_WallClockWatchdog":+ self._thread.start()+ return self++ def kick(self) -> None:+ """Progress was made: restart the budget."""+ with self._lock:+ self.kicked = True+ self._deadline = self._clamp(time.monotonic() + self._budget_s)++ def cancel(self) -> None:+ self._cancelled.set()++ def _run(self) -> None:+ while True:+ with self._lock:+ remaining = self._deadline - time.monotonic()+ if remaining <= 0:+ if not self._cancelled.is_set():+ self._on_expiry()+ return+ # Capped so a mid-sleep kick is seen; the deadline is re-read, not trusted.+ if self._cancelled.wait(min(remaining, 1.0)):+ return+++# For a wedge no per-action timeout can bound. Exit 2 propagates through the workflow's `set -e`. Keep deadline_s+# above the longest single wait, or that wait is cut off before it can name itself.def install_wall_clock_watchdog(deadline_s: float,*,label: str = "playwright",info: Callable[[str], None] | None = None,-) -> threading.Timer:- """Start a daemon Timer that hard-exits the process at `deadline_s`; returned- so the caller can `.cancel()` on clean exit (daemonised, dies with process)."""+ total_deadline_s: float | None = None,+) -> _WallClockWatchdog:+ """Hard-exit `deadline_s` after the last `kick()`, or at `total_deadline_s` from arming+ if that comes first; returned so the caller can `.cancel()`."""def _kaboom() -> None:+ # A caller that kicks is measuring inactivity, one that does not is measuring the+ # whole run. Saying "no step" to a script that never reports one sends its reader+ # looking for a step that was never going to come.+ if total_deadline_s and watchdog.at_ceiling():+ spent = f"hit the {total_deadline_s:.0f}s total cap"+ elif watchdog.kicked:+ spent = f"{deadline_s:.0f}s with no step reported"+ else:+ spent = f"hit {deadline_s:.0f}s wall-clock deadline"msg = (- f"[{label}] WATCHDOG: hit {deadline_s:.0f}s wall-clock "- f"deadline; forcing exit(2). The script wedged somewhere "+ f"[{label}] WATCHDOG: {spent}; "+ f"forcing exit(2). The script wedged somewhere "f"the per-action timeouts could not bound. Inspect the "f"most recent step printed above to localise.")@@ -769,14 +835,24 @@ def install_wall_clock_watchdog(sys.stderr.flush()except Exception:pass+ # The last step printed is where the script ENTERED, not where it blocked. Under the+ # sync API the main thread stops at the driver loop: driver vs our code, no finer.+ try:+ import faulthandler+ faulthandler.dump_traceback(file = sys.stderr, all_threads = True)+ sys.stderr.flush()+ except Exception:+ passos._exit(2)- timer = threading.Timer(deadline_s, _kaboom)- timer.daemon = True- timer.start()+ # Bound before started: at deadline_s <= 0 the thread reaches _kaboom during start(),+ # and a _kaboom that closed over an unbound name dies of NameError in that thread+ # instead of exiting, leaving the run with no watchdog at all.+ watchdog = _WallClockWatchdog(deadline_s, _kaboom, total_deadline_s)+ watchdog.start()if info is not None:- info(f"watchdog armed: hard-exit at {deadline_s:.0f}s")- return timer+ info(f"watchdog armed: hard-exit {deadline_s:.0f}s after the last step")+ return watchdogdef click_forced(diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.pyindex e1ab0bf8456..2a71fe980ac 100644--- a/tests/studio/playwright_chat_ui.py+++ b/tests/studio/playwright_chat_ui.py@@ -54,9 +54,6 @@ TURN_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_TURN_TIMEOUT_MS", "180000"))# cannot close the gap, and paid once per run.RAPID_FIRST_TURN_HOLD_S = 3.0-# Wall-clock cap for the whole script (healthy run is 5-9 min).-WALL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_WALL_TIMEOUT_S", "720"))-PERMISSION_ONLY = os.environ.get("STUDIO_UI_PERMISSION_ONLY", "0") == "1"# Default stays Chromium for CI. Local runs can select firefox/webkit or a Chromium channel such as chrome/msedge.@@ -72,15 +69,49 @@ CPU_THROTTLE = float(os.environ.get("STUDIO_UI_CPU_THROTTLE", "0") or 0)FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_FETCH_TIMEOUT_MS", "30000"))LOAD_FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_LOAD_TIMEOUT_MS", "180000"))+# Budget for ONE wait, restarted by `wall_kick()`. It must outlast the longest single wait+# or it hard-exits mid-wait and the run says only "wedged somewhere". All three candidates+# are env vars, so take the max of all three rather than whichever wins at today's values:+# the rapid-submit settle at 2x the turn timeout (1080s where studio-mac-ui-smoke.yml sets+# 540000, against the 720s this was pinned at), the load fetch (600s on the Kaggle lane),+# and the ordinary fetch. Not a total: `send_and_wait` budgets 4x the turn timeout across+# seven turns. Linux, raising none of them, keeps its 720s floor.+_WALL_FLOOR_S = 720.0+_LONGEST_WAIT_S = max(+ (TURN_TIMEOUT_MS / 1000) * 2,+ LOAD_FETCH_TIMEOUT_MS / 1000,+ FETCH_TIMEOUT_MS / 1000,+)+WALL_TIMEOUT_S = float(+ os.environ.get(+ "STUDIO_UI_WALL_TIMEOUT_S",+ max(_WALL_FLOOR_S, _LONGEST_WAIT_S + 120),+ )+)+# Off by default, so the run has no total; see `_WallClockWatchdog`. Set by the callers+# that must size an outer bound around this process: tests/kaggle/studio_gpu and+# .github/scripts/run-studio-permission-browser.sh.+TOTAL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_TOTAL_TIMEOUT_S", "0")) or None+_n = [0]+_watchdog = None # armed below; everything above it runs before there is one+++def wall_kick():+ """Restart the budget. Call after a bounded wait, or the next inherits its leftover."""+ if _watchdog is not None:+ _watchdog.kick()+def step(s):print(f"[ui] STEP {s}", flush = True)+ wall_kick()def info(s):print(f"[ui] {s}", flush = True)+ wall_kick()def fail(m):@@ -674,6 +705,7 @@ with sync_playwright() as p:WALL_TIMEOUT_S,label = "ui",info = info,+ total_deadline_s = TOTAL_TIMEOUT_S,)# Pre-flight: macos-14 can surface a 200 /api/health while the auth DB is still migrating;# this 30s probe catches that gap before we sink 60s into a change-password timeout.@@ -766,6 +798,9 @@ with sync_playwright() as p:# page/reload between tries so a mid-try rerender doesn't poison the next.form_err: Exception | None = Nonefor _form_attempt in range(3):+ # Forward progress that reports itself with bare print(), so nothing else here+ # resets the budget: two failed attempts spend the whole Linux 720s.+ wall_kick()try:page.goto(f"{BASE}/change-password", wait_until = "domcontentloaded", timeout = 60_000)try:@@ -855,6 +890,7 @@ with sync_playwright() as p:composer = page.locator('textarea[aria-label="Message input"]')last_err: Exception | None = Nonefor _attempt in range(2):+ wall_kick() # as in the change-password loop: the retry logs with bare print()try:composer.wait_for(state = "visible", timeout = 60_000)last_err = None@@ -1118,6 +1154,7 @@ with sync_playwright() as p:state = "attached",timeout = TURN_TIMEOUT_MS,)+ wall_kick()try:page.wait_for_selector('button[aria-label="Stop generating"]',@@ -1131,6 +1168,7 @@ with sync_playwright() as p:state = "detached",timeout = TURN_TIMEOUT_MS,)+ wall_kick()# 2. Snapshot total bubble count before send; we wait for it to grow by exactly 1. We do NOT require# non-empty text: an empty assistant response is legitimate (gemma-3-270m does this at temp 0), and the@@ -1165,6 +1203,7 @@ with sync_playwright() as p:arg = bubbles_before + 1,timeout = TURN_TIMEOUT_MS,)+ wall_kick()# 4. Wait for this turn's streaming to finish. Stop may never appear (gemma-3-270m can finish before it# paints), so its appearance is best-effort; then wait for it to detach.@@ -1176,6 +1215,7 @@ with sync_playwright() as p:)except Exception:pass+ wall_kick()try:page.wait_for_selector('button[aria-label="Stop generating"]',@@ -1362,6 +1402,8 @@ with sync_playwright() as p:# Settle before the five-turn sequence below: two bubbles, nothing streaming, nothing queued.# What the turns SAID is not checked here and never was;# the queue behaviour this step exists to prove is `state.queueSeen` above.+ # The longest single wait in the script, so it starts on a full budget.+ wall_kick()page.wait_for_function("""(want) => {const replies = Array.from(diff --git a/tests/studio/test_playwright_wall_clock_watchdog.py b/tests/studio/test_playwright_wall_clock_watchdog.pynew file mode 100644index 00000000000..1b606658470--- /dev/null+++ b/tests/studio/test_playwright_wall_clock_watchdog.py@@ -0,0 +1,213 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The watchdog may not hard-exit a wait still inside its own timeout, or the run reports+only "wedged somewhere". So the budget must restart on `wall_kick()`, and no two+turn-scaled waits may run without a kick between them. `playwright_chat_ui.py` drives+Playwright at import, so its half is read from source; the watchdog is run for real.+"""++from __future__ import annotations++import ast+import contextlib+import io+import sys+import threading+import time+from pathlib import Path+from unittest import mock++STUDIO_TESTS = Path(__file__).resolve().parent+sys.path.insert(0, str(STUDIO_TESTS))++import _playwright_robust as robust # noqa: E402+from _playwright_robust import _WallClockWatchdog # noqa: E402++CHAT_UI = STUDIO_TESTS / "playwright_chat_ui.py"+CHAT_UI_SRC = CHAT_UI.read_text(encoding = "utf-8")+CHAT_UI_TREE = ast.parse(CHAT_UI_SRC)+++def _fired(+ budget_s,+ *,+ kicks = (),+ cancel_after = None,+ run_for = None,+):+ """Run a watchdog for `run_for` seconds, kicking at each offset in `kicks`."""+ fired = threading.Event()+ watchdog = _WallClockWatchdog(budget_s, fired.set).start()+ started = time.monotonic()+ try:+ for at in kicks:+ time.sleep(max(0.0, started + at - time.monotonic()))+ watchdog.kick()+ if cancel_after is not None:+ time.sleep(max(0.0, started + cancel_after - time.monotonic()))+ watchdog.cancel()+ time.sleep(max(0.0, started + run_for - time.monotonic()))+ finally:+ watchdog.cancel()+ return fired.is_set()+++# Every deadline in these three has 0.6s of slack, the last one included: a kick at 0.8s+# moves expiry to 1.8s and observation stops at 1.2s, so only a 600ms overshoot could+# decide the result rather than the watchdog.+def test_an_unkicked_watchdog_still_fires_at_its_budget():+ assert _fired(0.4, run_for = 1.6)+++def test_a_kick_restarts_the_budget():+ """The defect: setup ran before the wait, so the wait inherited what setup left."""+ assert not _fired(1.0, kicks = (0.4, 0.8), run_for = 1.2)+++def test_a_cancelled_watchdog_does_not_fire():+ assert not _fired(0.8, cancel_after = 0.2, run_for = 1.4)+++def test_a_watchdog_that_expires_during_start_still_exits():+ # `install_wall_clock_watchdog` builds the handle its own expiry callback reads, so at+ # a deadline of 0 the thread can reach that callback before the name is bound. It then+ # dies of NameError in the daemon thread and the run silently loses its watchdog.+ codes = []+ with mock.patch.object(robust.os, "_exit", codes.append):+ robust.install_wall_clock_watchdog(0.0, label = "ui")+ time.sleep(0.5)+ assert codes == [2], codes+++def test_a_total_cap_is_a_ceiling_no_kick_can_move():+ """What makes an outer bound around this process a sum instead of a guess.++ Without it a caller sizing a backstop has nothing to size against: every kick moves+ the deadline, so the exit lands at a wall-clock time the caller cannot predict."""+ fired = threading.Event()+ watchdog = _WallClockWatchdog(10.0, fired.set, total_deadline_s = 0.7).start()+ started = time.monotonic()+ try:+ while time.monotonic() - started < 1.4:+ time.sleep(0.05)+ watchdog.kick() # kicking throughout must not push past the ceiling+ finally:+ watchdog.cancel()+ assert fired.is_set()+ assert watchdog.at_ceiling()+++def test_without_a_total_cap_nothing_is_at_the_ceiling():+ watchdog = _WallClockWatchdog(10.0, lambda: None)+ watchdog.kick()+ assert not watchdog.at_ceiling()+++def _watchdog_message(kick):+ """The line a real `install_wall_clock_watchdog` prints on expiry, minus the exit."""+ watchdog = robust.install_wall_clock_watchdog(30.0, label = "ui")+ watchdog.cancel()+ if kick:+ watchdog.kick()+ buf = io.StringIO()+ with mock.patch.object(robust.os, "_exit"), contextlib.redirect_stderr(buf):+ watchdog._on_expiry()+ return buf.getvalue().splitlines()[0]+++def test_the_message_names_what_actually_ran_out():+ # The scripts that never kick are measuring the whole run, not inactivity; telling+ # their reader to look for a step sends them after one that never existed.+ assert "hit 30s wall-clock deadline" in _watchdog_message(kick = False)+ assert "30s with no step reported" in _watchdog_message(kick = True)+++def _chat_ui_wall_timeout_s(+ turn_timeout_ms,+ load_timeout_ms = 180_000,+ fetch_timeout_ms = 30_000,+):+ """Evaluate the script's own WALL_TIMEOUT_S expression at a given set of budgets."""+ wanted = {+ "TURN_TIMEOUT_MS",+ "LOAD_FETCH_TIMEOUT_MS",+ "FETCH_TIMEOUT_MS",+ "_WALL_FLOOR_S",+ "_LONGEST_WAIT_S",+ "WALL_TIMEOUT_S",+ }+ body = [+ node+ for node in CHAT_UI_TREE.body+ if isinstance(node, ast.Assign)+ and isinstance(node.targets[0], ast.Name)+ and node.targets[0].id in wanted+ ]+ assert {n.targets[0].id for n in body} == wanted, "the wall-timeout constants moved"+ env = {+ "STUDIO_UI_TURN_TIMEOUT_MS": str(turn_timeout_ms),+ "STUDIO_UI_LOAD_TIMEOUT_MS": str(load_timeout_ms),+ "STUDIO_UI_FETCH_TIMEOUT_MS": str(fetch_timeout_ms),+ }+ ns = {"os": type("_os", (), {"environ": env})}+ exec(compile(ast.Module(body = body, type_ignores = []), str(CHAT_UI), "exec"), ns)+ return ns["WALL_TIMEOUT_S"], ns["_LONGEST_WAIT_S"]+++def test_the_wall_budget_outlasts_the_longest_single_wait():+ # 540000 is studio-mac-ui-smoke.yml's; 180000 is every other runner's default.+ for turn_timeout_ms in (180_000, 540_000):+ wall, longest_wait = _chat_ui_wall_timeout_s(turn_timeout_ms)+ assert wall >= longest_wait + 120, (turn_timeout_ms, wall, longest_wait)+++def test_a_raised_fetch_budget_also_raises_the_wall():+ # Every budget in the max is an env var, so none of them may be left out on the+ # grounds that no lane raises it today.+ wall, longest_wait = _chat_ui_wall_timeout_s(180_000, fetch_timeout_ms = 900_000)+ assert longest_wait == 900.0+ assert wall >= 900.0 + 120+++def test_a_raised_load_budget_also_raises_the_wall():+ # The Kaggle lane sets STUDIO_UI_LOAD_TIMEOUT_MS to 600000 and leaves the turn timeout+ # at its default, so the load fetch, not the turn, is the longest wait there.+ wall, longest_wait = _chat_ui_wall_timeout_s(180_000, load_timeout_ms = 600_000)+ assert longest_wait == 600.0+ assert wall >= 600.0 + 120+++def test_the_linux_default_keeps_the_budget_it_had():+ assert _chat_ui_wall_timeout_s(180_000)[0] == 720.0+++def _turn_scaled_wait_lines():+ """Lines of the waits whose timeout is TURN_TIMEOUT_MS or a multiple of it."""+ lines = []+ for node in ast.walk(CHAT_UI_TREE):+ if not isinstance(node, ast.Call):+ continue+ for kw in node.keywords:+ if kw.arg != "timeout":+ continue+ names = {n.id for n in ast.walk(kw.value) if isinstance(n, ast.Name)}+ if "TURN_TIMEOUT_MS" in names:+ lines.append(kw.value.lineno)+ return sorted(lines)+++def test_every_pair_of_turn_scaled_waits_is_separated_by_a_kick():+ kicks = sorted(+ node.lineno+ for node in ast.walk(CHAT_UI_TREE)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Name)+ and node.func.id == "wall_kick"+ )+ waits = _turn_scaled_wait_lines()+ assert len(waits) >= 5, waits+ for first, second in zip(waits, waits[1:]):+ assert any(+ first < k < second for k in kicks+ ), f"the waits at lines {first} and {second} share one watchdog budget"diff --git a/tests/studio/test_ui_scripts_report_signals.py b/tests/studio/test_ui_scripts_report_signals.pyindex cf204dc3d98..13f1c82bb62 100644--- a/tests/studio/test_ui_scripts_report_signals.py+++ b/tests/studio/test_ui_scripts_report_signals.py@@ -94,3 +94,20 @@ def test_the_guard_reads_real_files() -> None:for name in WRAPPERS:assert (SCRIPTS / name).is_file(), nameassert len(_body(name)) > 400, name+++def test_a_wall_budget_in_a_wrapper_comes_with_a_total():+ """A wrapper's STUDIO_UI_WALL_TIMEOUT_S is a cap on the whole invocation.++ It stopped being one when playwright_chat_ui.py started restarting that budget on+ every step, so a run that keeps reporting progress runs past it. Three browsers share+ one step here, so the cap is what keeps a wedged first browser from taking the other+ two with it. STUDIO_UI_TOTAL_TIMEOUT_S is the ceiling no progress report moves."""+ for name in WRAPPERS:+ text = (SCRIPTS / name).read_text(encoding = "utf-8")+ wall = re.search(r"STUDIO_UI_WALL_TIMEOUT_S=(\d+)", text)+ if wall is None:+ continue+ total = re.search(r"STUDIO_UI_TOTAL_TIMEOUT_S=(\d+)", text)+ assert total is not None, f"{name} caps the wall but not the run"+ assert total.group(1) == wall.group(1), name