Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Playwright helpers: named per-step budgets, fail-fast step report, condition waits (#11979)

Daniel Han3f79df8708cb
diff --git a/tests/studio/_playwright_robust.py b/tests/studio/_playwright_robust.py
index b0c69aa203f..89e205ddae6 100644
--- a/tests/studio/_playwright_robust.py
+++ b/tests/studio/_playwright_robust.py
@@ -806,6 +806,12 @@ class _WallClockWatchdog:
# to size against. `total_deadline_s` is a ceiling no kick moves, which makes that
# bound a sum. Off by default: a ceiling is what cuts a wait off mid-flight.
self._ceiling = started + float(total_deadline_s) if total_deadline_s else None
+ # The step now running, set by begin_step(). A watchdog nobody names a step to
+ # behaves exactly as before.
+ self.step_name: str | None = None
+ self.step_budget_s: float | None = None
+ self._step_started: float | None = None
+ self._step_ceiling: float | None = None
self._deadline = self._clamp(started + self._budget_s)
self._cancelled = threading.Event()
self._thread = threading.Thread(target = self._run, daemon = True)
@@ -816,8 +822,22 @@ class _WallClockWatchdog:
with self._lock:
return self._ceiling is not None and self._deadline >= self._ceiling
+ def at_step_ceiling(self) -> bool:
+ """Did the running step's own budget, rather than inactivity, decide the deadline?"""
+ with self._lock:
+ return self._step_ceiling is not None and self._deadline >= self._step_ceiling
+
+ def step_elapsed_s(self) -> float | None:
+ """Seconds since the running step began, or None before the first begin_step()."""
+ with self._lock:
+ started = self._step_started
+ return None if started is None else time.monotonic() - started
+
def _clamp(self, deadline: float) -> float:
- return deadline if self._ceiling is None else min(deadline, self._ceiling)
+ for ceiling in (self._ceiling, self._step_ceiling):
+ if ceiling is not None:
+ deadline = min(deadline, ceiling)
+ return deadline
def start(self) -> "_WallClockWatchdog":
self._thread.start()
@@ -829,6 +849,27 @@ class _WallClockWatchdog:
self.kicked = True
self._deadline = self._clamp(time.monotonic() + self._budget_s)
+ def begin_step(
+ self,
+ name: str,
+ budget_s: float | None = None,
+ ) -> None:
+ """Step `name` starts now: a kick that also records what is running.
+
+ With `budget_s`, the step gets its own ceiling, which kicks inside the step cannot
+ move: a step that keeps reporting progress but never finishes still ends at
+ start + budget_s, and the exit names it. Without one, only the inactivity budget
+ applies, as before. The next begin_step() replaces both.
+ """
+ with self._lock:
+ now = time.monotonic()
+ self.kicked = True
+ self.step_name = name
+ self._step_started = now
+ self.step_budget_s = float(budget_s) if budget_s else None
+ self._step_ceiling = now + self.step_budget_s if self.step_budget_s else None
+ self._deadline = self._clamp(now + self._budget_s)
+
def cancel(self) -> None:
self._cancelled.set()
@@ -861,8 +902,13 @@ def install_wall_clock_watchdog(
# A caller that kicks is measuring inactivity, one that does not is measuring the
# whole run. Saying "no step" to a script that never reports one sends its reader
# looking for a step that was never going to come.
+ step = watchdog.step_name
if total_deadline_s and watchdog.at_ceiling():
spent = f"hit the {total_deadline_s:.0f}s total cap"
+ elif step is not None and watchdog.at_step_ceiling():
+ spent = f"step {step!r} used its whole {watchdog.step_budget_s:.0f}s budget"
+ elif step is not None:
+ spent = f"{deadline_s:.0f}s with no progress in step {step!r}"
elif watchdog.kicked:
spent = f"{deadline_s:.0f}s with no step reported"
else:
@@ -898,6 +944,144 @@ def install_wall_clock_watchdog(
return watchdog
+# Per-step budgets are sized for a hosted Linux runner. A slower lane stretches every one
+# of them at once with this, instead of each script growing its own knob.
+STEP_BUDGET_SCALE_ENV = "STUDIO_PW_STEP_BUDGET_SCALE"
+
+
+def step_budget_s(seconds: float) -> float:
+ """`seconds`, stretched by $STUDIO_PW_STEP_BUDGET_SCALE (default 1, never below 1)."""
+ try:
+ scale = float(os.environ.get(STEP_BUDGET_SCALE_ENV) or 1.0)
+ except ValueError:
+ scale = 1.0
+ return float(seconds) * max(1.0, scale)
+
+
+def report_failing_step(watchdog: _WallClockWatchdog, *, label: str = "playwright") -> None:
+ """On an uncaught exception, end the output with the step it happened in.
+
+ A step that times out raises, and the run stops there: nothing after it waits out its
+ own timeout. But the traceback names a line, and the reader wants the step. This adds
+ one line after it, `[label] FAIL in step 'X' after 12.3s: TimeoutError: ...`, taken
+ from the watchdog's begin_step() record. No step begun, nothing added.
+ """
+ previous = sys.excepthook
+
+ def _hook(exc_type, exc, tb) -> None:
+ previous(exc_type, exc, tb)
+ name = watchdog.step_name
+ if name is None or issubclass(exc_type, KeyboardInterrupt):
+ return
+ elapsed = watchdog.step_elapsed_s() or 0.0
+ first = (str(exc).strip().splitlines() or [""])[0][:300]
+ try:
+ sys.stderr.write(
+ f"[{label}] FAIL in step {name!r} after {elapsed:.1f}s: "
+ f"{exc_type.__name__}: {first}\n"
+ )
+ sys.stderr.flush()
+ except Exception:
+ pass
+
+ sys.excepthook = _hook
+
+
+def wait_until(
+ predicate: Callable[[], Any],
+ *,
+ timeout_s: float,
+ what: str,
+ interval_s: float = 0.1,
+ page: Any = None,
+) -> Any:
+ """Poll `predicate` until it returns something truthy, and return that.
+
+ Raises TimeoutError naming `what` and the last value seen once `timeout_s` passes, so
+ a wait that never comes true fails where it waited instead of at a later assertion.
+
+ Pass `page` whenever the predicate reads state that Playwright event handlers fill in
+ (`page.on("request", ...)` lists and the like). The sync API only dispatches events
+ while it is inside a Playwright call, so the pause between polls has to be
+ `page.wait_for_timeout`, not `time.sleep`, or the list never grows.
+ """
+ deadline = time.monotonic() + float(timeout_s)
+ while True:
+ value = predicate()
+ if value:
+ return value
+ if time.monotonic() >= deadline:
+ raise TimeoutError(
+ f"{what}: still not true after {timeout_s:g}s (last value {value!r})"
+ )
+ if page is not None:
+ page.wait_for_timeout(interval_s * 1000.0)
+ else:
+ time.sleep(interval_s)
+
+
+# "settled" once the element has kept the same box, with none of its (or its subtree's) Web
+# Animations / CSS transitions running, for `frames` animation frames in a row; "detached" once
+# the node is gone (a re-render replaced it), so the caller can look it up again. The nonce
+# keeps one call's count from carrying into the next.
+_SETTLED_JS = """
+([el, frames, nonce]) => {
+ if (!el || !el.isConnected) return "detached";
+ const r = el.getBoundingClientRect();
+ const sig = [r.left, r.top, r.width, r.height].map((v) => Math.round(v * 10)).join(",");
+ const running = typeof el.getAnimations === "function"
+ && el.getAnimations({ subtree: true }).some((a) => a.playState === "running");
+ if (!window.__pwSettled) window.__pwSettled = new WeakMap();
+ const prev = window.__pwSettled.get(el);
+ const count = !running && prev && prev.nonce === nonce && prev.sig === sig ? prev.count + 1 : 0;
+ window.__pwSettled.set(el, { nonce, sig, count });
+ return count >= frames ? "settled" : false;
+}
+"""
+_settle_nonce = [0]
+
+
+def wait_for_settled(
+ locator: Any,
+ *,
+ frames: int = 3,
+ timeout_ms: int = 10_000,
+) -> None:
+ """Wait for `locator.first` to stop moving: same box, nothing animating, `frames` frames running.
+
+ The condition behind "wait N ms for the transition / reflow to finish": a resize, an
+ expand, a slide-in. It returns as soon as the element is still, and on a runner slow
+ enough that N was not enough it keeps waiting instead of measuring mid-flight. If a
+ re-render replaces the node mid-wait, the locator is resolved again and the count
+ restarts on the new node. Raises Playwright's TimeoutError if nothing settles within
+ `timeout_ms`.
+ """
+ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+ target = locator.first
+ deadline = time.monotonic() + timeout_ms / 1000.0
+ while True:
+ remaining_ms = int((deadline - time.monotonic()) * 1000)
+ if remaining_ms <= 0:
+ raise PlaywrightTimeoutError(f"wait_for_settled: not settled within {timeout_ms}ms")
+ _settle_nonce[0] += 1
+ handle = target.element_handle(timeout = remaining_ms)
+ try:
+ outcome = target.page.wait_for_function(
+ _SETTLED_JS,
+ arg = [handle, int(frames), _settle_nonce[0]],
+ polling = "raf",
+ timeout = max(1, remaining_ms),
+ ).json_value()
+ finally:
+ try:
+ handle.dispose()
+ except Exception:
+ pass
+ if outcome == "settled":
+ return
+
+
def click_forced(
locator: Any,
*,
diff --git a/tests/studio/test_playwright_wall_clock_watchdog.py b/tests/studio/test_playwright_wall_clock_watchdog.py
index 1b606658470..fe5ddab5e2c 100644
--- a/tests/studio/test_playwright_wall_clock_watchdog.py
+++ b/tests/studio/test_playwright_wall_clock_watchdog.py
@@ -211,3 +211,139 @@ def test_every_pair_of_turn_scaled_waits_is_separated_by_a_kick():
assert any(
first < k < second for k in kicks
), f"the waits at lines {first} and {second} share one watchdog budget"
+
+
+# Named steps and per-step budgets.
+
+
+def test_a_step_budget_is_a_ceiling_kicks_inside_the_step_cannot_move():
+ """A step that keeps reporting progress but never finishes still ends, and ends as that step."""
+ fired = threading.Event()
+ watchdog = _WallClockWatchdog(10.0, fired.set).start()
+ started = time.monotonic()
+ try:
+ watchdog.begin_step("slow step", budget_s = 0.6)
+ while time.monotonic() - started < 1.4:
+ time.sleep(0.05)
+ watchdog.kick()
+ finally:
+ watchdog.cancel()
+ assert fired.is_set()
+ assert watchdog.at_step_ceiling()
+ assert watchdog.step_name == "slow step"
+
+
+def test_the_next_step_gets_its_own_budget():
+ # Each of the three steps runs 0.5s against a 0.8s budget: none may inherit the last.
+ fired = threading.Event()
+ watchdog = _WallClockWatchdog(10.0, fired.set).start()
+ try:
+ for name in ("one", "two", "three"):
+ watchdog.begin_step(name, budget_s = 0.8)
+ time.sleep(0.5)
+ finally:
+ watchdog.cancel()
+ assert not fired.is_set()
+
+
+def test_a_step_without_a_budget_keeps_the_inactivity_budget():
+ watchdog = _WallClockWatchdog(10.0, lambda: None)
+ watchdog.begin_step("unbudgeted")
+ assert not watchdog.at_step_ceiling()
+ assert watchdog.step_budget_s is None
+
+
+def _named_step_message(budget_s, *, expire_step):
+ watchdog = robust.install_wall_clock_watchdog(30.0, label = "ui")
+ watchdog.cancel()
+ watchdog.begin_step("theme toggle x3", budget_s = budget_s)
+ if expire_step:
+ with watchdog._lock:
+ watchdog._deadline = watchdog._step_ceiling
+ buf = io.StringIO()
+ with mock.patch.object(robust.os, "_exit"), contextlib.redirect_stderr(buf):
+ watchdog._on_expiry()
+ return buf.getvalue().splitlines()[0]
+
+
+def test_the_message_names_the_step_that_ran_out():
+ assert "step 'theme toggle x3' used its whole 90s budget" in _named_step_message(
+ 90.0, expire_step = True
+ )
+ assert "30s with no progress in step 'theme toggle x3'" in _named_step_message(
+ None, expire_step = False
+ )
+
+
+def test_an_uncaught_exception_is_reported_with_its_step():
+ watchdog = _WallClockWatchdog(10.0, lambda: None)
+ original = sys.excepthook
+ seen = []
+ try:
+ sys.excepthook = lambda *a: seen.append(a[0])
+ robust.report_failing_step(watchdog, label = "ui")
+ watchdog.begin_step("model picker: open + drive search bar")
+ buf = io.StringIO()
+ with contextlib.redirect_stderr(buf):
+ try:
+ raise TimeoutError("Timeout 30000ms exceeded.\nCall log: ...")
+ except TimeoutError:
+ sys.excepthook(*sys.exc_info())
+ finally:
+ sys.excepthook = original
+ assert seen == [TimeoutError], "the previous hook must still print the traceback"
+ line = buf.getvalue().strip()
+ assert line.startswith("[ui] FAIL in step 'model picker: open + drive search bar' after ")
+ assert line.endswith("TimeoutError: Timeout 30000ms exceeded."), line
+
+
+def test_no_step_begun_adds_nothing():
+ watchdog = _WallClockWatchdog(10.0, lambda: None)
+ original = sys.excepthook
+ try:
+ sys.excepthook = lambda *a: None
+ robust.report_failing_step(watchdog, label = "ui")
+ buf = io.StringIO()
+ with contextlib.redirect_stderr(buf):
+ sys.excepthook(ValueError, ValueError("x"), None)
+ finally:
+ sys.excepthook = original
+ assert buf.getvalue() == ""
+
+
+def test_wait_until_returns_the_first_truthy_value():
+ values = iter([None, 0, [], "ready"])
+ assert (
+ robust.wait_until(lambda: next(values), timeout_s = 5, what = "x", interval_s = 0.01) == "ready"
+ )
+
+
+def test_wait_until_names_what_it_waited_for():
+ started = time.monotonic()
+ try:
+ robust.wait_until(lambda: 0, timeout_s = 0.3, what = "monitor resumes polling", interval_s = 0.05)
+ except TimeoutError as exc:
+ assert "monitor resumes polling" in str(exc)
+ assert "last value 0" in str(exc)
+ else:
+ raise AssertionError("wait_until returned for a predicate that never came true")
+ assert time.monotonic() - started < 2.0
+
+
+def test_wait_until_pauses_through_the_page_when_given_one():
+ # Event handlers only run inside a Playwright call, so the pause must be one.
+ page = mock.Mock()
+ values = iter([False, True])
+ robust.wait_until(lambda: next(values), timeout_s = 5, what = "x", interval_s = 0.25, page = page)
+ page.wait_for_timeout.assert_called_once_with(250.0)
+
+
+def test_step_budgets_stretch_but_never_shrink(monkeypatch):
+ monkeypatch.delenv(robust.STEP_BUDGET_SCALE_ENV, raising = False)
+ assert robust.step_budget_s(60) == 60.0
+ monkeypatch.setenv(robust.STEP_BUDGET_SCALE_ENV, "3")
+ assert robust.step_budget_s(60) == 180.0
+ monkeypatch.setenv(robust.STEP_BUDGET_SCALE_ENV, "0.1")
+ assert robust.step_budget_s(60) == 60.0
+ monkeypatch.setenv(robust.STEP_BUDGET_SCALE_ENV, "fast")
+ assert robust.step_budget_s(60) == 60.0
diff --git a/tests/studio/test_wait_for_settled.py b/tests/studio/test_wait_for_settled.py
new file mode 100644
index 00000000000..f7a5f93d8c9
--- /dev/null
+++ b/tests/studio/test_wait_for_settled.py
@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+"""`wait_for_settled` replaces "sleep N ms, then measure" in the Playwright smokes.
+
+The fixed sleep is wrong in both directions: on a fast box it waits long after the
+transition ended, and on a loaded runner it measures a box mid-flight. The helper has to
+return once the element is still, keep waiting while it moves, and time out on an
+element that never stops. Run against a real headless Chromium; skipped where there is
+no Playwright or no browser build.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from _playwright_robust import chromium_launch_args, wait_for_settled # noqa: E402
+
+sync_api = pytest.importorskip("playwright.sync_api")
+
+
[email protected](scope = "module")
+def page():
+ with sync_api.sync_playwright() as p:
+ try:
+ browser = p.chromium.launch(headless = True, args = chromium_launch_args())
+ except Exception as exc: # no browser build on this machine
+ pytest.skip(f"chromium unavailable: {exc}")
+ page = browser.new_page()
+ yield page
+ browser.close()
+
+
+def test_a_still_element_settles_at_once(page):
+ page.set_content('<div id="box" style="width:100px;height:40px">x</div>')
+ started = time.monotonic()
+ wait_for_settled(page.locator("#box"), timeout_ms = 5_000)
+ assert time.monotonic() - started < 2.0
+
+
+def test_it_waits_for_a_transition_to_finish(page):
+ page.set_content(
+ '<div id="box" style="width:100px;height:40px;transition:width 900ms linear">x</div>'
+ )
+ page.eval_on_selector("#box", "el => { el.getBoundingClientRect(); el.style.width = '400px'; }")
+ wait_for_settled(page.locator("#box"), timeout_ms = 10_000)
+ assert page.eval_on_selector("#box", "el => el.getBoundingClientRect().width") == 400
+
+
+def test_an_element_that_never_stops_times_out(page):
+ page.set_content(
+ "<style>@keyframes slide { from { margin-left: 0 } to { margin-left: 300px } }</style>"
+ '<div id="box" style="width:50px;height:40px;animation:slide 400ms linear infinite">x</div>'
+ )
+ with pytest.raises(sync_api.TimeoutError):
+ wait_for_settled(page.locator("#box"), timeout_ms = 1_500)
+
+
+def test_a_node_replaced_mid_wait_is_looked_up_again(page):
+ # A re-render that swaps the node must not strand the wait on the detached one.
+ page.set_content('<div id="wrap"><div id="box" style="width:100px;height:40px">x</div></div>')
+ page.evaluate(
+ """() => setTimeout(() => {
+ document.getElementById("wrap").innerHTML =
+ '<div id="box" style="width:220px;height:40px">y</div>';
+ }, 30)"""
+ )
+ wait_for_settled(page.locator("#box"), frames = 20, timeout_ms = 5_000)
+ assert page.eval_on_selector("#box", "el => el.textContent") == "y"