Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Playwright: condition waits and per-step budgets in extra_ui and update_banner_layout (#11990)

Daniel Han5aa19d3bbc82
diff --git a/tests/studio/playwright_extra_ui.py b/tests/studio/playwright_extra_ui.py
index 03050d70ba9..cb345c8106f 100644
--- a/tests/studio/playwright_extra_ui.py
+++ b/tests/studio/playwright_extra_ui.py
@@ -12,7 +12,7 @@ import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
-from playwright.sync_api import sync_playwright
+from playwright.sync_api import expect, sync_playwright
# Run as a plain script (not via pytest), so prepend the dir to sys.path.
sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -24,7 +24,9 @@ from _playwright_robust import ( # noqa: E402
install_wall_clock_watchdog,
is_benign_page_error,
recover_or_replace_page,
+ report_failing_step,
robust_evaluate,
+ step_budget_s,
wait_for_first,
wait_for_health,
click_forced,
@@ -71,12 +73,26 @@ WHEEL_DEADLINE_S = 30.0
# indefinitely.
WHEEL_DEADLINE_MAX_S = 75.0
+# Per-step ceilings. A step that overruns its own stops the run there, named, instead of the
+# steps after it each waiting out their own timeouts until the 720s wall. The largest step
+# (Compare: two 60s bubble waits plus a 60s composer mount; Settings: up to 75s of wheel
+# wait) fits in a quarter of this. Setup retries itself and has no ceiling of its own; the
+# whole-run wall still bounds it. Both stretch with STUDIO_UI_TURN_TIMEOUT_MS, the knob the
+# slow lanes already raise, and with STUDIO_PW_STEP_BUDGET_SCALE.
+_SLOW_LANE = max(1.0, TURN_TIMEOUT_MS / 180_000)
+STEP_BUDGET_S = step_budget_s(240 * _SLOW_LANE)
+NO_STEP_CEILING = 0
+
_n = [0]
_failed: list[str] = []
+_watchdog = None # armed below
-def step(s: str) -> None:
+def step(s: str, budget_s: float | None = None) -> None:
+ """Start step `s`; it may run `budget_s` (default STEP_BUDGET_S) before the run stops."""
print(f"[ui-extra] STEP {s}", flush = True)
+ if _watchdog is not None:
+ _watchdog.begin_step(s, STEP_BUDGET_S if budget_s is None else budget_s)
def info(s: str) -> None:
@@ -113,11 +129,15 @@ def page_crashed(pg, exc: Exception) -> bool:
with sync_playwright() as p:
+ # The wall stays a whole-run cap (this script never kicked it): begin_step() restarts the
+ # inactivity budget, so the same number is also passed as the total no step can move.
_watchdog = install_wall_clock_watchdog(
WALL_TIMEOUT_S,
label = "ui-extra",
info = info,
+ total_deadline_s = WALL_TIMEOUT_S,
)
+ report_failing_step(_watchdog, label = "ui-extra")
# Health pre-flight: bash-side health wait can pass before the auth DB migrates on macos-14.
wait_for_health(BASE, timeout = 30.0, info = info)
# Chromium launch args: see tests/studio/_playwright_robust.py.
@@ -243,7 +263,7 @@ with sync_playwright() as p:
info(f"WARN: screenshot {name} failed: {_shoot_err}")
# Setup: change-password through the UI + model load.
- step("setup: change-password + model load")
+ step("setup: change-password + model load", NO_STEP_CEILING)
# 3-attempt retry: form re-renders mid-fill on macos-14 can detach the password fields.
form_err: Exception | None = None
for _form_attempt in range(3):
@@ -440,10 +460,17 @@ with sync_playwright() as p:
)
soft_fail(f"Compare nav not found: {missing} never appeared")
else:
- page.wait_for_timeout(1500)
+ # The compare view and its composer, instead of a fixed 1.5 s after the click.
+ view = wait_for_first(page.locator('[data-tour="chat-compare-view"]'), timeout_ms = 15_000)
+ if view is not None:
+ wait_for_first(
+ page.get_by_placeholder(re.compile(r"Send to both models", re.I)).or_(
+ view.locator("textarea")
+ ),
+ timeout_ms = 10_000,
+ )
shoot("02-compare-opened")
- view = page.locator('[data-tour="chat-compare-view"]').first
- if view.count() == 0:
+ if view is None:
soft_fail("[data-tour='chat-compare-view'] not found after Compare click")
else:
ok_count_before = len(page.locator('[data-role="assistant"]').all())
@@ -463,6 +490,7 @@ with sync_playwright() as p:
cmp_composer.press("Enter")
# Expect 2 new assistant bubbles (one per pane). Panes have no explicit model in this CI flow so the
# backend may reject; downgrade to runtime_warn but keep the structural assertions.
+ first_pair_arrived = False
try:
page.wait_for_function(
"""(want) => {
@@ -474,6 +502,7 @@ with sync_playwright() as p:
timeout = 60_000,
)
info("OK Compare: 2 new assistant bubbles after first prompt")
+ first_pair_arrived = True
except Exception as exc:
runtime_warn(
f"Compare: 2 bubbles didn't appear (panes likely "
@@ -481,26 +510,32 @@ with sync_playwright() as p:
)
shoot("03-compare-after-A")
- # Second prompt -> 4 total new bubbles (same runtime-flaky caveat).
- cmp_composer.fill("Reply with: B")
- cmp_composer.press("Enter")
- try:
- page.wait_for_function(
- """(want) => {
- return document.querySelectorAll(
- '[data-role="assistant"]'
- ).length >= want;
- }""",
- arg = ok_count_before + 4,
- timeout = 60_000,
- )
- info("OK Compare: 4 total new assistant bubbles after second prompt")
- except Exception as exc:
+ # Second prompt -> 4 total new bubbles (same runtime-flaky caveat). Only worth waiting for when the
+ # first pair came: panes that answered nothing in 60 s will not answer twice as much in the next 60.
+ if not first_pair_arrived:
runtime_warn(
- f"Compare: 4 bubbles didn't appear (panes likely "
- f"have no model selected): {exc!r}"
+ "Compare: skipped the second prompt's 60s wait; the first prompt's bubbles never appeared"
)
- shoot("04-compare-after-B")
+ else:
+ cmp_composer.fill("Reply with: B")
+ cmp_composer.press("Enter")
+ try:
+ page.wait_for_function(
+ """(want) => {
+ return document.querySelectorAll(
+ '[data-role="assistant"]'
+ ).length >= want;
+ }""",
+ arg = ok_count_before + 4,
+ timeout = 60_000,
+ )
+ info("OK Compare: 4 total new assistant bubbles after second prompt")
+ except Exception as exc:
+ runtime_warn(
+ f"Compare: 4 bubbles didn't appear (panes likely "
+ f"have no model selected): {exc!r}"
+ )
+ shoot("04-compare-after-B")
# Back to single chat for subsequent steps.
page.goto(f"{BASE}/chat")
@@ -510,7 +545,8 @@ with sync_playwright() as p:
# 2. Recipes editor.
step("Recipes editor: click first template + Preview dialog")
page.goto(f"{BASE}/data-recipes")
- page.wait_for_timeout(1500)
+ # The route is rendered once its template list is, not after a fixed 1.5 s.
+ wait_for_first(page.locator('[data-tour="recipes-templates"] button'), timeout_ms = 15_000)
shoot("05-recipes-list")
# Template cards render as <button> elements.
templates = page.locator("main button").filter(has_not_text = re.compile(r"^(\+|Create)"))
@@ -522,7 +558,15 @@ with sync_playwright() as p:
try:
templates.first.scroll_into_view_if_needed()
templates.first.click()
- page.wait_for_timeout(2000)
+ # What the click opens: the React-Flow editor, a dialog, or (for the first `main button`, which is the
+ # "New Recipe" trigger) its menu. Wait for whichever arrives, not a fixed 2 s.
+ wait_for_first(
+ page.locator(
+ ".react-flow__renderer, .react-flow, [data-testid*='react-flow'], "
+ "[role='dialog'], [role='menu']"
+ ),
+ timeout_ms = 15_000,
+ )
shoot("06-recipe-opened")
# The recipe-studio canvas uses React-Flow; look for the renderer.
canvas = page.locator(
@@ -539,7 +583,13 @@ with sync_playwright() as p:
# 3. Export route.
step(f"Export route ({'chat-only self-gated' if chat_only else 'form fields'})")
page.goto(f"{BASE}/export")
- page.wait_for_timeout(1500)
+ # Either the export form's CTA or the chat-only gate, instead of a fixed 1.5 s.
+ wait_for_first(
+ page.locator('[data-tour="export-cta"]').or_(
+ page.get_by_text(re.compile(r"Export unavailable", re.I))
+ ),
+ timeout_ms = 15_000,
+ )
shoot("07-export")
if chat_only:
if "/export" not in page.url:
@@ -557,22 +607,14 @@ with sync_playwright() as p:
soft_fail("[data-tour='export-cta'] not found in /export")
else:
info("OK [data-tour='export-cta'] visible")
- # HF-token field is lazy-loaded behind a disclosure; poll for ~8s and log at info (non-blocking).
- hf_token = None
- for _try in range(8):
- page.wait_for_timeout(1000)
- for cand in (
- page.get_by_placeholder(re.compile(r"hf[_\\.\\-]", re.I)).first,
- page.locator(
- 'input[placeholder*="token" i], input[placeholder*="huggingface" i]'
- ).first,
- page.locator('input[name="hf_token"], input[id*="hf-token"]').first,
- ):
- if cand.count() > 0:
- hf_token = cand
- break
- if hf_token is not None:
- break
+ # HF-token field is lazy-loaded behind a disclosure; wait up to the same ~8s for any of its locators and log
+ # at info (non-blocking). Ends as soon as one is there instead of polling once a second.
+ hf_token = wait_for_first(
+ page.get_by_placeholder(re.compile(r"hf[_\\.\\-]", re.I))
+ .or_(page.locator('input[placeholder*="token" i], input[placeholder*="huggingface" i]'))
+ .or_(page.locator('input[name="hf_token"], input[id*="hf-token"]')),
+ timeout_ms = 8_000,
+ )
if hf_token is not None:
info("OK HF token input visible")
else:
@@ -585,7 +627,15 @@ with sync_playwright() as p:
# 4. Unsloth training route.
step(f"Unsloth route ({'chat-only redirect' if chat_only else 'tabs + sections'})")
page.goto(f"{BASE}/studio")
- page.wait_for_timeout(1500)
+ # The redirect (chat-only) or the training page's own anchors, instead of a fixed 1.5 s.
+ if chat_only:
+ try:
+ page.wait_for_url(lambda u: "/studio" not in u, timeout = 15_000)
+ except Exception:
+ pass # the check below reports it
+ else:
+ wait_for_first(page.locator('[data-tour="studio-params"]'), timeout_ms = 15_000)
+ wait_for_first(page.get_by_role("tab"), timeout_ms = 5_000)
shoot("08-studio")
if chat_only:
if "/studio" in page.url:
@@ -619,14 +669,13 @@ with sync_playwright() as p:
info("OK Chat Dictate control is type=button")
page.keyboard.press("Control+,")
- page.wait_for_timeout(800)
- settings = page.get_by_role("dialog").first
- if settings.count() == 0:
+ # The dialog itself, not 800 ms; five seconds per shortcut is still far past any render.
+ settings = wait_for_first(page.get_by_role("dialog"), timeout_ms = 5_000)
+ if settings is None:
# macOS shortcut is Cmd-,.
page.keyboard.press("Meta+,")
- page.wait_for_timeout(800)
- settings = page.get_by_role("dialog").first
- if settings.count() == 0:
+ settings = wait_for_first(page.get_by_role("dialog"), timeout_ms = 5_000)
+ if settings is None:
soft_fail("Settings dialog didn't open with Cmd/Ctrl-,")
else:
shoot("09-settings-open")
@@ -650,7 +699,20 @@ with sync_playwright() as p:
continue
try:
btn.click()
- page.wait_for_timeout(400)
+ # The active tab carries the animated pill; its content renders in the same commit. Then give the body
+ # up to 5 s to fill, instead of reading it 400 ms after the click. Both are best-effort: the length
+ # check below still decides.
+ try:
+ btn.locator("span.bg-accent").first.wait_for(state = "attached", timeout = 5_000)
+ page.wait_for_function(
+ """() => {
+ const dialog = document.querySelector('[role="dialog"]');
+ return !!dialog && (dialog.innerText || '').trim().length > 30;
+ }""",
+ timeout = 5_000,
+ )
+ except Exception:
+ pass
# Tab body must be non-empty.
body_text = page.evaluate(
"""() => {
@@ -873,8 +935,16 @@ with sync_playwright() as p:
# cosmetic teardown rather than re-raise TargetClosedError on it.
if not page.is_closed():
shoot("10-settings-tabs-visited")
- page.keyboard.press("Escape")
- page.wait_for_timeout(300)
+ # Cosmetic teardown, best-effort as the fixed pause was. The first Escape may only close the voice
+ # picker's popover, so a dialog still up after it gets a second one.
+ dialogs = page.get_by_role("dialog")
+ for _escape in range(2):
+ page.keyboard.press("Escape")
+ try:
+ expect(dialogs).to_have_count(0, timeout = 2_000)
+ break
+ except AssertionError:
+ pass
info(f"visited Settings tabs: {seen_tabs}")
if not seen_tabs:
soft_fail("no Settings tabs were visitable")
diff --git a/tests/studio/playwright_update_banner_layout.py b/tests/studio/playwright_update_banner_layout.py
index db3e823061f..0ea21cf0c3c 100644
--- a/tests/studio/playwright_update_banner_layout.py
+++ b/tests/studio/playwright_update_banner_layout.py
@@ -30,6 +30,7 @@ from __future__ import annotations
import json
import os
import sys
+import time
import urllib.error
import urllib.request
from pathlib import Path
@@ -41,7 +42,10 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
from _playwright_robust import ( # noqa: E402
chromium_launch_args,
install_wall_clock_watchdog,
+ report_failing_step,
+ step_budget_s,
wait_for_health,
+ wait_for_settled,
)
# The wall this suite did not have, matching playwright_chat_ui.py and playwright_extra_ui.py. It has six raw
@@ -61,8 +65,9 @@ PLAYWRIGHT_CHANNEL = os.environ.get("STUDIO_PLAYWRIGHT_CHANNEL") or None
# The web check fires 5s after mount and the llama.cpp one after 1s; this is the ceiling on waiting for them, not the
# wait itself.
SETTLE_MS = int(os.environ.get("STUDIO_UI_BANNER_SETTLE_MS", "9000"))
-# What the cards need after they mount, to animate in and lay out.
-SETTLED_MS = int(os.environ.get("STUDIO_UI_BANNER_SETTLED_MS", "900"))
+# The ceiling on the cards animating in and laying out after they mount. Waited for as a condition (every card holds
+# one box with nothing animating for a few frames), so this is how long a card may take, not how long each one does.
+SETTLED_MS = int(os.environ.get("STUDIO_UI_BANNER_SETTLED_MS", "10000"))
# Must match the name use-web-update-check.ts reads. Kept short rather than zero so the card still arrives after
# first paint, which is the situation the layout checks exist for.
@@ -232,9 +237,15 @@ RESIZE_SWEEP = [
(320, 568),
]
-# What a resize needs before it has settled: the ResizeObserver, the placement it feeds, and the reflow after that.
-# Nothing is fetched, so this is short.
-RESIZE_SETTLE_MS = int(os.environ.get("STUDIO_UI_BANNER_RESIZE_MS", "700"))
+# The ceiling on a resize settling: the ResizeObserver, the placement it feeds, and the reflow after that. Waited for
+# as a condition, like SETTLED_MS.
+RESIZE_SETTLE_MS = int(os.environ.get("STUDIO_UI_BANNER_RESIZE_MS", "10000"))
+
+# Per-phase ceilings. A phase that overruns its own stops the run there, named, instead of every later phase waiting out
+# its own timeouts until the 720s wall. A viewport pass boots three routes at up to SETTLE_MS each; the resize sweep
+# walks 23 sizes. Stretched by STUDIO_PW_STEP_BUDGET_SCALE for slower lanes.
+PHASE_BUDGET_S = step_budget_s(180)
+SWEEP_BUDGET_S = step_budget_s(420)
# The four runtime status endpoints the loaded models card reads, shaped as
# tests/studio/playwright_loaded_models_indicator.py has them. Only chat holds anything: one loaded model is all it
@@ -372,12 +383,20 @@ APPEARANCE_STORE_VERSION = 5
failures: list[str] = []
checks = [0]
+_watchdog = [None]
def info(s: str) -> None:
print(f"[banner] {s}", flush = True)
+def phase(name: str, budget_s: float = PHASE_BUDGET_S) -> None:
+ """Start phase `name`; it may run `budget_s` before the run stops, naming it."""
+ info(f"STEP {name}")
+ if _watchdog[0] is not None:
+ _watchdog[0].begin_step(name, budget_s)
+
+
def check(
name: str,
ok: bool,
@@ -867,9 +886,42 @@ def settle_stack(
seen = now
if stable >= 2:
return
+ # Kept: the poll interval of this stability loop, which already ends as soon as the rail holds still.
page.wait_for_timeout(gap_ms)
+def settle_cards(page, timeout_ms: int = SETTLED_MS) -> None:
+ """Wait for the rail and each card in it to hold one box with nothing animating.
+
+ Replaces a fixed pause after mount or resize: it ends as soon as the stack is still, and on a runner slow enough
+ that the pause was not enough it keeps waiting instead of measuring mid-transition. A stack that never settles is
+ reported and measured anyway, so the checks, not this wait, say what is wrong with it.
+ """
+ for selector in (
+ '[data-testid="overlay-rail"]',
+ '[data-testid="web-update-banner"]',
+ '[data-testid="llama-update-banner"]',
+ ):
+ target = page.locator(selector)
+ if target.count() == 0:
+ continue
+ # In short slices, each on a freshly resolved element: wait_for_settled holds one element handle, and a card
+ # React remounts mid-wait leaves that handle detached, which would read as "never settled" until the timeout.
+ deadline = time.monotonic() + timeout_ms / 1000
+ while True:
+ remaining_ms = int((deadline - time.monotonic()) * 1000)
+ if remaining_ms <= 0:
+ info(f"WARN {selector} did not settle within {timeout_ms}ms; measuring anyway")
+ break
+ if target.count() == 0:
+ break
+ try:
+ wait_for_settled(target, timeout_ms = min(2_000, remaining_ms))
+ break
+ except PlaywrightTimeoutError:
+ continue
+
+
def boot(page, path: str) -> None:
page.goto(f"{BASE}{path}", wait_until = "domcontentloaded")
# Both cards are on a timer, so wait for them rather than for the worst case: this step runs 24 times and the job it
@@ -883,7 +935,7 @@ def boot(page, path: str) -> None:
# or where.
pass
# The banners animate in, and a box measured mid-transition is not the box.
- page.wait_for_timeout(SETTLED_MS)
+ settle_cards(page)
settle_stack(page)
landed = page.evaluate("location.pathname")
if landed.startswith(("/login", "/change-password")):
@@ -1013,11 +1065,15 @@ def main() -> int:
return 1
with sync_playwright() as p:
- install_wall_clock_watchdog(
+ # begin_step() restarts the inactivity budget, so the same number is also passed as the total no phase can
+ # move: the wall stays the whole-run cap it always was.
+ _watchdog[0] = install_wall_clock_watchdog(
WALL_TIMEOUT_S,
label = "ui-update-banner",
info = info,
+ total_deadline_s = WALL_TIMEOUT_S,
)
+ report_failing_step(_watchdog[0], label = "ui-update-banner")
launch_kwargs: dict = {"headless": True}
if PLAYWRIGHT_BROWSER == "chromium":
launch_kwargs["args"] = chromium_launch_args()
@@ -1030,6 +1086,7 @@ def main() -> int:
llama_payload = [LLAMA_STATUS]
for width, height in VIEWPORTS[2:5] if SPOT else VIEWPORTS:
+ phase(f"update cards at {width}x{height}")
context = browser.new_context(
viewport = {"width": width, "height": height},
reduced_motion = "reduce",
@@ -1079,7 +1136,18 @@ def main() -> int:
toggle = page.locator('[data-testid="web-update-release-notes-toggle"]')
if toggle.count() == 1:
toggle.click()
- page.wait_for_timeout(1500)
+ # Expanded, then still, instead of a fixed 1.5 s.
+ try:
+ page.wait_for_selector(
+ '[data-testid="web-update-release-notes-toggle"][aria-expanded="true"]',
+ state = "attached",
+ timeout = 10_000,
+ )
+ except PlaywrightTimeoutError:
+ info(
+ f"WARN {size} {name}: the notes toggle never reported aria-expanded=true"
+ )
+ settle_cards(page)
measure(page, f"{size} {name} expanded")
toggle.click()
if path == "/":
@@ -1100,6 +1168,7 @@ def main() -> int:
# Exercise the compact app card omitted by the notes-bearing fixtures.
for width, height in NO_PREVIEW_VIEWPORTS[:1] if SPOT else NO_PREVIEW_VIEWPORTS:
+ phase(f"app card with no notes preview at {width}x{height}")
context = browser.new_context(
viewport = {"width": width, "height": height},
reduced_motion = "reduce",
@@ -1127,7 +1196,7 @@ def main() -> int:
if toggle.count() == 1:
toggle.click()
panel.wait_for(state = "visible", timeout = 10_000)
- page.wait_for_timeout(SETTLED_MS)
+ settle_cards(page)
measure(page, f"{width}x{height} with no preview, expanded")
context.close()
@@ -1136,6 +1205,7 @@ def main() -> int:
# the rail and move the whole stack. That is the case to check.
# #8346 ships it off by default, so nothing above this point sees it.
for width, height in INDICATOR_VIEWPORTS:
+ phase(f"loaded models indicator in the rail at {width}x{height}")
context = browser.new_context(
viewport = {"width": width, "height": height},
reduced_motion = "reduce",
@@ -1195,6 +1265,7 @@ def main() -> int:
browser.close()
return 1 if failures else 0
+ phase("resize sweep and restore cycles", SWEEP_BUDGET_S)
# One page, many window sizes.
# Every check the core matrix runs, at every resolution in RESIZE_SWEEP, for the price of one boot: the cards
# are already mounted and a resize is all a maximise or a restore ever is. It also exercises the path a fresh
@@ -1215,13 +1286,12 @@ def main() -> int:
boot(page, "/")
for width, height in RESIZE_SWEEP:
page.set_viewport_size({"width": width, "height": height})
- # Long enough for the ResizeObserver, the placement it feeds and the reflow that follows. Short because
- # nothing is being fetched.
- page.wait_for_timeout(RESIZE_SETTLE_MS)
+ # The ResizeObserver, the placement it feeds and the reflow that follows, waited for rather than slept.
+ settle_cards(page, RESIZE_SETTLE_MS)
settle_stack(page)
measure(page, f"{width}x{height} resized")
page.set_viewport_size({"width": 1280, "height": 830})
- page.wait_for_timeout(RESIZE_SETTLE_MS)
+ settle_cards(page, RESIZE_SETTLE_MS)
page.screenshot(path = str(ART / "resize-sweep-end.png"))
# Parked small and brought back. A minimised window cannot be photographed, but the restore is where a cached
@@ -1229,9 +1299,10 @@ def main() -> int:
# merely looking tidy.
for (small_w, small_h), (back_w, back_h) in RESTORE_CYCLES:
page.set_viewport_size({"width": small_w, "height": small_h})
- page.wait_for_timeout(RESIZE_SETTLE_MS)
+ # Parked long enough to have laid out small, or the restore restores nothing.
+ settle_cards(page, RESIZE_SETTLE_MS)
page.set_viewport_size({"width": back_w, "height": back_h})
- page.wait_for_timeout(RESIZE_SETTLE_MS)
+ settle_cards(page, RESIZE_SETTLE_MS)
restored = measure(page, f"{back_w}x{back_h} restored from {small_w}x{small_h}")
fresh_context = browser.new_context(
viewport = {"width": back_w, "height": back_h},
@@ -1273,6 +1344,7 @@ def main() -> int:
set_ui_font_size(session["access_token"], UI_FONT_SIZE_MAX)
try:
for width, height in FONT_SCALE_VIEWPORTS:
+ phase(f"{UI_FONT_SIZE_MAX}px type at {width}x{height}")
context = browser.new_context(
viewport = {"width": width, "height": height},
reduced_motion = "reduce",