Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

CI: wait on events instead of fixed sleeps in Playwright smokes, backend tests and Stop Unsloth steps (#13712)

Daniel Han1fdcdb5dfb8d
diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml
index 3ff82a32913..522237403dc 100644
--- a/.github/workflows/studio-api-smoke.yml
+++ b/.github/workflows/studio-api-smoke.yml
@@ -236,7 +236,9 @@ jobs:
if: always()
run: |
kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
+ # Wait for it to actually exit (was a flat `sleep 2`); SIGKILL after 10 s.
+ for _ in $(seq 1 100); do kill -0 "${STUDIO_PID}" 2>/dev/null || break; sleep 0.1; done
+ kill -9 "${STUDIO_PID}" 2>/dev/null || true
- name: Upload API smoke logs
if: always()
diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml
index b1521c5d2ca..07e9310b6b6 100644
--- a/.github/workflows/studio-inference-smoke.yml
+++ b/.github/workflows/studio-inference-smoke.yml
@@ -386,7 +386,9 @@ jobs:
if: always()
run: |
kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
+ # Wait for it to actually exit (was a flat `sleep 2`); SIGKILL after 10 s.
+ for _ in $(seq 1 100); do kill -0 "${STUDIO_PID}" 2>/dev/null || break; sleep 0.1; done
+ kill -9 "${STUDIO_PID}" 2>/dev/null || true
ss -tln | grep ":${STUDIO_PORT}" || true
- name: Upload logs
@@ -964,7 +966,9 @@ jobs:
if: always()
run: |
kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
+ # Wait for it to actually exit (was a flat `sleep 2`); SIGKILL after 10 s.
+ for _ in $(seq 1 100); do kill -0 "${STUDIO_PID}" 2>/dev/null || break; sleep 0.1; done
+ kill -9 "${STUDIO_PID}" 2>/dev/null || true
ss -tln | grep ":${STUDIO_PORT}" || true
# Capture backend + llama-server logs so a 500 has a server-side traceback.
mkdir -p logs/server-logs
@@ -1266,7 +1270,9 @@ jobs:
if: always()
run: |
kill "${STUDIO_PID}" 2>/dev/null || true
- sleep 2
+ # Wait for it to actually exit (was a flat `sleep 2`); SIGKILL after 10 s.
+ for _ in $(seq 1 100); do kill -0 "${STUDIO_PID}" 2>/dev/null || break; sleep 0.1; done
+ kill -9 "${STUDIO_PID}" 2>/dev/null || true
ss -tln | grep ":${STUDIO_PORT}" || true
- name: Upload logs
diff --git a/studio/backend/tests/test_cached_gguf_routes.py b/studio/backend/tests/test_cached_gguf_routes.py
index fc1cb614138..0c983d25e69 100644
--- a/studio/backend/tests/test_cached_gguf_routes.py
+++ b/studio/backend/tests/test_cached_gguf_routes.py
@@ -2353,9 +2353,12 @@ def test_gguf_variants_route_answers_when_a_header_read_never_returns(monkeypatc
"""One syscall that never returns cannot be interrupted from inside the walk, so the
route bounds it. Without that the listing waits on it and the picker has nothing to click."""
(tmp_path / "model-Q4_K_M.gguf").write_bytes(b"x")
+ # Hung until the test is over, not for a flat 5 s: an abandoned read outlives the test, and
+ # a later test's _drain_context_threads() then sat waiting for this sleep to run out.
+ release = threading.Event()
def hung_read(_path):
- time.sleep(5)
+ release.wait(5)
return 8192
monkeypatch.setattr("utils.models.gguf_metadata.read_gguf_context_length", hung_read)
@@ -2386,7 +2389,10 @@ def test_gguf_variants_route_answers_when_a_header_read_never_returns(monkeypatc
)
return answer, time.monotonic() - began
- result, elapsed = asyncio.run(drive())
+ try:
+ result, elapsed = asyncio.run(drive())
+ finally:
+ release.set()
assert [v.quant for v in result.variants] == ["Q4_K_M"]
assert result.context_length is None
assert elapsed < 3
@@ -4194,6 +4200,9 @@ def test_switching_cache_storage_does_not_join_a_stuck_scan(monkeypatch, tmp_pat
cache_home[0] = tmp_path / "healthyvol"
second = asyncio.ensure_future(GV.get_gguf_variants_answer("org/repo", offline = True))
done, _ = await asyncio.wait({second}, timeout = 1.5)
+ # Measured; now unwedge the old scan, or asyncio.run's executor shutdown joins its
+ # thread and the test sits out the rest of the 5 s wait.
+ wedged.set()
for task in (stuck, second):
task.cancel()
return bool(done)
diff --git a/studio/backend/tests/test_gpu_query_cache.py b/studio/backend/tests/test_gpu_query_cache.py
index 00fca399935..ecb1ead56fd 100644
--- a/studio/backend/tests/test_gpu_query_cache.py
+++ b/studio/backend/tests/test_gpu_query_cache.py
@@ -456,20 +456,29 @@ def test_a_child_the_kernel_cannot_reap_does_not_hold_the_caller(monkeypatch):
monkeypatch.delenv("UNSLOTH_GPU_QUERY_CACHE", raising = False)
monkeypatch.setenv("UNSLOTH_NVIDIA_LIBRARY_PROBE", "0")
calls = []
+ # Held until the test is done rather than for a flat 12 s: the worker stays stuck for as long
+ # as the caller is being measured, and teardown does not then wait the sleep out.
+ released = threading.Event()
def unreapable(argv, **kwargs):
calls.append(argv)
- time.sleep(12.0)
+ released.wait(12.0)
raise subprocess.TimeoutExpired(argv, kwargs.get("timeout"))
monkeypatch.setattr(subprocess, "run", unreapable)
- for _ in range(2):
- t0 = time.monotonic()
- out = nvidia.get_visible_gpu_utilization([0, 1])
- assert out["available"] is False
- assert time.monotonic() - t0 < 7.0
- assert len(calls) == 2
- gpu_query.reset()
+ # The caller's 5 s budget shrunk to 0.5 s: the bound under test is "timeout + 1 s reap grace,
+ # never the child's 12 s", and it holds at any timeout, so the test need not spend 2 x 6 s.
+ monkeypatch.setattr(gpu_query, "run_nvidia_smi", _with_timeout(gpu_query.run_nvidia_smi, 0.5))
+ try:
+ for _ in range(2):
+ t0 = time.monotonic()
+ out = nvidia.get_visible_gpu_utilization([0, 1])
+ assert out["available"] is False
+ assert time.monotonic() - t0 < 3.0
+ assert len(calls) == 2
+ finally:
+ released.set()
+ gpu_query.reset()
def test_slow_cli_answers_a_display_read_from_the_last_good_value(smi, monkeypatch):
diff --git a/studio/backend/tests/test_research_runs_hardening.py b/studio/backend/tests/test_research_runs_hardening.py
index 1f530183503..30aa3d026ef 100644
--- a/studio/backend/tests/test_research_runs_hardening.py
+++ b/studio/backend/tests/test_research_runs_hardening.py
@@ -1789,8 +1789,14 @@ class _QueuedThenSilentResponse:
# Queueing is not charged to the request budget, so with no wall clock behind it a backend
# that queues then goes quiet would hold the run open forever.
def test_unlimited_still_bounds_silence_after_a_queue_notice(monkeypatch):
+ from core.inference import llama_admission
+
_install_fake_client(monkeypatch, [_QueuedThenSilentResponse()])
monkeypatch.setattr(research_runs, "_MODEL_OUTPUT_IDLE_TIMEOUT_SECONDS", 0.2)
+ # The gap allowed between queue notices also clears 3 keepalive heartbeats (5 s each by
+ # default, so 15 s here). A 0.3 s heartbeat makes it the 1 s first-output budget instead;
+ # the bound under test is the same, the test just stops waiting 15 s for it.
+ monkeypatch.setenv(llama_admission.ADMISSION_KEEPALIVE_INTERVAL_ENV, "0.3")
supervisor = _make_supervisor(_noop_check_active)
run = _waiting_run(0)
run["config"]["budgets"]["firstOutputTimeoutSeconds"] = 1
diff --git a/tests/studio/playwright_find_in_page.py b/tests/studio/playwright_find_in_page.py
index 63d1d7e7dcc..3c346f27c9b 100644
--- a/tests/studio/playwright_find_in_page.py
+++ b/tests/studio/playwright_find_in_page.py
@@ -116,6 +116,7 @@ def settle(
condition: str,
*,
timeout: int = 10000,
+ arg = None,
) -> None:
"""Give `condition` until `timeout` to hold, and do not fail here if it never does.
@@ -126,11 +127,45 @@ def settle(
with the same message, so nothing is swallowed here.
"""
try:
- page.wait_for_function(condition, timeout = timeout)
+ page.wait_for_function(condition, arg = arg, timeout = timeout)
except PlaywrightTimeout:
pass
+def frames(page) -> None:
+ """Let the commit that just changed state reach layout and paint (two animation frames)."""
+ page.evaluate(
+ "() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))"
+ )
+
+
+# The counter is empty while a query is pending and reads `active/count` (`0/0` for no match)
+# once it has settled, so a non-empty counter is the bar's own "search done" signal.
+COUNTED = "() => !!window.__findSmoke.counter()"
+COUNTER_IS = "expected => window.__findSmoke.counter() === expected"
+SEARCH_HAS = "text => (document.querySelector('[role=\"search\"]')?.innerText ?? '').includes(text)"
+COUNTER_MOVED = "prev => { const c = window.__findSmoke.counter(); return !!c && c !== prev; }"
+
+
+def settled_counter(page) -> str | None:
+ """Wait for the typed query to settle and paint, then read its counter.
+
+ The first settled render can still show the previous query's results (the search runs in an
+ effect after it), so give that effect two more frames to land before reading."""
+ settle(page, COUNTED)
+ frames(page)
+ frames(page)
+ return counter(page)
+
+
+def step(page, key: str) -> None:
+ """Press a walk key and wait for the active match to move (every query here has many)."""
+ before = counter(page)
+ page.keyboard.press(key)
+ settle(page, COUNTER_MOVED, arg = before)
+ frames(page)
+
+
def close_bar(page) -> None:
page.keyboard.press("Escape")
settle(page, "() => !window.__findSmoke.state().open")
@@ -173,14 +208,14 @@ def check_chord(page, engine: str, mode: str, mod: str) -> None:
def check_counting(page, engine: str, mode: str, mod: str) -> None:
open_bar(page, mod)
page.keyboard.type("unsloth")
- page.wait_for_timeout(500)
- shown = counter(page)
+ shown = settled_counter(page)
check(engine, mode, "a query is counted", bool(shown), f"counter={shown!r}")
# An off-route workspace is kept mounted under `inert`; its matches must not
# be counted or walked to.
page.evaluate("() => window.__findSmoke.setWorkspace('other')")
- page.wait_for_timeout(700)
+ settle(page, COUNTER_MOVED, arg = shown)
+ frames(page)
other = counter(page)
check(
engine,
@@ -190,7 +225,8 @@ def check_counting(page, engine: str, mode: str, mod: str) -> None:
f"{shown!r} -> {other!r}",
)
page.evaluate("() => window.__findSmoke.setWorkspace('chat')")
- page.wait_for_timeout(700)
+ settle(page, COUNTER_IS, arg = shown)
+ frames(page)
check(
engine,
mode,
@@ -216,9 +252,19 @@ def check_close_preserves_position(page, engine: str, mode: str, mod: str) -> No
search_page.wait_for_function("() => !window.__findSmoke.counter()")
field.fill("unsloth")
search_page.wait_for_function("() => !!window.__findSmoke.counter()")
+ start = search_page.evaluate("() => window.__findSmoke.counter()")
for _ in range(10):
field.press("Enter")
- search_page.wait_for_timeout(300)
+ # Wait for all ten steps to land: the active match ten places on, wrapping.
+ settle(
+ search_page,
+ """start => {
+ const [a, n] = start.split('/').map(Number);
+ return window.__findSmoke.counter() === `${((a - 1 + 10) % n) + 1}/${n}`;
+ }""",
+ arg = start,
+ )
+ frames(search_page)
before = search_page.evaluate("() => window.__findSmoke.scrollTop()")
if dismiss == "Escape":
close_bar(search_page)
@@ -250,11 +296,10 @@ def check_close_preserves_position(page, engine: str, mode: str, mod: str) -> No
def check_walk(page, engine: str, mode: str, mod: str) -> None:
open_bar(page, mod)
page.keyboard.type("unsloth")
- page.wait_for_timeout(500)
+ settled_counter(page)
start = page.evaluate("() => window.__findSmoke.scrollTop()")
for _ in range(6):
- page.keyboard.press("Enter")
- page.wait_for_timeout(120)
+ step(page, "Enter")
moved = page.evaluate("() => window.__findSmoke.scrollTop()")
check(
engine,
@@ -265,8 +310,7 @@ def check_walk(page, engine: str, mode: str, mod: str) -> None:
)
# Shift+Enter walks back.
- page.keyboard.press("Shift+Enter")
- page.wait_for_timeout(200)
+ step(page, "Shift+Enter")
check(
engine,
mode,
@@ -280,14 +324,28 @@ def check_walk(page, engine: str, mode: str, mod: str) -> None:
def check_streaming(page, engine: str, mode: str, mod: str) -> None:
open_bar(page, mod)
page.keyboard.type("unsloth")
- page.wait_for_timeout(500)
+ settled_counter(page)
for _ in range(4):
- page.keyboard.press("Enter")
- page.wait_for_timeout(120)
+ step(page, "Enter")
before_scroll = page.evaluate("() => window.__findSmoke.scrollTop()")
before_count = counter(page)
- page.evaluate("() => window.__findSmoke.stream('a streamed unsloth reply', 5)")
- page.wait_for_timeout(1600)
+ reply = "a streamed unsloth reply"
+ page.evaluate("text => window.__findSmoke.stream(text, 5)", reply)
+ # Done streaming, and the reindex that counts the new match has landed.
+ settle(
+ page,
+ """([text, prev]) => {
+ const rows = document.querySelectorAll('[data-streamed]');
+ const last = rows[rows.length - 1];
+ const c = window.__findSmoke.counter();
+ return last?.textContent === text && !!c && c !== prev;
+ }""",
+ arg = [reply, before_count],
+ )
+ # Observation window, not a readiness guess: one more reindex interval (300 ms) must pass
+ # without the reader being scrolled, since a later reindex is where a jump would come from.
+ page.wait_for_timeout(350)
+ frames(page)
check(
engine,
mode,
@@ -300,7 +358,7 @@ def check_streaming(page, engine: str, mode: str, mod: str) -> None:
mode,
"a streamed reply does not move the reader",
page.evaluate("() => window.__findSmoke.scrollTop()") == before_scroll,
- f"scrollTop {before_scroll} -> " f"{page.evaluate('() => window.__findSmoke.scrollTop()')}",
+ f"scrollTop {before_scroll} -> {page.evaluate('() => window.__findSmoke.scrollTop()')}",
)
close_bar(page)
@@ -310,7 +368,7 @@ def check_paint_and_teardown(page, engine: str, mode: str, mod: str) -> None:
has_api = page.evaluate("() => typeof CSS !== 'undefined' && !!CSS.highlights")
open_bar(page, mod)
page.keyboard.type("unsloth")
- page.wait_for_timeout(500)
+ settled_counter(page)
if has_api:
painted = page.evaluate(
@@ -365,8 +423,13 @@ def check_paint_and_teardown(page, engine: str, mode: str, mod: str) -> None:
def check_modal_gate(page, engine: str, mode: str, mod: str) -> None:
"""A modal backgrounds the scope, and the chord must go back to the browser."""
page.evaluate("() => window.__findSmoke.setModal(true)")
- page.wait_for_timeout(250)
+ settle(
+ page,
+ "() => document.querySelector('[data-find-scope]')?.getAttribute('aria-hidden') === 'true'",
+ )
page.keyboard.press(f"{mod}+f")
+ # Observation window, not a readiness guess: the bar must NOT open, so give it the time an
+ # open would take (its chunk is already loaded by the earlier probes).
page.wait_for_timeout(250)
check(
engine,
@@ -375,7 +438,7 @@ def check_modal_gate(page, engine: str, mode: str, mod: str) -> None:
state(page).get("open") is False,
)
page.evaluate("() => window.__findSmoke.setModal(false)")
- page.wait_for_timeout(250)
+ settle(page, "() => !document.querySelector('[data-find-scope]')?.hasAttribute('aria-hidden')")
open_bar(page, mod)
check(
engine,
@@ -414,11 +477,10 @@ def check_hidden_text(page, engine: str, mode: str, mod: str) -> None:
scope.appendChild(wrapper);
}"""
)
- page.wait_for_timeout(500)
+ frames(page)
open_bar(page, mod)
page.keyboard.type("zqxjkvbrmp")
- page.wait_for_timeout(700)
- shown = counter(page)
+ shown = settled_counter(page)
check(
engine,
mode,
@@ -461,15 +523,13 @@ def check_content_visibility_reveal(page, engine: str, mode: str, mod: str) -> N
planted == 140,
f"placeholder height={planted}",
)
- page.wait_for_timeout(600)
+ frames(page)
open_bar(page, mod)
# Inserted whole, the way a paste arrives, rather than typed. Typing hides this: every
# keystroke reveals again, and those repeats do by accident what the fix does on purpose.
# One change is what a paste, an Enter, or a click on the walk button each produce.
page.keyboard.insert_text("zqxjcvneedle")
- page.wait_for_timeout(1200)
- where = page.evaluate(
- """() => {
+ where_js = """() => {
const p = [...document.querySelectorAll('#cv-probe p')]
.find((n) => n.textContent.includes('zqxjcvneedle'));
const r = p.getBoundingClientRect();
@@ -478,7 +538,10 @@ def check_content_visibility_reveal(page, engine: str, mode: str, mod: str) -> N
real: Math.round(document.getElementById('cv-probe').getBoundingClientRect().height),
inViewport: r.top >= 0 && r.bottom <= window.innerHeight };
}"""
- )
+ # Wait for exactly what is asserted below: the block rendered and its match on screen.
+ settle(page, f"() => {{ const w = ({where_js})(); return w.real > 1000 && w.inViewport; }}")
+ frames(page)
+ where = page.evaluate(where_js)
check(
engine,
mode,
@@ -512,11 +575,12 @@ def check_spelling_variants(page, engine: str, mode: str, mod: str) -> None:
}"""
)
check(engine, mode, "the planted text really is decomposed", planted is True)
- page.wait_for_timeout(500)
+ frames(page)
open_bar(page, mod)
# Typed composed, against text written decomposed.
page.keyboard.type("caf\u00e9 vbnmqz")
- page.wait_for_timeout(700)
+ settle(page, COUNTER_IS, arg = "1/1")
+ frames(page)
shown = counter(page)
check(
engine,
@@ -551,7 +615,8 @@ def check_typing_burst(page, engine: str, mode: str, mod: str) -> None:
open_bar(page, mod)
field = page.locator('[role="search"] input')
field.fill("")
- page.wait_for_timeout(150)
+ settle(page, COUNTER_IS, arg = "")
+ frames(page)
page.evaluate(
"""() => {
window.__findSmokePaints = 0;
@@ -573,9 +638,8 @@ def check_typing_burst(page, engine: str, mode: str, mod: str) -> None:
)
page.keyboard.type("doing", delay = 10)
immediate = field.input_value()
- page.wait_for_timeout(400)
+ shown = settled_counter(page)
paints = page.evaluate("() => window.__findSmokePaints")
- shown = counter(page)
check(
engine,
mode,
@@ -604,16 +668,22 @@ def check_reverted_query_repaints(page, engine: str, mode: str, mod: str) -> Non
"() => !!window.__findSmoke.counter()",
timeout = 10000,
)
- page.wait_for_timeout(150)
+ frames(page)
before = page.evaluate(
"""() => CSS.highlights.has('unsloth-find') &&
CSS.highlights.has('unsloth-find-active')"""
)
field.press("End")
field.type("x")
+ # Deliberate: the Backspace has to land inside the 100 ms settle window.
page.wait_for_timeout(20)
field.press("Backspace")
- page.wait_for_timeout(250)
+ settle(
+ page,
+ """() => CSS.highlights.has('unsloth-find') && CSS.highlights.has('unsloth-find-active')
+ && !!window.__findSmoke.counter()""",
+ )
+ frames(page)
after = page.evaluate(
"""() => ({
painted: CSS.highlights.has('unsloth-find') &&
@@ -639,7 +709,7 @@ def check_pending_query_stays_unpainted(page, engine: str, mode: str, mod: str)
field = page.locator('[role="search"] input')
field.fill("unsloth")
page.wait_for_function("() => !!window.__findSmoke.counter()", timeout = 10000)
- page.wait_for_timeout(150)
+ frames(page)
field.press("End")
field.type("x")
page.evaluate(
@@ -669,7 +739,7 @@ def check_pending_query_stays_unpainted(page, engine: str, mode: str, mod: str)
and not pending["painted"],
f"pending state={pending}",
)
- page.wait_for_timeout(150)
+ settled_counter(page)
close_bar(page)
page.evaluate("() => document.getElementById('pending-repaint-probe')?.remove()")
@@ -688,7 +758,7 @@ def check_katex_mutation_reindexes(page, engine: str, mode: str, mod: str) -> No
scope.appendChild(host);
}"""
)
- page.wait_for_timeout(500)
+ frames(page)
open_bar(page, mod)
page.locator('[role="search"] input').fill("mutablekatex")
page.wait_for_function(
@@ -698,7 +768,8 @@ def check_katex_mutation_reindexes(page, engine: str, mode: str, mod: str) -> No
page.evaluate(
"() => { document.querySelector('#probe-katex-mutation .katex-html').textContent = 'changedkatex'; }"
)
- page.wait_for_timeout(900)
+ settle(page, COUNTER_IS, arg = "0/0")
+ frames(page)
after = counter(page)
check(
engine,
@@ -723,18 +794,18 @@ def check_skip_attribute(page, engine: str, mode: str, mod: str) -> None:
scope.appendChild(panel);
}"""
)
- page.wait_for_timeout(500)
+ frames(page)
open_bar(page, mod)
page.keyboard.type("wqzlmxtvbn")
- page.wait_for_timeout(700)
- before = counter(page)
+ before = settled_counter(page)
check(engine, mode, "the planted panel is counted", bool(before), f"{before!r}")
# Adding the attribute is the direction that used to be filtered out of the
# observer's own batch, leaving the region counted until something else
# happened to mutate.
page.evaluate("() => document.getElementById('probe-skip')?.setAttribute('data-find-skip', '')")
- page.wait_for_timeout(900)
+ settle(page, COUNTER_MOVED, arg = before)
+ frames(page)
after = counter(page)
check(
engine,
@@ -941,7 +1012,8 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
}));
}""",
)
- page.wait_for_timeout(500)
+ settle(page, SEARCH_HAS, arg = "1/8")
+ frames(page)
check(
engine,
mode,
@@ -969,7 +1041,8 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
"() => window.__findSmoke.stream("
"'definitely-no-such-match definitely-no-such-match', 1)",
)
- page.wait_for_timeout(1600)
+ settle(page, SEARCH_HAS, arg = "1/2")
+ frames(page)
check(
engine,
mode,
@@ -979,7 +1052,8 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
)
loaded = page.locator('[role="search"] input')
loaded.fill("unsloth")
- page.wait_for_timeout(500)
+ settle(page, SEARCH_HAS, arg = "1/28")
+ frames(page)
check(
engine,
mode,
@@ -1003,7 +1077,8 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
state = "detached",
timeout = max(15000, ENTRY_DELAY_MS + 10000),
)
- page.wait_for_timeout(500)
+ settle(page, SEARCH_HAS, arg = "1/28")
+ frames(page)
check(
engine,
mode,
@@ -1021,6 +1096,8 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
loading.wait_for(state = "visible", timeout = 5000)
field = loading.locator("input")
field.dispatch_event("compositionstart", {"data": "u"})
+ # Observation window, not a readiness guess: the composition has to hold the loading input
+ # mounted for the whole entry delay, so the wait spans it.
page.wait_for_timeout(ENTRY_DELAY_MS + 500)
check(
engine,
@@ -1073,6 +1150,7 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
"cancelable": True,
},
)
+ # Observation window: a composing Escape must NOT close the session.
page.wait_for_timeout(100)
check(
engine,
@@ -1090,7 +1168,7 @@ def run_entry_chunk_delay(browser, engine: str) -> None:
loading.wait_for(state = "visible", timeout = 5000)
page.locator("[data-find-scope]").click(position = {"x": 20, "y": 200})
page.keyboard.press("Escape")
- page.wait_for_timeout(100)
+ settle(page, "() => !document.querySelector('[role=\"search\"]')")
check(
engine,
mode,
diff --git a/tests/studio/playwright_nonmodal_menus.py b/tests/studio/playwright_nonmodal_menus.py
index 35a416d220e..13cd544e0fc 100644
--- a/tests/studio/playwright_nonmodal_menus.py
+++ b/tests/studio/playwright_nonmodal_menus.py
@@ -22,6 +22,7 @@ import os
import sys
from pathlib import Path
+from playwright.sync_api import TimeoutError as PlaywrightTimeout
from playwright.sync_api import sync_playwright
sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -79,6 +80,43 @@ def open_control(page) -> None:
page.wait_for_selector("[role=menu]", state = "visible")
+def until(
+ page,
+ condition: str,
+ arg = None,
+ timeout: int = 5000,
+) -> None:
+ """Wait until `condition` holds, without failing here: a broken build spends the timeout and
+ then fails on the SAME check below with the same message. Replaces flat sleeps, which asserted
+ the runner's speed along with the behaviour."""
+ try:
+ page.wait_for_function(condition, arg = arg, timeout = timeout)
+ except PlaywrightTimeout:
+ pass
+
+
+MENUS = "document.querySelectorAll('[role=menu]').length"
+
+
+def until_menus(page, expr: str) -> None:
+ """Wait for the open-menu count to satisfy `expr`, e.g. `== 0` or `>= 2`."""
+ until(page, f"() => {MENUS} {expr}")
+
+
+def frames(page) -> None:
+ """Let a scroll or focus change reach its listeners (two animation frames)."""
+ page.evaluate("() => new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))")
+
+
+def clicks_at_least(page, n: int) -> None:
+ until(page, "n => window.probe.documentClicks() >= n", arg = n)
+ frames(page)
+
+
+def focused_label(page, label: str) -> None:
+ until(page, "l => document.activeElement?.getAttribute('aria-label') === l", arg = label)
+
+
def raw_click(page, selector: str) -> None:
"""A real pointer press at the element's centre, with no actionability wait.
@@ -93,15 +131,16 @@ def raw_click(page, selector: str) -> None:
def close_all(page) -> None:
for _ in range(6):
- if menus_open(page) == 0:
+ before = menus_open(page)
+ if before == 0:
return
page.keyboard.press("Escape")
- page.wait_for_timeout(200)
+ until_menus(page, f"< {before}")
def reset_list(page) -> None:
page.evaluate("() => { document.getElementById('list').scrollTop = 0; }")
- page.wait_for_timeout(250)
+ frames(page)
def check_document_is_untouched(page, checks: Checks) -> None:
@@ -137,10 +176,12 @@ def check_scroll_closes(page, checks: Checks) -> None:
open_row(page, 1)
page.mouse.move(120, 300)
page.mouse.wheel(0, 400)
- page.wait_for_timeout(400)
+ until_menus(page, "== 0")
+ frames(page)
scrolled = page.evaluate("() => document.getElementById('list').scrollTop")
checks.record("a wheel behind an open menu closes it", menus_open(page) == 0)
checks.record("the list keeps the scroll the wheel gave it", scrolled > 0, scrolled)
+ # Observation window: a snap-back would arrive after the close, so give it time to.
page.wait_for_timeout(300)
settled = page.evaluate("() => document.getElementById('list').scrollTop")
checks.record(
@@ -152,19 +193,19 @@ def check_scroll_closes(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 2)
page.evaluate("() => { document.getElementById('list').scrollTop = 500; }")
- page.wait_for_timeout(400)
+ until_menus(page, "== 0")
checks.record("a programmatic scroll closes the menu", menus_open(page) == 0)
reset_list(page)
open_row(page, 3)
page.evaluate("() => document.getElementById('list').scrollBy({top: 200, behavior: 'smooth'})")
- page.wait_for_timeout(600)
+ until_menus(page, "== 0")
checks.record("a smooth scroll closes the menu", menus_open(page) == 0)
reset_list(page)
open_row(page, 6)
page.evaluate("() => window.scrollTo(0, 200)")
- page.wait_for_timeout(400)
+ until_menus(page, "== 0")
checks.record("a window scroll closes the menu", menus_open(page) == 0)
page.evaluate("() => window.scrollTo(0, 0)")
@@ -177,24 +218,28 @@ def check_scrolls_that_must_not_close(page, checks: Checks) -> None:
"() => { const v = document.querySelector('[data-slot=dropdown-menu-viewport]');"
" v.scrollTop = 120; }"
)
+ # Observation window: the menu must NOT close, so give a close the time it would take.
page.wait_for_timeout(400)
checks.record("scrolling the menu's own viewport leaves it open", menus_open(page) >= 1)
close_all(page)
open_row(page, 4)
page.evaluate("() => { document.getElementById('other').scrollTop = 300; }")
+ # Observation window: the menu must NOT close.
page.wait_for_timeout(400)
checks.record("scrolling an unrelated list leaves the menu open", menus_open(page) >= 1)
close_all(page)
open_row(page, 5)
page.get_by_test_id("export-5").click()
- page.wait_for_timeout(400)
+ until_menus(page, ">= 2")
+ frames(page)
with_sub = menus_open(page)
page.evaluate(
"() => { const vs = document.querySelectorAll('[data-slot=dropdown-menu-viewport]');"
" vs[vs.length - 1].scrollTop = 100; }"
)
+ # Observation window: neither menu may close.
page.wait_for_timeout(400)
checks.record(
"scrolling an open submenu leaves both menus open",
@@ -209,7 +254,8 @@ def check_dismiss_guard(page, checks: Checks) -> None:
open_row(page, 7)
page.evaluate("() => window.probe.resetClicks()")
raw_click(page, "#outside-button")
- page.wait_for_timeout(300)
+ until_menus(page, "== 0")
+ frames(page)
swallowed = page.evaluate("() => window.probe.documentClicks()")
checks.record(
"the click that dismisses a menu does not reach what is under it",
@@ -218,7 +264,7 @@ def check_dismiss_guard(page, checks: Checks) -> None:
)
page.evaluate("() => window.probe.resetClicks()")
raw_click(page, "#outside-button")
- page.wait_for_timeout(200)
+ clicks_at_least(page, 1)
delivered = page.evaluate("() => window.probe.documentClicks()")
checks.record("the click after that one is delivered", delivered == 1, delivered)
@@ -229,10 +275,11 @@ def check_dismiss_guard(page, checks: Checks) -> None:
for delay in EXIT_WINDOW_MS:
opener()
page.keyboard.press("Escape")
+ # Deliberate: the click has to land this far into the close's exit window.
page.wait_for_timeout(delay)
page.evaluate("() => window.probe.resetClicks()")
raw_click(page, "#outside-button")
- page.wait_for_timeout(250)
+ clicks_at_least(page, 1)
landed = page.evaluate("() => window.probe.documentClicks()")
checks.record(
f"a click {delay}ms after closing {label} is delivered", landed == 1, landed
@@ -248,7 +295,9 @@ def check_dismiss_guard(page, checks: Checks) -> None:
):
opener()
raw_click(page, "#outside-input")
- page.wait_for_timeout(700)
+ # Focus is restored (or not) when the closed menu unmounts after its exit animation.
+ until_menus(page, "== 0")
+ frames(page)
focus_after[label] = page.evaluate(
"() => document.activeElement?.getAttribute('data-slot')"
" ?? document.activeElement?.id ?? null"
@@ -267,7 +316,7 @@ def check_focus_return(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 10)
page.keyboard.press("Escape")
- page.wait_for_timeout(300)
+ focused_label(page, "Row options 10")
checks.record(
"Escape returns focus to the trigger that opened the menu",
page.evaluate(active) == "Row options 10",
@@ -279,11 +328,11 @@ def check_focus_return(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 11)
page.evaluate("() => { document.getElementById('list').scrollTop = 300; }")
- page.wait_for_timeout(400)
+ until_menus(page, "== 0")
reset_list(page)
open_row(page, 12)
page.keyboard.press("Escape")
- page.wait_for_timeout(300)
+ focused_label(page, "Row options 12")
checks.record(
"a scroll-close does not change how the next Escape restores focus",
page.evaluate(active) == "Row options 12",
@@ -297,9 +346,12 @@ def check_focus_return(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 14)
page.evaluate("() => { document.getElementById('list').scrollTop = 40; }")
+ # Deliberate: the click has to land while the scroll-closed menu is still animating out.
page.wait_for_timeout(20)
raw_click(page, "#outside-input")
- page.wait_for_timeout(900)
+ # The close-autofocus this guards against runs when the menu unmounts.
+ until_menus(page, "== 0")
+ frames(page)
landed = page.evaluate(
"() => document.activeElement?.getAttribute('data-slot')"
" ?? document.activeElement?.id ?? null"
@@ -313,7 +365,8 @@ def check_focus_return(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 13)
page.evaluate("() => { document.getElementById('list').scrollTop = 600; }")
- page.wait_for_timeout(500)
+ until_menus(page, "== 0")
+ frames(page)
checks.record(
"a scroll-close keeps the scroll it was closed by",
page.evaluate("() => document.getElementById('list').scrollTop") == 600,
@@ -326,23 +379,26 @@ def check_keyboard(page, checks: Checks) -> None:
# frame or two later, which races the open rather than testing it.
reset_list(page)
page.get_by_label("Row options 2", exact = True).focus()
- page.wait_for_timeout(250)
+ frames(page)
page.keyboard.press("Enter")
+ # Kept as a flat wait on purpose: a keyboard-opened menu drops an ArrowDown pressed right
+ # after it opens, even with its first item focused and its open animation finished (measured;
+ # no DOM state marks when it starts taking arrows), so there is no event to wait on here.
page.wait_for_timeout(300)
checks.record("Enter opens the menu", menus_open(page) >= 1)
page.keyboard.press("ArrowDown")
page.keyboard.press("ArrowRight")
- page.wait_for_timeout(400)
+ until_menus(page, ">= 2")
checks.record("ArrowRight opens the submenu", menus_open(page) >= 2, menus_open(page))
page.keyboard.press("ArrowLeft")
- page.wait_for_timeout(400)
+ until_menus(page, "== 1")
checks.record("ArrowLeft closes only the submenu", menus_open(page) == 1, menus_open(page))
close_all(page)
page.get_by_label("Row options 3", exact = True).focus()
- page.wait_for_timeout(250)
+ frames(page)
page.keyboard.press("Space")
- page.wait_for_timeout(300)
+ until_menus(page, ">= 1")
checks.record("Space opens the menu", menus_open(page) >= 1)
close_all(page)
@@ -351,14 +407,14 @@ def check_lifetime(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 16)
page.evaluate("() => window.probe.removeRow(16)")
- page.wait_for_timeout(500)
+ until_menus(page, "== 0")
checks.record(
"deleting the row while its menu is open strands no menu",
menus_open(page) == 0,
menus_open(page),
)
page.evaluate("() => window.probe.reset()")
- page.wait_for_timeout(300)
+ frames(page)
reopened: dict[str, int] = {}
reset_list(page)
@@ -368,8 +424,10 @@ def check_lifetime(page, checks: Checks) -> None:
):
opener()
page.keyboard.press("Escape")
+ # Deliberate: reopen inside the exit animation.
page.wait_for_timeout(30)
page.get_by_label(name, exact = True).click()
+ # Observation window: a second menu would show up in this time.
page.wait_for_timeout(500)
reopened[label] = menus_open(page)
close_all(page)
@@ -389,7 +447,8 @@ def check_lifetime(page, checks: Checks) -> None:
page.keyboard.press("Escape")
page.wait_for_timeout(60)
close_all(page)
- page.wait_for_timeout(400)
+ until_menus(page, "== 0")
+ frames(page)
after = page.evaluate("() => window.probe.scrollListeners()")
checks.record(
"30 open and close cycles leak no scroll listener", after == before, f"{before} -> {after}"
@@ -399,7 +458,7 @@ def check_lifetime(page, checks: Checks) -> None:
"() => { const l = document.getElementById('list');"
" l.scrollTop = 8 * 40 - l.clientHeight + 20; }"
)
- page.wait_for_timeout(300)
+ frames(page)
# Press the sliver the scroller leaves visible. Not click_forced: that scrolls the
# trigger into view first, which is the one thing this case must not do.
trigger = page.get_by_label("Row options 8", exact = True).bounding_box()
@@ -407,7 +466,9 @@ def check_lifetime(page, checks: Checks) -> None:
top = max(trigger["y"], clip["y"])
bottom = min(trigger["y"] + trigger["height"], clip["y"] + clip["height"])
page.mouse.click(trigger["x"] + trigger["width"] / 2, (top + bottom) / 2)
- page.wait_for_timeout(500)
+ until_menus(page, ">= 1")
+ # Observation window: the menu must also STAY open (a scroll-into-view would close it).
+ page.wait_for_timeout(300)
checks.record(
"a partly visible trigger opens a menu that stays open",
menus_open(page) >= 1,
@@ -418,6 +479,7 @@ def check_lifetime(page, checks: Checks) -> None:
reset_list(page)
open_row(page, 19)
page.get_by_label("Row options 20", exact = True).click()
+ # Observation window: a second menu would show up in this time.
page.wait_for_timeout(400)
checks.record(
"clicking another row's trigger does not leave two menus open",
diff --git a/tests/studio/playwright_research_freeze.py b/tests/studio/playwright_research_freeze.py
index fb2e5fccd2a..ceb74669f60 100644
--- a/tests/studio/playwright_research_freeze.py
+++ b/tests/studio/playwright_research_freeze.py
@@ -34,6 +34,7 @@ import re
import sys
from pathlib import Path
+from playwright.sync_api import TimeoutError as PlaywrightTimeout
from playwright.sync_api import sync_playwright
sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -100,6 +101,23 @@ def info(message: str) -> None:
print(f"[research-freeze] {message}", flush = True)
+def until(
+ page,
+ condition: str,
+ timeout: int = 5000,
+) -> None:
+ """Wait for the state a check reads, without failing here: a broken build spends the bound and
+ then fails on the same check. Replaces flat sleeps, which also asserted the runner's speed."""
+ try:
+ page.wait_for_function(condition, timeout = timeout)
+ except PlaywrightTimeout:
+ pass
+
+
+def frames(page) -> None:
+ page.evaluate("() => new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))")
+
+
def build_report(sections: int) -> str:
body = "\n".join(REPORT_SECTION.format(n = n) for n in range(1, sections + 1))
return f"# Deep research report\n\n{body}\n\n$$\\sum_{{i=1}}^{{n}} x_i^2$$\n"
@@ -146,6 +164,8 @@ LONGTASK_INIT = """
SCROLLER = '[aria-label="Research activity timeline"]'
+DIALOG_OPEN = "() => Boolean(document.querySelector('[role=\"dialog\"]'))"
+BODY_LOCKED = "() => document.body.style.pointerEvents === 'none'"
DISTANCE_FROM_BOTTOM = (
f"() => {{ const el = document.querySelector('{SCROLLER}');"
" return Math.round(el.scrollHeight - el.scrollTop - el.clientHeight); }"
@@ -193,7 +213,8 @@ def run() -> dict:
cdp.send("Performance.enable")
page.evaluate("window.__research.seed()")
- page.wait_for_timeout(500)
+ until(page, f"() => Boolean(document.querySelector('{SCROLLER}'))")
+ frames(page)
activities_before_stream = page.evaluate("window.__research.state().activities")
before = metrics(cdp)
@@ -213,7 +234,7 @@ def run() -> dict:
[DELTA_COUNT, DELTA_GAP_MS],
)
# The tail is part of the measurement: re-arming after the last event is the failure mode, so count those
- # frames rather than stopping the clock.
+ # frames rather than stopping the clock. A measurement window, not a readiness wait.
page.wait_for_timeout(1200)
stream_window_ms = DELTA_COUNT * DELTA_GAP_MS + 1200
stream_raf = page.evaluate("window.__rafCount")
@@ -237,7 +258,7 @@ def run() -> dict:
# Idle after the stream: the follow loop must stop when the list goes quiet.
page.evaluate("window.__rafCount = 0")
- page.wait_for_timeout(2000)
+ page.wait_for_timeout(2000) # measurement window: rAF callbacks counted over 2 s idle
results["idle_raf_callbacks_per_2s"] = page.evaluate("window.__rafCount")
report = build_report(int(os.environ.get("SMOKE_REPORT_SECTIONS", "40")))
@@ -272,7 +293,7 @@ def run() -> dict:
except Exception as exc:
report_click_landed = False
info(f"the click during the report parse never became actionable: {exc!r}")
- page.wait_for_timeout(3000)
+ page.wait_for_timeout(3000) # measurement window: the stall probe samples the whole parse
# Disarm only after the window closes, so a stall at its very end is still sampled.
page.evaluate(
"""() => {
@@ -299,22 +320,24 @@ def run() -> dict:
# under it. Either one stranding `body { pointer-events: none }` leaves the whole window unclickable.
page.evaluate("window.__research.clearReport()")
page.evaluate("window.__research.awaitApproval()")
- page.wait_for_timeout(600)
+ until(page, f"() => ({DIALOG_OPEN})() && ({BODY_LOCKED})()")
dialog_open = page.evaluate(
"() => Boolean(document.querySelector('[role=\\\"dialog\\\"]'))"
)
body_during = page.evaluate("() => document.body.style.pointerEvents")
page.evaluate("window.__research.approve()")
- page.wait_for_timeout(600)
+ until(page, f"() => !({DIALOG_OPEN})() && !({BODY_LOCKED})()")
+ frames(page)
body_after_approve = page.evaluate("() => document.body.style.pointerEvents")
clicks_before = page.evaluate("window.__research.clicks()")
page.click('[data-smoke="click-probe"]', timeout = 5000)
clicks_after_approve = page.evaluate("window.__research.clicks()")
page.evaluate("window.__research.awaitApproval()")
- page.wait_for_timeout(600)
+ until(page, f"() => ({DIALOG_OPEN})() && ({BODY_LOCKED})()")
page.evaluate("window.__research.closePanel()")
- page.wait_for_timeout(600)
+ until(page, f"() => !document.querySelector('{SCROLLER}') && !({BODY_LOCKED})()")
+ frames(page)
body_after_close = page.evaluate("() => document.body.style.pointerEvents")
page.click('[data-smoke="click-probe"]', timeout = 5000)
clicks_after_close = page.evaluate("window.__research.clicks()")
@@ -333,9 +356,17 @@ def run() -> dict:
# bottom threshold: a still-pending follow step used to run anyway and reconcile
# isAtBottom back to true, so "Latest" never appeared and nothing corrected it.
page.evaluate("window.__research.openPanel()")
- page.wait_for_timeout(300)
+ until(page, f"() => Boolean(document.querySelector('{SCROLLER}'))")
+ frames(page)
page.evaluate("() => { for (let i = 100; i < 130; i += 1) window.__research.step(i); }")
- page.wait_for_timeout(1500)
+ # The state the checks below read: the list overflows and the follow loop has pinned it.
+ until(
+ page,
+ f"() => {{ const el = document.querySelector('{SCROLLER}');"
+ " return el.scrollHeight > el.clientHeight"
+ " && el.scrollHeight - el.scrollTop - el.clientHeight <= 2; }",
+ )
+ frames(page)
overflowing = page.evaluate(
f"() => {{ const el = document.querySelector('{SCROLLER}'); return el.scrollHeight > el.clientHeight; }}"
)
@@ -353,7 +384,8 @@ def run() -> dict:
el.dispatchEvent(new Event("scroll"));
}}"""
)
- page.wait_for_timeout(600)
+ until(page, LATEST_BUTTON_VISIBLE, timeout = 2000)
+ frames(page)
results["detach"] = {
"overflowing": overflowing,
"distance_while_following": followed_distance,
diff --git a/tests/studio/playwright_settings_tabs.py b/tests/studio/playwright_settings_tabs.py
index 88b95b43618..1e4236ecca7 100644
--- a/tests/studio/playwright_settings_tabs.py
+++ b/tests/studio/playwright_settings_tabs.py
@@ -20,6 +20,7 @@ import sys
import time
from pathlib import Path
+from playwright.sync_api import TimeoutError as PlaywrightTimeout
from playwright.sync_api import sync_playwright
sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -121,6 +122,29 @@ SNAPSHOT_JS = """(sel) => {
}"""
+def until(
+ page,
+ condition: str,
+ arg = None,
+ timeout: int = 5000,
+) -> None:
+ """Wait for the state a check reads, without failing here: a broken build spends the bound and
+ then fails on the same check. Replaces flat sleeps, which also asserted the runner's speed."""
+ try:
+ page.wait_for_function(condition, arg = arg, timeout = timeout)
+ except PlaywrightTimeout:
+ pass
+
+
+DIALOG_GONE = "() => !document.querySelector('div[role=\"dialog\"]')"
+ROWS = "[...document.querySelectorAll('[data-settings-label]')].map(r => r.dataset.settingsLabel)"
+
+
+def close_dialog(page) -> None:
+ page.evaluate("() => window.__settingsSmoke.close()")
+ until(page, DIALOG_GONE)
+
+
def snapshot(page) -> dict:
return page.evaluate(SNAPSHOT_JS, PANEL)
@@ -275,8 +299,7 @@ def run_abandoned_deep_open(page) -> None:
fail("an abandoned archive deep-open reopened the archive on the next visit to Data")
else:
log("an abandoned deep-open leaves the next visit to Data on the main page")
- page.evaluate("() => window.__settingsSmoke.close()")
- page.wait_for_timeout(200)
+ close_dialog(page)
# Dropping the abandoned ones must not drop the honoured ones.
page.evaluate("() => window.__settingsSmoke.openArchived('chats')")
@@ -288,8 +311,7 @@ def run_abandoned_deep_open(page) -> None:
fail(f"a deep-open the panel did reach no longer opens the archive ({honoured})")
else:
log("a deep-open the panel reaches still opens the archive")
- page.evaluate("() => window.__settingsSmoke.close()")
- page.wait_for_timeout(200)
+ close_dialog(page)
report["steps"].append("abandoned-deep-open")
@@ -311,6 +333,8 @@ def run_chunk_fail(page) -> None:
if nav_before < 2:
fail(f"the settings nav was already empty before blocking anything ({nav_before})")
click_forced(page.locator(f'[data-testid="settings-tab-{CHUNK_FAIL}"]'), timeout = 15000)
+ # Observation window: the blocked panel's failure (and the idle prefetch's) must play out
+ # without taking the dialog or the app down, so this watches for a fixed time.
page.wait_for_timeout(3000)
state = page.evaluate(
"""() => ({
@@ -475,7 +499,7 @@ def run_keystroke_search(page) -> None:
box = page.locator(f"{PANEL} input")
toggle = page.locator(f"{PANEL} button[aria-pressed]")
click_forced(toggle, timeout = 15000)
- page.wait_for_timeout(SETTLE_MS)
+ until(page, "el => el?.getAttribute('aria-pressed') === 'true'", arg = toggle.element_handle())
# Mod is Cmd on macOS and Ctrl everywhere else, and bindingFromEvent drops a Meta
# chord off macOS outright, so a hardcoded Meta would record nothing on the Linux
# runner. Same read as isMacPlatform(), which is what the app itself goes by.
@@ -488,18 +512,24 @@ def run_keystroke_search(page) -> None:
# A chord narrows to what answers to it: ⇧⌘O is New chat's, and New standalone
# chat sits on ⌥⌘O, which does not carry the Shift.
page.keyboard.press(f"{mod}+Shift+KeyO")
- page.wait_for_timeout(SETTLE_MS)
+ until(
+ page,
+ f"() => {{ const rows = {ROWS}; return rows.includes('New chat')"
+ " && !rows.includes('New standalone chat'); }",
+ )
state["chord"] = box.input_value()
state["chord_rows"] = page.evaluate(rows_js)
# The bare key widens to every chord built on it, ⌥⌘O included.
page.keyboard.press("KeyO")
- page.wait_for_timeout(SETTLE_MS)
+ until(page, f"() => {ROWS}.includes('New standalone chat')")
state["bare_rows"] = page.evaluate(rows_js)
# Escape backs out a step at a time, and the dialog stays open for both. A dialog
# gone here is the bug this covers: the press reaching Radix before the box.
page.keyboard.press("Escape")
+ # Observation window, not a readiness guess: the bug is the dialog closing, and a closing
+ # dialog stays mounted through its exit animation, so it needs the time to finish leaving.
page.wait_for_timeout(SETTLE_MS)
open_after_first = page.locator('div[role="dialog"]').count()
state["after_first_escape"] = {
@@ -509,7 +539,9 @@ def run_keystroke_search(page) -> None:
}
if open_after_first:
page.keyboard.press("Escape")
- page.wait_for_timeout(SETTLE_MS)
+ until(
+ page, "el => el?.getAttribute('aria-pressed') === 'false'", arg = toggle.element_handle()
+ )
open_after_second = page.locator('div[role="dialog"]').count()
state["after_second_escape"] = {
"dialog": open_after_second,
@@ -580,8 +612,7 @@ def run(page) -> None:
# --- 3. every lan address has actions bound to that address ----------------------
run_lan_address_actions(page)
- page.evaluate("() => window.__settingsSmoke.close()")
- page.wait_for_timeout(300)
+ close_dialog(page)
for tab in ("voice", "api-keys", "data", "about", "connections"):
open_dialog(page, tab)
state = page.evaluate("() => window.__settingsSmoke.state()")
@@ -598,8 +629,7 @@ def run(page) -> None:
)
else:
log(f"deep-open {tab}: matches the clicked-to panel")
- page.evaluate("() => window.__settingsSmoke.close()")
- page.wait_for_timeout(200)
+ close_dialog(page)
report["steps"].append("deep-open")
# --- 5. search, then jump to a result and confirm the scroll target flashed ------
@@ -611,7 +641,12 @@ def run(page) -> None:
settle_panel(page)
search = page.locator('div[role="dialog"] aside input').first
search.fill(query)
- page.wait_for_timeout(400)
+ until(
+ page,
+ """label => [...document.querySelectorAll('div[role="dialog"] aside button:not([data-testid])')]
+ .some(b => b.innerText.trim() === label)""",
+ arg = target_label,
+ )
entries = page.locator('div[role="dialog"] aside button:not([data-testid])')
texts = [entries.nth(i).inner_text().strip() for i in range(entries.count())]
report["search"] = {"query": query, "target_label": target_label, "result_texts": texts}
diff --git a/tests/studio/probe_dismiss_guard.py b/tests/studio/probe_dismiss_guard.py
index baee079edc5..0ee274fcbd0 100644
--- a/tests/studio/probe_dismiss_guard.py
+++ b/tests/studio/probe_dismiss_guard.py
@@ -290,6 +290,29 @@ MENU_BLANK_JS = """
"""
+async def until(
+ page,
+ condition: str,
+ timeout: int,
+ arg = None,
+) -> None:
+ """Wait for `condition` up to `timeout` ms without raising: a broken build spends the bound and
+ is then judged on the same facts it always was. Replaces flat sleeps sized for a fast machine."""
+ try:
+ await page.wait_for_function(condition, arg = arg, timeout = timeout)
+ except Exception: # noqa: BLE001 (PlaywrightTimeout; the verdict below reads the state)
+ pass
+
+
+async def frames(page) -> None:
+ await page.evaluate(
+ "() => new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))"
+ )
+
+
+MENU_CLOSED = "() => !document.querySelector('.aui-action-bar-more-content')"
+
+
async def hover_last_assistant(page) -> None:
await page.evaluate(
"() => window.__heavyThread.lastAssistantMessage()?.scrollIntoView({ block: 'center' })"
@@ -301,7 +324,9 @@ async def hover_last_assistant(page) -> None:
)
if box:
await page.mouse.move(box["x"], box["y"])
- await page.wait_for_timeout(600)
+ # What the cases need from the hover: a Copy button whose centre hit-tests to itself.
+ await until(page, f"() => ({FACTS_JS})().targetUncovered", 3000)
+ await frames(page)
async def one_case(
@@ -318,7 +343,7 @@ async def one_case(
if not control["targetRect"]:
return {"case": case, "error": "no Copy button"}
await page.mouse.click(control["targetRect"]["x"], control["targetRect"]["y"])
- await page.wait_for_timeout(200)
+ await until(page, f"n => ({FACTS_JS})().targetClicks > n", 2000, arg = control["targetClicks"])
counted = (await page.evaluate(FACTS_JS))["targetClicks"] - control["targetClicks"]
if counted != 1:
return {
@@ -440,9 +465,15 @@ async def one_case(
if spot.get("error"):
return {"case": case, "error": spot["error"]}
await page.mouse.click(spot["x"], spot["y"])
- await page.wait_for_timeout(300)
+ # A focus steal would come from the close-autofocus, which runs once the menu unmounts.
+ await until(page, MENU_CLOSED, 1500)
+ await frames(page)
await page.keyboard.type("zz")
- await page.wait_for_timeout(400)
+ await until(
+ page,
+ "() => (window.__heavyThread.composer()?.value ?? '').endsWith('zz')",
+ 1000,
+ )
after = await page.evaluate(FACTS_JS)
typed = await page.evaluate(
"() => { const c = window.__heavyThread.composer(); return c ? c.value : ''; }"
@@ -520,7 +551,7 @@ async def one_case(
if not spot:
return {"case": case, "error": "no menu item to select"}
await page.mouse.click(spot["x"], spot["y"])
- await page.wait_for_timeout(800)
+ await until(page, "() => window.__dismissProbe.itemClicks > 0", 800)
return {
"case": case,
"itemClicks": await page.evaluate("() => window.__dismissProbe.itemClicks"),
@@ -583,7 +614,7 @@ async def one_case(
# for a gesture that cannot produce a click at all.
await page.wait_for_timeout(150)
await page.mouse.click(spot["x"], spot["y"])
- await page.wait_for_timeout(600)
+ await until(page, "() => window.__dismissProbe.neutralClicks > 0", 600)
clicks = await page.evaluate("() => window.__dismissProbe.neutralClicks")
return {
"case": case,
@@ -601,7 +632,7 @@ async def one_case(
await page.mouse.click(item["x"], item["y"])
await page.wait_for_timeout(int(os.environ.get("PROBE_EXIT_DELAY_MS", "40")))
await page.mouse.click(spot["x"], spot["y"])
- await page.wait_for_timeout(600)
+ await until(page, "() => window.__dismissProbe.neutralClicks > 0", 600)
clicks = await page.evaluate("() => window.__dismissProbe.neutralClicks")
return {
"case": case,
@@ -616,7 +647,7 @@ async def one_case(
await page.mouse.click(spot["x"], spot["y"])
await page.wait_for_timeout(200)
await page.mouse.click(spot["x"], spot["y"])
- await page.wait_for_timeout(600)
+ await until(page, "() => window.__dismissProbe.neutralClicks > 0", 600)
clicks = await page.evaluate("() => window.__dismissProbe.neutralClicks")
return {
"case": case,
@@ -626,7 +657,12 @@ async def one_case(
else:
return {"case": case, "error": f"unknown case {case}"}
- await page.wait_for_timeout(1500)
+ # Wait for the menu's exit rather than a flat 1500 ms: same bound, but it returns as soon as
+ # the menu is gone. A case that leaves it open on purpose (Space reopening it) still waits the
+ # whole bound. Then a short observation window, because the verdict is mostly "no click
+ # reached Copy", and a click a touch gesture synthesizes can land after the menu has closed.
+ await until(page, MENU_CLOSED, 1500)
+ await page.wait_for_timeout(300)
after = await page.evaluate(FACTS_JS)
result = {
"case": case,
diff --git a/tests/studio/reasoning-codex-checks.js b/tests/studio/reasoning-codex-checks.js
index bef981868ca..11cae04d41e 100644
--- a/tests/studio/reasoning-codex-checks.js
+++ b/tests/studio/reasoning-codex-checks.js
@@ -19,7 +19,9 @@ async (page) => {
await page.setViewportSize({ width: 1280, height: 900 });
await page.waitForFunction(() => window.__reasoning);
await page.evaluate(() => window.__reasoning.run({ size: 250000, kind: "code", chunk: 512, gap: 12 }));
- await page.waitForFunction(() => window.__reasoning.stats().done, { timeout: 60000 });
+ // The options are the THIRD argument; in second place they are the page function's arg and the
+ // 60 s timeout was silently ignored (default 30 s).
+ await page.waitForFunction(() => window.__reasoning.stats().done, null, { timeout: 60000 });
await page.waitForTimeout(400);
const streaming = await page.evaluate(() => ({ ...window.__reasoning.stats(), transfers: window.__fenceTransfers }));
check(streaming.mounted < 30000, "live code mounted unbounded content");