Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Comprehensive Unsloth chat UI test, run locally + in CI."""
import json
import os
import re
import socket
import subprocess
import sys
import time
import urllib.request
import urllib.error
from pathlib import Path
from playwright.sync_api import expect, sync_playwright
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _playwright_robust import ( # noqa: E402
chromium_launch_args,
click_and_wait_for_response,
evaluate_fetch,
install_view_transition_killer,
install_wall_clock_watchdog,
is_benign_console_error,
is_benign_page_error,
recover_or_replace_page as _robust_recover_or_replace_page,
report_failing_step,
robust_evaluate,
step_budget_s,
wait_for_first,
wait_for_health,
wait_for_settled,
wait_until,
click_forced,
)
BASE = os.environ["BASE_URL"]
OLD = os.environ["STUDIO_OLD_PW"]
NEW = os.environ["STUDIO_NEW_PW"]
NEW2 = os.environ.get("STUDIO_NEW2_PW", NEW + "X9!")
GGUF_REPO = os.environ.get("GGUF_REPO", "unsloth/gemma-3-270m-it-GGUF")
GGUF_VARIANT = os.environ.get("GGUF_VARIANT", "UD-Q4_K_XL")
ART_DIR = os.environ.get("PW_ART_DIR", "logs/playwright")
ART = Path(ART_DIR)
ART.mkdir(parents = True, exist_ok = True)
STRICT = os.environ.get("STUDIO_UI_STRICT", "0") == "1"
TURN_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_TURN_TIMEOUT_MS", "180000"))
# Only needs to outlast the 100 ms follow-up wait.
RAPID_FIRST_TURN_HOLD_S = 3.0
PERMISSION_ONLY = os.environ.get("STUDIO_UI_PERMISSION_ONLY", "0") == "1"
PLAYWRIGHT_BROWSER = os.environ.get("STUDIO_PLAYWRIGHT_BROWSER", "chromium").lower()
PLAYWRIGHT_CHANNEL = os.environ.get("STUDIO_PLAYWRIGHT_CHANNEL") or None
# Emulates 4 vCPU boxes (rapid-submit bug needs 4x/8x); off unless set, Chromium only.
CPU_THROTTLE = float(os.environ.get("STUDIO_UI_CPU_THROTTLE", "0") or 0)
FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_FETCH_TIMEOUT_MS", "30000"))
LOAD_FETCH_TIMEOUT_MS = int(os.environ.get("STUDIO_UI_LOAD_TIMEOUT_MS", "180000"))
# Must outlast the longest single wait; all candidates are env vars, so take the max of all three.
_WALL_FLOOR_S = 720.0
_LONGEST_WAIT_S = max(
(TURN_TIMEOUT_MS / 1000) * 2,
LOAD_FETCH_TIMEOUT_MS / 1000,
FETCH_TIMEOUT_MS / 1000,
)
WALL_TIMEOUT_S = float(
os.environ.get(
"STUDIO_UI_WALL_TIMEOUT_S",
max(_WALL_FLOOR_S, _LONGEST_WAIT_S + 120),
)
)
# Off by default; set by callers that size an outer bound around this process.
TOTAL_TIMEOUT_S = float(os.environ.get("STUDIO_UI_TOTAL_TIMEOUT_S", "0")) or None
_SLOW_LANE = max(1.0, TURN_TIMEOUT_MS / 180_000)
UI_STEP_BUDGET_S = step_budget_s(180 * _SLOW_LANE)
MODEL_STEP_BUDGET_S = step_budget_s(WALL_TIMEOUT_S)
NO_STEP_CEILING = 0
_n = [0]
_watchdog = None
def wall_kick():
"""Restart the budget. Call after a bounded wait, or the next inherits its leftover."""
if _watchdog is not None:
_watchdog.kick()
def step(s, budget_s = None):
"""Start step `s`; it may run `budget_s` (default UI_STEP_BUDGET_S) before the run stops."""
print(f"[ui] STEP {s}", flush = True)
if _watchdog is not None:
_watchdog.begin_step(s, UI_STEP_BUDGET_S if budget_s is None else budget_s)
def info(s):
print(f"[ui] {s}", flush = True)
wall_kick()
def fail(m):
raise AssertionError(f"[ui] FAIL: {m}")
def apply_cpu_throttle(ctx, page):
"""Throttle this page, if the option is set. No-op otherwise."""
if CPU_THROTTLE <= 1:
return page
ctx.new_cdp_session(page).send("Emulation.setCPUThrottlingRate", {"rate": CPU_THROTTLE})
info(f"CPU throttled {CPU_THROTTLE}x")
return page
def new_throttled_page(ctx):
"""CPU throttle is scoped to the page target, so every page must come from here to stay throttled."""
page = ctx.new_page()
page.set_default_timeout(60_000)
apply_cpu_throttle(ctx, page)
return page
def close_page_without_waiting(ctx, page) -> None:
"""Close a page without waiting on its renderer, on Chromium; page.close() elsewhere.
page.close() returns only once the page reports itself closed, and has no timeout. On the
Windows lane it has sat out a whole 540 s step budget on the persisted-monitor reset, the
last line printed being "closing the stale page". Attaching a CDP session, reading the
target id and Target.closeTarget are all answered by the browser process, so none of them
waits on a busy or throttled renderer, and closeTarget returns as soon as the close is
queued. Requests the page sends before it is gone are the caller's to discount.
"""
if PLAYWRIGHT_BROWSER == "chromium":
try:
target = ctx.new_cdp_session(page).send("Target.getTargetInfo")["targetInfo"][
"targetId"
]
ctx.browser.new_browser_cdp_session().send("Target.closeTarget", {"targetId": target})
return
except Exception as exc:
info(
f"WARN closing the page browser-side failed ({exc!r}); falling back to page.close()"
)
page.close()
def recover_or_replace_page(page, ctx, **kwargs):
"""Emulation.setCPUThrottlingRate is per page target, so a replacement page must be throttled again."""
replacement = _robust_recover_or_replace_page(page, ctx, **kwargs)
if replacement is not page:
apply_cpu_throttle(ctx, replacement)
return replacement
def expected_default_model():
override = os.environ.get("EXPECTED_DEFAULT_MODEL")
if override:
return override
# Parse as a literal: the --no-torch Playwright install cannot import defaults.py's deps.
import ast
defaults_path = (
Path(__file__).resolve().parents[2]
/ "studio"
/ "backend"
/ "core"
/ "inference"
/ "defaults.py"
)
try:
tree = ast.parse(defaults_path.read_text(encoding = "utf-8"))
except Exception as exc:
fail(f"could not read {defaults_path}: {exc}")
models = None
for node in tree.body:
if not isinstance(node, ast.Assign):
continue
if not any(isinstance(t, ast.Name) and t.id == "DEFAULT_MODELS_GGUF" for t in node.targets):
continue
try:
models = ast.literal_eval(node.value)
except Exception as exc:
fail(f"could not eval DEFAULT_MODELS_GGUF literal: {exc}")
break
if not models:
fail("DEFAULT_MODELS_GGUF not found or empty in defaults.py")
return models[0]
def soft_fail(m):
"""Hard fail in STRICT mode, info-warn otherwise."""
if STRICT:
fail(m)
info(f"WARN (strict-off): {m}")
FULL_ACCESS_TITLE = '[data-slot="alert-dialog-title"]'
FULL_ACCESS_CANCEL = '[data-slot="alert-dialog-cancel"]'
FULL_ACCESS_CONFIRM = '[data-slot="alert-dialog-action"]'
PERMISSION_PILL = 'button[aria-label="Permission level for tool calls"]:visible'
ROUTE_FALLBACK_TEXT = (
"Loading..." # RouteFallback in studio/frontend/src/app/routes/__root.tsx (common.loading)
)
class ShellMissingControl(AssertionError):
"""The chat shell booted (composer up, no route fallback) without the permission pill."""
def wait_for_settled_shell(
page,
composer,
settle_s = 2.0,
fallback_budget_s = 30.0,
missing_pill_s = 5.0,
):
"""Waits for the composer and pill to stay up together for settle_s; a route fallback is waited
out up to fallback_budget_s, then raises a boot failure; a booted shell missing the pill for
missing_pill_s raises ShellMissingControl, which callers must not retry."""
pill = page.locator(PERMISSION_PILL).first
fallback = page.get_by_text(ROUTE_FALLBACK_TEXT, exact = True)
deadline = time.monotonic() + fallback_budget_s
stable_since = None
pill_missing_since = None
while True:
now = time.monotonic()
has_composer = composer.is_visible()
has_pill = pill.is_visible()
in_fallback = fallback.count() > 0 and fallback.first.is_visible()
if has_composer and has_pill:
pill_missing_since = None
stable_since = stable_since or now
if now - stable_since >= settle_s:
return
else:
stable_since = None
if has_composer and not in_fallback:
pill_missing_since = pill_missing_since or now
if now - pill_missing_since >= missing_pill_s:
raise ShellMissingControl(
f"the composer is up with no route fallback, but the permission pill has been missing for "
f"{missing_pill_s:.0f}s; page.url={page.url}"
)
else:
pill_missing_since = None
if now >= deadline:
raise TimeoutError(
f"chat shell did not settle in {fallback_budget_s:.0f}s: composer={has_composer} pill={has_pill} "
f"route_fallback={in_fallback}; page.url={page.url}"
)
page.wait_for_timeout(100)
def exercise_permission_mode_controls(page, shoot):
"""Exercise labels, migration, persistence, confirmation, and focus."""
step("permission levels: labels, persistence, confirmation, and focus")
pill = page.locator(PERMISSION_PILL).first
expect(pill).to_be_visible()
sandbox_answer = {"ready": True}
def answer_sandbox_capability(route):
ready = sandbox_answer["ready"]
route.fulfill(
status = 200,
content_type = "application/json",
body = json.dumps(
{
"python_os_isolated": ready,
"terminal_os_isolated": ready,
"backend": "bubblewrap",
"platform": "linux",
"reason": "" if ready else "bwrap: setting up uid map: Permission denied",
"setup_action": None,
"manual_command": "" if ready else "apt-get install -y bubblewrap",
"can_run_setup": False,
}
),
)
page.route("**/api/sandbox/capability*", answer_sandbox_capability)
def expect_mode(label):
expect(pill).to_have_attribute("data-pill-label", label)
expect(pill).to_contain_text(label)
def open_menu():
pill.click()
menu = page.get_by_role("menu").last
expect(menu).to_be_visible()
return menu
def choose(label):
menu = open_menu()
item = menu.get_by_role("menuitem").filter(has_text = label).first
expect(item).to_be_visible()
item.click()
def pick_sandbox_level(menu, current, target):
# The "Sandbox <level>" chip in the permission menu opens a level picker beside the menu.
chip = menu.get_by_role("menuitem", name = re.compile(r"^Sandbox"))
expect(chip).to_contain_text(current)
chip.click()
picker = page.get_by_role("menu").filter(has_text = "How should code be sandboxed?")
expect(picker).to_be_visible()
picker.get_by_role("menuitem", name = re.compile(rf"^{target}")).click()
# The old page can write the level back after the clear, so an init script re-sets it.
page.add_init_script(
"""(() => {
const pending = sessionStorage.getItem("__pw_permission_storage");
if (pending === null) return;
sessionStorage.removeItem("__pw_permission_storage");
const legacyValue = JSON.parse(pending);
localStorage.removeItem("unsloth_chat_permission_mode");
if (legacyValue === null) {
localStorage.removeItem("unsloth_chat_confirm_tool_calls");
} else {
localStorage.setItem("unsloth_chat_confirm_tool_calls", legacyValue);
}
})();"""
)
def set_legacy_confirm(legacy_value):
page.evaluate(
"""(legacyValue) => {
localStorage.removeItem("unsloth_chat_permission_mode");
if (legacyValue === null) {
localStorage.removeItem("unsloth_chat_confirm_tool_calls");
} else {
localStorage.setItem(
"unsloth_chat_confirm_tool_calls",
legacyValue,
);
}
sessionStorage.setItem(
"__pw_permission_storage",
JSON.stringify(legacyValue),
);
}""",
legacy_value,
)
# The level is an installation setting; refuse the hydrating GET so each run opens like a first browser.
def refuse_settings_hydration(route):
if route.request.method == "GET":
route.fulfill(
status = 503,
content_type = "application/json",
body = json.dumps({"detail": "hydration disabled for this step"}),
)
else:
route.continue_()
# Wait for the pill, not networkidle (it hangs); reload once only if the shell never boots.
def _boot_state():
try:
return page.evaluate(
"""() => ({
url: location.href,
readyState: document.readyState,
composer: !!document.querySelector('textarea[aria-label="Message input"]'),
root: (document.getElementById("root")?.innerText || "").trim().slice(0, 80),
})"""
)
except Exception as exc:
return {"evaluate_failed": repr(exc)}
def reload_and_wait_for_pill():
page.reload(wait_until = "load")
try:
expect(pill).to_be_visible(timeout = 30_000)
return
except AssertionError:
state = _boot_state()
shoot("04-permission-pill-missing")
info(f"WARN permission pill missing 30s after reload; page state {state}")
if state.get("composer") or state.get("evaluate_failed"):
raise
page.reload(wait_until = "load")
expect(pill).to_be_visible(timeout = 30_000)
info(
"WARN the app did not boot on the first reload and did on the second; see the state above"
)
# choose() only drives THIS tab.
# The mirror to /api/chat/settings is a 400ms trailing-edge debounce (SETTINGS_DEBOUNCE_MS, chat-runtime-store.ts)
# whose only early flush is the beforeunload keepalive, so the pill turning over proves the click landed locally,
# not that the installation stored it.
# Measured on webkit, timed from the choose() below: choose returns at t+238ms, set_legacy_confirm at t+248ms, and
# the reload starts there.
# The debounce would not have fired until ~t+630ms, so the only PUT that goes out at all is the beforeunload
# keepalive at t+252ms, and the reloaded page's hydrating GET arrives at t+697ms.
# The assertion after the reload was therefore never testing the level this step chose. It was betting that an
# unload-time keepalive beats a hydrating GET by 445ms of loopback, on every engine, every run.
#
# So: wait for the level to actually be ON the installation before reloading and asserting on it. Assert what was
# achieved, not what was commanded.
def expect_server_mode(
expected,
timeout_ms = 15_000,
key = "permissionMode",
):
deadline = time.monotonic() + timeout_ms / 1000.0
seen = "<never read>"
while True:
seen = page.evaluate(
"""async (key) => {
const token = localStorage.getItem("unsloth_auth_token");
const res = await fetch("/api/chat/settings", {
headers: token ? { Authorization: "Bearer " + token } : {},
cache: "no-store",
});
if (!res.ok) return "<http " + res.status + ">";
const body = await res.json();
return (body && body.settings && body.settings[key]) ?? null;
}""",
key,
)
if seen == expected:
return
if time.monotonic() >= deadline:
break
page.wait_for_timeout(100)
fail(
f"permission level never reached the installation: /api/chat/settings "
f"reports {key}={seen!r} after {timeout_ms}ms, expected "
f"{expected!r} -- the debounced mirror never landed"
)
page.route("**/api/chat/settings", refuse_settings_hydration)
set_legacy_confirm(None)
reload_and_wait_for_pill()
expect_mode("Approve for me")
menu = open_menu()
for label in (
"Ask for approval",
"Approve for me",
"Run automatically",
"Full access",
):
expect(menu.get_by_role("menuitem").filter(has_text = label).first).to_be_visible()
if menu.get_by_text("Off", exact = True).count() != 0:
fail("legacy Off label is still visible")
if menu.locator('[role="menuitem"] button, [role="menuitem"] [role="button"]').count():
fail("permission menu contains nested interactive controls")
page.keyboard.press("Escape")
expect(pill).to_be_focused()
choose("Approve for me")
expect_mode("Approve for me")
expect(page.get_by_role("alertdialog")).to_have_count(0)
compact_width = 390
page.set_viewport_size({"width": compact_width, "height": 844})
expect(pill).to_be_visible()
# set_viewport_size returns before layout responds, and the pill is already visible at its old width.
def fits_compact(box) -> bool:
return box is not None and box["x"] >= 0 and box["x"] + box["width"] <= compact_width
deadline = time.time() + 5
box = pill.bounding_box()
while not fits_compact(box) and time.time() < deadline:
page.wait_for_timeout(50)
box = pill.bounding_box()
if not fits_compact(box):
fail(f"permission pill is clipped in compact layout: {box!r}")
page.set_viewport_size({"width": 1280, "height": 900})
# Legacy migration: true -> ask, false -> off, absent -> auto; a stored level would win, hence the refused GET.
migration_cases = (
("true", "Ask for approval"),
("false", "Run automatically"),
(None, "Approve for me"),
)
try:
for legacy_value, expected_label in migration_cases:
set_legacy_confirm(legacy_value)
reload_and_wait_for_pill()
expect_mode(expected_label)
finally:
page.unroute("**/api/chat/settings", refuse_settings_hydration)
choose("Ask for approval")
expect_mode("Ask for approval")
expect_server_mode("ask")
set_legacy_confirm("false")
reload_and_wait_for_pill()
expect_mode("Ask for approval")
cached = page.evaluate("() => localStorage.getItem('unsloth_chat_permission_mode')")
if cached != "ask":
fail(f"hydration left the local cache at {cached!r}, expected 'ask'")
choose("Run automatically")
expect_mode("Run automatically")
expect(page.locator('button[data-pill-label="Search"]:visible').first).to_be_visible()
expect(page.locator('button[data-pill-label="Tools"]:visible').first).to_be_visible()
stored = page.evaluate("() => localStorage.getItem('unsloth_chat_permission_mode')")
if stored != "off":
fail(f"Run automatically persisted {stored!r}, expected 'off'")
# Find the dialog by its alert-dialog slots, not its wording, which has been rewritten before.
choose("Full access")
dialog = page.get_by_role("alertdialog")
expect(dialog).to_be_visible()
expect(dialog.locator(FULL_ACCESS_TITLE)).to_contain_text("Full access")
expect(dialog).to_contain_text("sandbox")
dialog.locator(FULL_ACCESS_CANCEL).click()
expect(dialog).to_be_hidden()
expect_mode("Run automatically")
choose("Full access")
expect(dialog).to_be_visible()
dialog.locator(FULL_ACCESS_CONFIRM).click()
expect_mode("Full access")
active_icon = pill.locator(".composer-pill-glyph > :first-child")
pill.hover()
wait_for_settled(active_icon)
icon_opacity = float(active_icon.evaluate("el => getComputedStyle(el).opacity"))
if icon_opacity < 0.5:
fail(f"Full access icon disappeared on hover (opacity={icon_opacity})")
stored = page.evaluate("() => localStorage.getItem('unsloth_chat_permission_mode')")
if stored != "off":
fail(f"Full access overwrote persisted mode with {stored!r}")
reload_and_wait_for_pill()
expect_mode("Run automatically")
# Without a sandbox: Run automatically applies at once, the Sandbox chip reads Low, and picking
# High in its level picker opens the install popup, whose "Use Low sandbox" keeps Low.
choose("Approve for me")
expect_mode("Approve for me")
# Must land on the install first, or the reload hydrates the previous "off" back.
expect_server_mode("auto")
sandbox_answer["ready"] = False
reload_and_wait_for_pill()
expect_mode("Approve for me")
choose("Run automatically")
expect_mode("Run automatically")
if page.get_by_role("alertdialog").count() != 0:
fail("picking Run automatically without a sandbox opened a dialog")
# Reopening the menu straight after a pick races its close; a reload also proves the level persisted.
# Landed on the install first, or the reload hydrates the previous "auto" back.
expect_server_mode("off")
reload_and_wait_for_pill()
expect_mode("Run automatically")
menu = open_menu()
expect(menu.get_by_text("Permissions", exact = True)).to_be_visible()
if menu.get_by_text("OS sandbox not available").count() != 0:
fail("the Run automatically row still carries the 'OS sandbox not available' hint")
pick_sandbox_level(menu, current = "Low", target = "High")
setup = page.get_by_role("alertdialog")
expect(setup.get_by_role("heading", name = "OS sandbox is not available")).to_be_visible()
# The command rides on Copy command (and Settings > Sandbox), not as a block of text in the popup.
expect(setup).not_to_contain_text("apt-get install -y bubblewrap")
expect(setup.get_by_role("button", name = "Copy command")).to_be_visible()
expect(setup.get_by_role("button", name = "Copy command")).to_have_attribute(
"title", re.compile(r"apt-get install -y bubblewrap")
)
expect(setup.get_by_role("button", name = "Learn more")).to_be_visible()
if setup.get_by_role("button", name = "Install sandbox").count() != 0:
fail("setup popup offered Install sandbox to a request the server did not allow")
setup.get_by_role("button", name = "Use Low sandbox").click()
expect(setup).to_be_hidden()
level = page.evaluate("() => localStorage.getItem('unsloth_chat_sandbox_level')")
if level != "low":
fail(f"Use Low sandbox stored {level!r}, expected 'low'")
expect_mode("Run automatically")
expect_server_mode("low", key = "sandboxLevel")
sandbox_answer["ready"] = True
reload_and_wait_for_pill()
# With a working sandbox picking High applies it at once, and it stays High for the next browser's run.
pick_sandbox_level(open_menu(), current = "Low", target = "High")
if page.get_by_role("alertdialog").count() != 0:
fail("picking High with a working sandbox opened the setup popup")
page.wait_for_function("() => localStorage.getItem('unsloth_chat_sandbox_level') === 'high'")
page.keyboard.press("Escape")
expect_server_mode("high", key = "sandboxLevel")
choose("Approve for me")
expect_mode("Approve for me")
expect_server_mode("auto")
shoot("04-permission-levels")
# Left in place the route disables this page's HTTP cache and wedged sign-out on Windows.
page.unroute("**/api/sandbox/capability*", answer_sandbox_capability)
def login_via_api(pw):
req = urllib.request.Request(
f"{BASE}/api/auth/login",
data = json.dumps({"username": "unsloth", "password": pw}).encode(),
method = "POST",
headers = {"Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(req, timeout = 10) as r:
return r.status
except urllib.error.HTTPError as exc:
return exc.code
# Opening a chat awaits four requests in a row; 300-700 ms on a shared Windows runner.
RECENTS_LOAD_TIMEOUT_MS = 20_000
def open_recent_thread_with_our_prompts(
page,
sent_prompts,
shoot,
our_thread_id = "",
):
"""Opens this run's Recents chat by id or title, and requires it to show our turns, not another's."""
wanted = [" ".join(p.lower().split()) for p in sent_prompts]
threads = page.locator('[data-testid="recent-thread"]')
try:
threads.first.wait_for(state = "visible", timeout = 5_000)
except Exception as _wait_err:
info(f"WARN no recent-thread testid surfaced within 5s: {_wait_err!s}")
n_threads = threads.count()
titles = [(threads.nth(i).text_content() or "").strip() for i in range(min(n_threads, 20))]
ours = [i for i, t in enumerate(titles) if " ".join(t.lower().split()) in wanted]
if not ours:
info(f"WARN no Recents title matches a prompt we sent; titles={titles[:5]!r}")
candidates = [(threads.nth(i), titles[i], i in ours) for i in ours] + [
(threads.nth(i), titles[i], False) for i in range(len(titles)) if i not in ours
]
if our_thread_id:
# By id: pinned and project chats share the testid and can sit ahead.
row = page.locator(f'[data-testid="recent-thread"][data-thread-id="{our_thread_id}"]').first
if row.count() == 0:
soft_fail(f"this run's chat {our_thread_id!r} is not in the sidebar")
return
candidates = [(row, (row.text_content() or "").strip(), True)]
def shown_user_turns():
return (
robust_evaluate(
page,
"""() => Array.from(document.querySelectorAll('[data-role="user"]'))
.map((el) => (el.innerText || "").toLowerCase().split(/\\s+/).join(" "))""",
)
or []
)
def landed(thread_id, before, before_thread):
"""Ours, other or None; a turn identical to before is stale unless its row was open."""
try:
handle = page.wait_for_function(
"""([threadId, wanted, before, beforeThread]) => {
const params = new URLSearchParams(location.search);
if (threadId && params.get("thread") !== threadId
&& params.get("compare") !== threadId) {
return false;
}
const users = Array.from(document.querySelectorAll('[data-role="user"]'))
.map((el) => (el.innerText || "").toLowerCase().split(/\\s+/).join(" "));
const stale = users.length > 0 && threadId !== beforeThread
&& JSON.stringify(users) === JSON.stringify(before);
if (stale) return false;
if (users.some((text) => wanted.some((prompt) => text.includes(prompt)))) {
return "ours";
}
return users.length ? "other" : false;
}""",
arg = [thread_id, wanted, before, before_thread],
timeout = RECENTS_LOAD_TIMEOUT_MS,
)
return handle.json_value()
except Exception:
return None
deadline = time.monotonic() + 60
clicked = 0
thread_id = ""
for i, (entry, title, strict) in enumerate(candidates[:5]):
if time.monotonic() > deadline:
break
before = shown_user_turns()
before_thread = robust_evaluate(
page, "() => new URLSearchParams(location.search).get('thread') || ''"
)
try:
thread_id = entry.get_attribute("data-thread-id") or ""
entry.scroll_into_view_if_needed()
entry.click(timeout = 5_000)
except Exception as _click_err:
info(f"recent-thread click {i} failed: {_click_err!s}")
continue
clicked += 1
info(f"OK clicked recent entry: {title[:60]!r}")
started = time.monotonic()
verdict = landed(thread_id, before, before_thread)
if verdict == "ours":
shoot("15d-recent-clicked")
info(
"OK landed on a thread that includes our prompts "
f"({(time.monotonic() - started) * 1000:.0f} ms after the click)"
)
return
if strict:
break
info(f"recent entry {i} is not this run's chat ({verdict or 'no turns'}); trying the next")
if not clicked:
soft_fail(f"no Recents entry was clickable within 60s deadline (n_threads={n_threads})")
return
turns_text = robust_evaluate(
page,
"""() => Array.from(
document.querySelectorAll('[data-role="user"], [data-role="assistant"]')
).map((e) => (e.innerText || "").toLowerCase()).join(" ")""",
)
shoot("15d-recent-clicked")
soft_fail(
"Recents-clicked thread doesn't contain any of our sent prompts after "
f"{RECENTS_LOAD_TIMEOUT_MS} ms; url={page.url!r} thread={thread_id!r} "
f"turns_text={turns_text[:120]!r}"
)
def parse_rgb(s):
m = re.search(r"rgba?\((\d+),\s*(\d+),\s*(\d+)", s or "")
return tuple(int(x) for x in m.groups()) if m else None
def exercise_floating_monitor_geometry(page):
"""Exercise content, drag, edge resize, and viewport geometry."""
monitor = page.get_by_test_id("floating-monitor")
monitor.wait_for(state = "visible", timeout = 10_000)
monitor_handle = page.get_by_test_id("floating-monitor-drag-handle")
viewport = page.viewport_size
if viewport is None:
fail("Playwright viewport unavailable for floating monitor check")
inset = 16
tolerance = 1
def monitor_box(label):
box = monitor.bounding_box()
if box is None:
fail(f"floating monitor has no bounding box during {label}")
return box
def wait_for_box(label, predicate):
deadline = time.time() + 5
box = monitor_box(label)
while not predicate(box) and time.time() < deadline:
page.wait_for_timeout(50)
box = monitor_box(label)
if not predicate(box):
fail(f"floating monitor did not settle during {label}: {box!r}")
return box
def pointer_drag(start_x, start_y, end_x, end_y):
page.mouse.move(start_x, start_y)
page.mouse.down()
page.mouse.move(end_x, end_y, steps = 10)
page.mouse.up()
wait_for_settled(monitor)
def drag_monitor_to(x, y):
box = monitor_handle.bounding_box()
if box is None:
fail("floating monitor handle has no bounding box")
pointer_drag(
box["x"] + box["width"] / 2,
box["y"] + box["height"] / 2,
x,
y,
)
return monitor_box("drag")
def resize_handle_center(edge):
handle = monitor.locator(f'[data-testid="floating-monitor-resize"][data-edge="{edge}"]')
box = handle.bounding_box()
if box is None:
fail(f"floating monitor {edge} resize handle has no bounding box")
return box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
def resize_monitor_by(
dx,
dy,
edge = "se",
):
before = monitor_box("resize")
start_x, start_y = resize_handle_center(edge)
pointer_drag(start_x, start_y, start_x + dx, start_y + dy)
return before, monitor_box("resize")
def content_fits(label):
clipped = monitor.get_by_test_id("floating-monitor-content").evaluate(
"node => node.getBoundingClientRect().height"
" - node.parentElement.getBoundingClientRect().height"
)
if abs(clipped) > tolerance:
fail(f"{label}: content and its scroll region differ by {clipped!r}px")
def expect_close(actual, expected, label):
if abs(actual - expected) > tolerance:
fail(f"{label}: expected {expected!r}, got {actual!r}")
def is_inside(box, surface):
return (
box["x"] >= inset - tolerance
and box["y"] >= inset - tolerance
and box["x"] + box["width"] <= surface["width"] - inset + tolerance
and box["y"] + box["height"] <= surface["height"] - inset + tolerance
)
# Wait for /api/system; before it, zero-filled defaults, and VRAM later adds ~59px.
try:
page.wait_for_function(
r"""() => {
const monitor = document.querySelector(
'[data-testid="floating-monitor"]'
);
const content = document.querySelector(
'[data-testid="floating-monitor-content"]'
);
if (!(monitor && content)) return false;
// DEFAULT_SYSTEM reports a 0 GiB RAM total; a real payload never does.
const readout = content.innerText.match(
/([\d.]+)\s*GiB\s*\/\s*([\d.]+)\s*GiB/
);
if (!readout || !(Number.parseFloat(readout[2]) > 0)) return false;
// The rows commit a pass before the panel resizes to them, so the
// panel is only done reacting once its scroll region exactly fits
// the content it was reconciled against.
const scroll = content.parentElement;
const monitorHeight = monitor.getBoundingClientRect().height;
const contentHeight = content.getBoundingClientRect().height;
const scrollHeight = scroll.getBoundingClientRect().height;
if (Math.abs(scrollHeight - contentHeight) > 1) return false;
// Row insertion also lands a frame before the gap between rows
// does, and that intermediate state is self-consistent. Require
// the geometry to hold for two consecutive animation frames --
// this runs under the default polling="raf", and it is how
// Playwright itself defines a stable element.
//
// Position belongs in the signature as well as size. An undragged
// panel is bottom-anchored by re-clamping its top against the new
// height, and that lands a frame AFTER the height it reacts to, so
// a size-only signature reports settled while the panel is still
// where the shorter version put it. Sampling there reads a bottom
// inset that is exactly the growth too low. Deliberately not
// waiting on the expected inset itself: that would gate on the
// very thing the assertions below check and turn a real
// misplacement into a timeout instead of a failure.
const rect = monitor.getBoundingClientRect();
const signature = [
monitorHeight, contentHeight,
Math.round(rect.top), Math.round(rect.left),
].join("x");
const settled = window.__unslothMonitorGeometry === signature;
window.__unslothMonitorGeometry = signature;
return settled;
}""",
timeout = 30_000,
)
except Exception as exc:
fail(f"floating monitor never settled on an /api/system payload: {exc!r}")
initial_box = monitor_box("initial placement")
expect_close(
initial_box["x"] + initial_box["width"],
viewport["width"] - inset,
"initial right inset",
)
expect_close(
initial_box["y"] + initial_box["height"],
viewport["height"] - inset,
"initial bottom inset",
)
monitor.get_by_test_id("floating-monitor-content").evaluate(
"""node => {
const probe = document.createElement("div");
probe.dataset.testid = "floating-monitor-growth-probe";
probe.style.height = "48px";
node.appendChild(probe);
}"""
)
grown_box = wait_for_box(
"content growth",
lambda box: box["height"] >= initial_box["height"] + 47,
)
expect_close(
grown_box["y"] + grown_box["height"],
viewport["height"] - inset,
"content growth bottom inset",
)
monitor.get_by_test_id("floating-monitor-growth-probe").evaluate("node => node.remove()")
initial_box = wait_for_box(
"content shrink",
lambda box: (
abs(box["height"] - initial_box["height"]) <= tolerance
and abs(box["y"] + box["height"] - viewport["height"] + inset) <= tolerance
),
)
# At the resting corner the right and bottom edges have no room, so an outward drag changes nothing.
_, blocked_box = resize_monitor_by(40, 40)
expect_close(blocked_box["width"], initial_box["width"], "blocked width")
expect_close(blocked_box["height"], initial_box["height"], "blocked height")
# The left edge grows the card leftward while it stays pinned to the corner.
_, left_grown = resize_monitor_by(-60, 0, edge = "w")
expect_close(left_grown["width"], initial_box["width"] + 60, "left edge width")
expect_close(
left_grown["x"] + left_grown["width"],
viewport["width"] - inset,
"left edge keeps right inset",
)
expect_close(
left_grown["y"] + left_grown["height"],
viewport["height"] - inset,
"left edge keeps bottom inset",
)
content_fits("left edge resize")
# Double-click on a handle restores the default width at the same corner.
start_x, start_y = resize_handle_center("w")
page.mouse.dblclick(start_x, start_y)
wait_for_settled(monitor)
initial_box = wait_for_box(
"width reset",
lambda box: (
abs(box["width"] - initial_box["width"]) <= tolerance
and abs(box["height"] - initial_box["height"]) <= tolerance
and abs(box["y"] + box["height"] - viewport["height"] + inset) <= tolerance
),
)
left_box = drag_monitor_to(0, viewport["height"] / 2)
expect_close(left_box["x"], inset, "left inset")
expect_close(left_box["width"], initial_box["width"], "post-drag width")
expect_close(left_box["height"], initial_box["height"], "post-drag height")
right_box = drag_monitor_to(viewport["width"], viewport["height"] / 2)
expect_close(
right_box["x"] + right_box["width"],
viewport["width"] - inset,
"right inset",
)
handle_box = monitor_handle.bounding_box()
if handle_box is None:
fail("floating monitor handle has no active-drag bounding box")
page.mouse.move(
handle_box["x"] + handle_box["width"] / 2,
handle_box["y"] + handle_box["height"] / 2,
)
page.mouse.down()
reduced_viewport = {"width": 500, "height": 400}
page.set_viewport_size(reduced_viewport)
page.mouse.move(498, 398, steps = 10)
page.mouse.up()
wait_for_box(
"active viewport shrink",
lambda box: is_inside(box, reduced_viewport),
)
narrow_viewport = {"width": 260, "height": 400}
page.set_viewport_size(narrow_viewport)
wait_for_box("narrow viewport", lambda box: is_inside(box, narrow_viewport))
page.set_viewport_size(viewport)
# The panel is anchored to the right edge here, so the restore moves it from the narrow viewport's spot back to
# the right. That spot is inside the restored viewport too, so is_inside alone can pass a frame before the
# layout catches up. A drag grabbed then starts at left 0 and ends at left 0, so the horizontal axis keeps its
# right-edge anchor, and the resize below snaps the panel back right with only 260 px of room to grow.
wait_for_settled(monitor)
wait_for_box("viewport restore", lambda box: is_inside(box, viewport))
# Width is the one user-set size: the text stays its size and the height follows the content,
# so a corner drag never adds empty space below it.
resize_start = drag_monitor_to(0, 0)
_, resized_box = resize_monitor_by(40, 40)
expect_close(resized_box["width"], resize_start["width"] + 40, "resize width")
if resized_box["height"] > resize_start["height"] + tolerance:
fail(f"a wider card grew taller: {resize_start!r} -> {resized_box!r}")
expect_close(resized_box["x"], resize_start["x"], "resize left edge")
expect_close(resized_box["y"], resize_start["y"], "resize top edge")
content_fits("corner resize")
_, minimum_box = resize_monitor_by(-200, 0)
expect_close(minimum_box["width"], 232, "minimum width")
content_fits("minimum resize")
drag_monitor_to(0, 0)
_, maximum_box = resize_monitor_by(viewport["width"], 0)
expect_close(maximum_box["width"], 520, "maximum width")
if not is_inside(maximum_box, viewport):
fail(f"maximum resize left the viewport: {maximum_box!r}")
content_fits("maximum resize")
monitor.get_by_role("button", name = "Close").click()
monitor.wait_for(state = "hidden")
info(
"OK floating monitor resizes from its edges and stays stable across "
"content, drag, and viewport changes"
)
with sync_playwright() as p:
_watchdog = install_wall_clock_watchdog(
WALL_TIMEOUT_S,
label = "ui",
info = info,
total_deadline_s = TOTAL_TIMEOUT_S,
)
report_failing_step(_watchdog, label = "ui")
# macOS can return 200 /api/health while the auth DB is still migrating.
wait_for_health(BASE, timeout = 30.0, info = info)
if PLAYWRIGHT_BROWSER not in ("chromium", "firefox", "webkit"):
fail(f"unsupported STUDIO_PLAYWRIGHT_BROWSER={PLAYWRIGHT_BROWSER!r}")
browser_type = getattr(p, PLAYWRIGHT_BROWSER)
launch_kwargs = {"headless": True}
if PLAYWRIGHT_BROWSER == "chromium":
launch_kwargs["args"] = chromium_launch_args()
if PLAYWRIGHT_CHANNEL:
launch_kwargs["channel"] = PLAYWRIGHT_CHANNEL
elif PLAYWRIGHT_CHANNEL:
fail("STUDIO_PLAYWRIGHT_CHANNEL requires chromium")
if CPU_THROTTLE > 1 and PLAYWRIGHT_BROWSER != "chromium":
# new_cdp_session is Chromium only; refuse here instead of an opaque mid-run CDP error.
fail(f"STUDIO_UI_CPU_THROTTLE requires chromium, not {PLAYWRIGHT_BROWSER}")
browser = browser_type.launch(**launch_kwargs)
ctx = browser.new_context(
viewport = {"width": 1280, "height": 900},
# View-transition animations would otherwise intercept pointer events.
reduced_motion = "reduce",
)
install_view_transition_killer(ctx)
system_requests: list[str] = []
# Pages closed without waiting (close_page_without_waiting) can still send a request or two while the
# browser tears them down; those are not the page the monitor steps observe.
stale_pages: set = set()
def _count_system_request(request) -> None:
if not request.url.split("?", 1)[0].endswith("/api/system"):
return
try:
if request.frame.page in stale_pages:
return
except Exception:
pass # no frame (a worker): counted, as before
system_requests.append(request.url)
ctx.on("request", _count_system_request)
page = new_throttled_page(ctx)
page_errors = []
page.on("pageerror", lambda e: page_errors.append(str(e)))
console_errors: list[str] = []
def _on_console(m):
if m.type != "error":
return
try:
text = m.text
except Exception:
return
console_errors.append(text)
page.on("console", _on_console)
chat_completions_responses: list[tuple[int, str]] = []
page.on(
"response",
lambda r: (
chat_completions_responses.append((r.status, r.url))
if "/v1/chat/completions" in r.url
else None
),
)
def shoot(name):
# Screenshots are diagnostic only; their webfont wait can be slow on macOS.
_n[0] += 1
try:
page.screenshot(
path = str(ART / f"{_n[0]:02d}-{name}.png"),
full_page = True,
timeout = 90_000,
animations = "disabled",
)
except Exception as _shoot_err:
info(f"WARN: screenshot {name} failed: {_shoot_err}")
# 1. Change-password through the UI ("Setup your account").
step("change-password through UI (Setup your account)", NO_STEP_CEILING)
# A late bootstrap poll can rerender the page and drop #new-password, so retry with a fresh page.
form_err: Exception | None = None
for _form_attempt in range(3):
# Bare print() progress does not reset the wall budget.
wall_kick()
try:
page.goto(f"{BASE}/change-password", wait_until = "domcontentloaded", timeout = 60_000)
try:
page.wait_for_load_state("networkidle", timeout = 30_000)
except Exception:
pass
pw_field = page.locator("#new-password")
pw_field.wait_for(state = "visible", timeout = 60_000)
# Do NOT shoot() here: the screenshot's font wait can let a background poll detach the form.
pw_field.fill(NEW, timeout = 60_000)
page.fill("#confirm-password", NEW, timeout = 60_000)
shoot("01-change-password-filled")
# Wait for the POST together with the click so ERR_NO_BUFFER_SPACE surfaces now.
status, _ = click_and_wait_for_response(
page,
url_substr = "/api/auth/change-password",
method = "POST",
do_click = lambda: page.locator('button[type="submit"]').click(),
timeout_ms = 30_000,
info = lambda m: print(f"[ui] {m}", flush = True),
)
if status is not None and status >= 400:
raise AssertionError(
f"change-password POST returned {status}; "
f"see console_errors={console_errors[:1]!r}"
)
form_err = None
break
except Exception as e:
form_err = e
try:
cur_url = page.url
except Exception:
cur_url = "<page closed>"
print(
f"[ui] change-password form attempt {_form_attempt + 1} failed: "
f"{type(e).__name__}: {str(e)[:200]}; page.url={cur_url}; "
f"page_errors={len(page_errors)} console_errors={len(console_errors)}",
flush = True,
)
if console_errors:
print(
f"[ui] first console.error: {console_errors[0][:200]!r}",
flush = True,
)
if page_errors:
print(f"[ui] first pageerror: {page_errors[0][:200]!r}", flush = True)
try:
shoot(f"01-change-password-attempt-{_form_attempt + 1}-fail")
except Exception:
pass
if _form_attempt < 2:
# ERR_NO_BUFFER_SPACE needs the OS to recover socket buffers.
if "ERR_NO_BUFFER_SPACE" in str(e):
backoff_s = 5 if _form_attempt == 0 else 15
print(
f"[ui] ENOBUFS detected; sleeping {backoff_s}s "
f"before retry to let OS recover socket buffers...",
flush = True,
)
time.sleep(backoff_s)
page = recover_or_replace_page(
page,
ctx,
default_timeout_ms = 60_000,
info = lambda m: print(f"[ui] recovery: {m}", flush = True),
)
if form_err is not None:
raise form_err
# 2. Chat surface mounts.
step("wait for composer to mount", NO_STEP_CEILING)
# On macOS racing into wait_for() after the login->chat rebuild can crash the renderer.
try:
page.wait_for_load_state("networkidle", timeout = 30_000)
except Exception:
pass
composer = page.locator('textarea[aria-label="Message input"]')
last_err: Exception | None = None
for _attempt in range(2):
wall_kick()
try:
composer.wait_for(state = "visible", timeout = 60_000)
wait_for_settled_shell(page, composer)
last_err = None
break
except ShellMissingControl:
# The shell booted and is missing a control: a product defect, not a boot failure. No second render.
raise
except Exception as e:
last_err = e
try:
cur_url = page.url
except Exception:
cur_url = "<page closed>"
try:
route_fallback = page.get_by_text(ROUTE_FALLBACK_TEXT, exact = True).count()
except Exception:
route_fallback = -1
print(
f"[ui] composer.wait_for attempt {_attempt + 1} failed: "
f"{type(e).__name__}: {str(e)[:200]}; page.url={cur_url}; "
f"route_fallback={route_fallback}; "
f"page_errors={len(page_errors)} console_errors={len(console_errors)}",
flush = True,
)
if console_errors:
print(
f"[ui] first console.error: {console_errors[0][:200]!r}",
flush = True,
)
if page_errors:
print(f"[ui] first pageerror: {page_errors[0][:200]!r}", flush = True)
try:
shoot(f"03-composer-wait-attempt-{_attempt + 1}-fail")
except Exception:
pass
if _attempt == 0:
page = recover_or_replace_page(
page,
ctx,
default_timeout_ms = 60_000,
goto_url = BASE,
settle_networkidle = True,
info = lambda m: print(f"[ui] recovery: {m}", flush = True),
)
composer = page.locator('textarea[aria-label="Message input"]')
if last_err is not None:
raise last_err
shoot("03-chat-loaded")
exercise_permission_mode_controls(page, shoot)
if PERMISSION_ONLY:
info(
"permission-only run passed "
f"(browser={PLAYWRIGHT_BROWSER}, channel={PLAYWRIGHT_CHANNEL or 'bundled'})"
)
browser.close()
sys.exit(0)
# The frontend stores the bearer under "unsloth_auth_token" (auth/session.ts).
token = robust_evaluate(
page,
"() => localStorage.getItem('unsloth_auth_token')",
)
if not token:
refresh_token = robust_evaluate(
page,
"() => localStorage.getItem('unsloth_auth_refresh_token')",
)
if refresh_token:
refresh_resp = evaluate_fetch(
page,
f"{BASE}/api/auth/refresh",
method = "POST",
headers = {"Content-Type": "application/json"},
body = {"refresh_token": refresh_token},
timeout_ms = FETCH_TIMEOUT_MS,
)
if refresh_resp.get("error"):
fail(f"/api/auth/refresh wedged: {refresh_resp['error']!r}")
refresh = refresh_resp.get("body") or {}
token = (refresh or {}).get("access_token")
next_refresh_token = (refresh or {}).get("refresh_token")
if token and next_refresh_token:
robust_evaluate(
page,
"""([accessToken, refreshToken]) => {
localStorage.setItem('unsloth_auth_token', accessToken);
localStorage.setItem('unsloth_auth_refresh_token', refreshToken);
}""",
[token, next_refresh_token],
)
elif token:
fail("/api/auth/refresh returned access_token but no refresh_token")
if not token:
fail("could not obtain auth token after change-password")
step("default_models[0] matches DEFAULT_MODELS_GGUF[0]")
EXPECTED_DEFAULT = expected_default_model()
defaults_resp = evaluate_fetch(
page,
f"{BASE}/api/models/list",
headers = {"Authorization": f"Bearer {token}"},
timeout_ms = FETCH_TIMEOUT_MS,
)
if defaults_resp.get("error") or defaults_resp.get("status") != 200:
fail(
f"/api/models/list failed: status={defaults_resp.get('status')!r} "
f"error={defaults_resp.get('error')!r}"
)
defaults = defaults_resp["body"] or {}
if not defaults.get("default_models"):
fail(f"/api/models/list returned no default_models: {defaults}")
if defaults["default_models"][0] != EXPECTED_DEFAULT:
fail(
f"default_models[0]={defaults['default_models'][0]!r}, "
f"expected {EXPECTED_DEFAULT!r}; defaults.py drift?"
)
info(f"OK default_models[0] = {EXPECTED_DEFAULT}")
selector_btn = page.locator(
'button:has-text("Select model"), '
'button:has-text("gemma"), '
'button:has-text("Qwen"), '
'button:has-text("Llama")'
).first
sel_text = ""
try:
sel_text = (selector_btn.text_content(timeout = 2_000) or "").strip()
except Exception as _sel_err:
info(f"WARN: model-selector probe skipped: {type(_sel_err).__name__}: {_sel_err}")
if sel_text:
info(f"model selector button text: {sel_text!r}")
shoot("03b-default-model-button")
# 3. Trigger model load via the endpoint the picker uses.
step("load GGUF via /api/inference/load (uses session cookie)", MODEL_STEP_BUDGET_S)
# AbortSignal-bounded: macOS has wedged on this fetch.
load_resp = evaluate_fetch(
page,
f"{BASE}/api/inference/load",
method = "POST",
headers = {
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
body = {
"model_path": GGUF_REPO,
"gguf_variant": GGUF_VARIANT,
"is_lora": False,
"max_seq_length": 2048,
},
timeout_ms = LOAD_FETCH_TIMEOUT_MS,
)
if load_resp.get("error"):
fail(f"/api/inference/load wedged: {load_resp['error']!r}")
if load_resp["status"] != 200:
fail(f"/api/inference/load returned {load_resp['status']}: {load_resp.get('body')!r}")
info(f"loaded model: {(load_resp['body'] or {}).get('display_name')}")
# Model state is cached in zustand; reload so the composer sees the loaded model.
page.reload()
composer = page.locator('textarea[aria-label="Message input"]')
composer.wait_for(state = "visible", timeout = 60_000)
# 3b. Model picker typeahead (no download).
step("model picker: open + drive search bar")
picker_btn = page.locator('[data-tour="chat-model-selector"]').first
if picker_btn.count() == 0:
picker_btn = page.locator(
'button:has-text("gemma-3-270m"), '
'button:has-text("Gemma 3"), '
'button:has-text("Select model")'
).first
if picker_btn.count() == 0:
soft_fail("model picker button not found")
else:
picker_btn.click()
search = wait_for_first(
page.get_by_placeholder(re.compile(r"Search.*models?", re.I)),
timeout_ms = 10_000,
)
shoot("03c-model-picker-open")
if search is None:
soft_fail("model picker search input not found")
else:
# The two queries must show different text, or an ignored-input regression passes silently.
def picker_visible_text():
return robust_evaluate(
page,
"""() => {
const el = document.querySelector(
'[role="dialog"], [role="listbox"], [role="menu"]'
);
return el ? (el.innerText || '').trim() : '';
}""",
)
def picker_text_after(query, previous):
search.fill(query)
try:
page.wait_for_function(
"""([previous]) => {
const el = document.querySelector(
'[role="dialog"], [role="listbox"], [role="menu"]'
);
const text = el ? (el.innerText || '').trim() : '';
const s = (window.__pickerText ||= { text: null, same: 0 });
s.same = text === s.text ? s.same + 1 : 0;
s.text = text;
return text !== previous && s.same >= 3;
}""",
arg = [previous],
polling = 100,
timeout = 10_000,
)
except Exception as exc:
info(f"WARN picker text did not change for {query!r}: {type(exc).__name__}")
return picker_visible_text()
qwen_text = picker_text_after("qwen", picker_visible_text())
shoot("03d-model-picker-search-qwen")
search.fill("")
llama_text = picker_text_after("llama", qwen_text)
shoot("03e-model-picker-search-llama")
if qwen_text and llama_text and qwen_text == llama_text:
soft_fail(
"model picker text was identical for qwen + llama "
"queries -- typeahead may not be filtering"
)
else:
info("OK search bar filtered (qwen text != llama text)")
page.keyboard.press("Escape")
try:
page.get_by_placeholder(re.compile(r"Search.*models?", re.I)).first.wait_for(
state = "hidden", timeout = 10_000
)
except Exception:
pass
# 4. A follow-up 100 ms after a send must queue behind it; without a synchronous per-thread
# reservation it cancels the first turn. 4b. Five chat turns.
prompts = [
"Reply with exactly: hello",
"What is 1+1? Reply with the digit only.",
"Reply with exactly: world",
"Reply with exactly: tree",
"What is 2+2? Reply with the digit only.",
]
def _bubble_count():
"""Total [data-role='assistant'] elements (empty or not)."""
return robust_evaluate(
page,
"""() => {
return document.querySelectorAll('[data-role="assistant"]').length;
}""",
)
def send_and_wait(prompt, idx):
# Send and Stop hot-swap in one DOM slot, so Stop detached alone is racy.
page.wait_for_selector(
'button[aria-label="Send message"]',
state = "attached",
timeout = TURN_TIMEOUT_MS,
)
wall_kick()
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "detached",
timeout = 5_000,
)
except Exception:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "detached",
timeout = TURN_TIMEOUT_MS,
)
wall_kick()
# An empty assistant reply is legitimate (gemma-3-270m at temp 0), so only count bubbles.
bubbles_before = _bubble_count()
# The update banners are fixed toasts that can cover Send; snooze them.
for prefix in ("llama", "web"):
snooze_btn = page.locator(f'[data-testid="{prefix}-update-snooze-button"]')
if snooze_btn.count():
try:
snooze_btn.first.click(timeout = 2_000)
page.wait_for_selector(
f'[data-testid="{prefix}-update-banner"]',
state = "detached",
timeout = 5_000,
)
except Exception:
pass
composer.click()
composer.fill(prompt)
page.locator('button[aria-label="Send message"]').click()
page.wait_for_function(
"""(want) => {
return document.querySelectorAll(
'[data-role="assistant"]'
).length >= want;
}""",
arg = bubbles_before + 1,
timeout = TURN_TIMEOUT_MS,
)
wall_kick()
# Stop may never appear if the model finishes before it paints.
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "attached",
timeout = 3_000,
)
except Exception:
pass
wall_kick()
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "detached",
timeout = TURN_TIMEOUT_MS,
)
except Exception:
shoot(f"04-turn-{idx}-still-streaming")
raise
step("rapid submit: 100 ms follow-up queues behind the first turn", MODEL_STEP_BUDGET_S)
rapid_bubbles_before = _bubble_count()
composer_form = page.locator('form:has(textarea[aria-label="Message input"])').first
# The follow-up runs in the page: a sync route handler blocks this thread, so a wrapped fetch sends
# the follow-up and holds the response itself, keeping the first turn provably running.
page.evaluate(
"""(args) => {
const [secondPrompt, holdMs] = args;
window.__unslothRapid = {
intercepted: false, preparing: false, submitted: false, queueSeen: false,
observed: false, error: null, seen: [], holdUntil: 0,
};
const state = window.__unslothRapid;
const realFetch = window.fetch;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const sendFollowUp = (deadline) => {
if (state.preparing || state.submitted || state.error) return;
if (deadline === undefined) deadline = Date.now() + 5000;
if (state.holdUntil) {
deadline = Math.min(deadline, state.holdUntil - 250);
}
// Re-query, and retry: this is the chat's first message, so
// sending it swaps the welcome composer for the dock composer.
// A node captured earlier is detached, and for a short window
// there is no connected composer at all.
const composer = document.querySelector(
'textarea[aria-label="Message input"]'
);
if (!composer || !composer.isConnected || !composer.form) {
// Never retry past the hold. The response is released when
// it expires, so a submit after that races a buffered reply
// finishing first and would report a queue failure for an
// application that behaved correctly.
if (Date.now() > deadline) {
state.error = "no connected composer for the follow-up";
return;
}
setTimeout(() => sendFollowUp(deadline), 25);
return;
}
// React tracks the value on the node, so a plain assignment is
// reverted on the next render.
const setValue = Object.getOwnPropertyDescriptor(
window.HTMLTextAreaElement.prototype, "value"
).set;
setValue.call(composer, secondPrompt);
composer.dispatchEvent(new Event("input", { bubbles: true }));
// Synthetic input and requestSubmit in the same JS turn can make
// the form callback read the previous controlled value. That is
// not how a user types, and it leaves the new text visible while
// the test incorrectly records a submit. Let React publish the
// input, then re-check the composer because the welcome bar can
// be replaced by the dock bar during this frame.
state.preparing = true;
requestAnimationFrame(() => {
state.preparing = false;
const current = document.querySelector(
'textarea[aria-label="Message input"]'
);
if (
!current || !current.isConnected || !current.form ||
current.value !== secondPrompt
) {
if (Date.now() > deadline) {
state.error = "follow-up composer did not settle";
return;
}
setTimeout(() => sendFollowUp(deadline), 25);
return;
}
current.form.requestSubmit();
state.submitted = true;
});
};
window.fetch = async (...a) => {
const url = String(
(a[0] && a[0].url) ? a[0].url : a[0]
);
const isTurn = url.includes("chat/completions");
if (isTurn && !state.intercepted) {
state.intercepted = true;
state.holdUntil = Date.now() + holdMs;
state.seen.push(url);
const response = realFetch(...a);
// The request is out and the turn is running. Send the
// follow-up now, then keep the response pending so it
// cannot complete first.
setTimeout(() => sendFollowUp(), 0);
await sleep(holdMs);
return response;
}
return realFetch(...a);
};
// Always restore. A wrapper left in place for the rest of the
// run is a monkeypatch with no teardown, and every later turn would
// pay for it.
window.__unslothRapidArm = () => setTimeout(() => sendFollowUp(), 100);
window.__unslothRapidRestore = () => {
window.fetch = realFetch;
clearInterval(poll);
};
const poll = setInterval(() => {
if (document.querySelector(
'button[aria-label="Remove queued prompt 1"]'
)) {
state.queueSeen = true;
clearInterval(poll);
}
}, 25);
setTimeout(() => {
clearInterval(poll);
if (!state.queueSeen && !state.error) {
// Resolve the wait rather than let it die on a generic
// Playwright timeout. That path is the regression this step
// exists to catch, and leaving it unresolved makes the
// screenshot, the explicit message and the fetch teardown
// below unreachable in exactly that case.
state.observed = true;
}
}, 20000);
}""",
["Reply with exactly: rapid-second", int(RAPID_FIRST_TURN_HOLD_S * 1000)],
)
composer.fill("Reply with exactly: rapid-first")
composer_form.evaluate("form => form.requestSubmit()")
page.evaluate("() => window.__unslothRapidArm && window.__unslothRapidArm()")
page.wait_for_function(
"""() => window.__unslothRapid
&& (window.__unslothRapid.queueSeen
|| window.__unslothRapid.observed
|| window.__unslothRapid.error)""",
timeout = 60_000,
)
state = page.evaluate("() => window.__unslothRapid")
page.evaluate("() => { if (window.__unslothRapidRestore) window.__unslothRapidRestore(); }")
if state["error"]:
shoot("04-rapid-submit-no-composer")
fail(f"could not send the follow-up: {state['error']}")
# Only demand the interception when no queue formed, so an unheld run cannot pass silently.
if not state["queueSeen"] and not state["intercepted"]:
fail(
"the first turn's request was never seen, so it was never held, "
f"and no queue formed; saw {state['seen']}"
)
if not state["queueSeen"]:
shoot("04-rapid-submit-no-queue")
fail(
"follow-up sent during a held first turn did not appear as queued "
f"work (submitted={state['submitted']}, intercepted="
f"{state['intercepted']})"
)
wall_kick()
page.wait_for_function(
"""(want) => {
const replies = Array.from(
document.querySelectorAll('[data-role="assistant"]')
).slice(-2);
return replies.length === 2 &&
document.querySelectorAll('[data-role="assistant"]').length >= want &&
!document.querySelector('button[aria-label="Stop generating"]') &&
!document.querySelector('button[aria-label="Remove queued prompt 1"]');
}""",
arg = rapid_bubbles_before + 2,
timeout = TURN_TIMEOUT_MS * 2,
)
shoot("04-rapid-submit-queued")
info("OK 100 ms follow-up waited and both assistant turns completed")
for i, p_ in enumerate(prompts, start = 1):
step(f"turn {i}: {p_!r}", MODEL_STEP_BUDGET_S)
send_and_wait(p_, i)
shoot("04-after-five-turns")
texts = robust_evaluate(
page,
"""() => Array.from(document.querySelectorAll('[data-role="assistant"]'))
.map(e => (e.innerText || '').trim())""",
)
if len(texts) < len(prompts):
fail(f"expected >= {len(prompts)} assistant bubbles, got {len(texts)}")
info(f"five turn lengths = {[len(t) for t in texts[:5]]}")
if chat_completions_responses:
statuses = [code for code, _ in chat_completions_responses]
bad = [code for code in statuses if code >= 400]
info(
f"/v1/chat/completions: {len(statuses)} request(s); "
f"statuses={statuses}; 4xx/5xx={len(bad)}"
)
# 5. Regenerate the last assistant turn.
step("regenerate last assistant turn", MODEL_STEP_BUDGET_S)
last_assistant = page.locator('[data-role="assistant"]').last
last_assistant.hover()
wait_for_first(last_assistant.locator(".aui-assistant-action-bar-root"), timeout_ms = 10_000)
# Exclude disabled controls: the picker's disabled "Reload model" button also matches and sorts first.
regen_btn = (
page.get_by_role(
"button",
name = re.compile(r"(reload|regenerate)", re.I),
)
.and_(page.locator("button:not([disabled])"))
.first
)
if regen_btn.count() > 0:
regen_btn.click()
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "detached",
timeout = 90_000,
)
except Exception:
pass
shoot("05-after-regenerate")
info("regenerate completed")
else:
# TODO: add a data-testid to the reload button; the tooltip-text locator is fragile.
info("WARN regenerate button not visible (known-fragile locator, skipped)")
# 6. Two more turns after regenerate.
extra = ["Reply with: yes", "Reply with: no"]
for j, p_ in enumerate(extra, start = 1):
step(f"extra turn {j}: {p_!r}", MODEL_STEP_BUDGET_S)
before_count = len(page.locator('[data-role="assistant"]').all())
send_and_wait(p_, before_count + 1)
shoot("06-after-extra-turns")
# By id, so Recents reopens THIS chat and not one an earlier run left with the same prompts.
our_thread_id = (
robust_evaluate(
page,
"""() => new URLSearchParams(location.search).get("thread")
|| document.querySelector('[data-testid="recent-thread"][data-active="true"]')
?.getAttribute("data-thread-id")
|| ''""",
)
or ""
)
info(f"this run's chat: {our_thread_id or 'no thread id on the page'}")
# ─────────────────────────────────────────────────────
# Toggle aria-labels flip Disable X / Enable X with state (shared-composer.tsx).
# ─────────────────────────────────────────────────────
step("composer toggle buttons (Thinking / Web search / Tools)")
for feature in ("thinking", "web search", "tools"):
# Match whichever of "Disable X" / "Enable X" is rendered.
toggle = page.locator(
f'button[aria-label="Disable {feature}"], button[aria-label="Enable {feature}"]'
).first
if toggle.count() == 0:
info(f"toggle '{feature}' not present on this layout")
continue
if toggle.is_disabled():
info(f"toggle '{feature}' is disabled for this model -- skip")
continue
before = toggle.get_attribute("aria-label") or ""
toggle.click()
toggle_now = page.locator(
f'button[aria-label="Disable {feature}"], button[aria-label="Enable {feature}"]'
).first
try:
expect(toggle_now).not_to_have_attribute("aria-label", before, timeout = 5_000)
except AssertionError:
pass
after = (
page.locator(
f'button[aria-label="Disable {feature}"], button[aria-label="Enable {feature}"]'
).first.get_attribute("aria-label")
or ""
)
if before == after:
info(f"WARN '{feature}' aria-label did not flip ({before!r})")
else:
info(f"OK '{feature}': {before!r} -> {after!r}")
try:
page.locator(
f'button[aria-label="Disable {feature}"], button[aria-label="Enable {feature}"]'
).first.click()
expect(toggle_now).to_have_attribute("aria-label", before, timeout = 5_000)
except Exception:
pass
shoot("07-toggles-cycled")
# 8. Configuration sheet.
cfg_open = page.locator('button[aria-label="Open configuration"]').first
if cfg_open.count() > 0:
step("Configuration sheet: drive Temperature + Top P + extras")
cfg_open.click()
sliders = page.locator('[role="slider"]')
wait_for_first(sliders, timeout_ms = 10_000)
shoot("08-config-open")
n_sliders = sliders.count()
info(f"configuration sheet exposes {n_sliders} slider(s)")
for idx in range(n_sliders):
try:
s = sliders.nth(idx)
s.scroll_into_view_if_needed()
s.focus()
page.keyboard.press("Home")
expect(s).to_have_attribute(
"aria-valuenow", s.get_attribute("aria-valuemin") or "", timeout = 5_000
)
except Exception as exc:
info(f" slider[{idx}] focus/Home failed: {exc!r}")
shoot("09-config-all-min")
info("Temperature set to slider min (0.0) for determinism")
close_btn = page.locator('button[aria-label="Close configuration"]').first
if close_btn.count() > 0:
close_btn.click()
else:
page.keyboard.press("Escape")
try:
expect(sliders).to_have_count(0, timeout = 10_000)
except AssertionError:
pass
def read_chat_typography():
"""Read message typography after a user-driven theme transition."""
return robust_evaluate(
page,
"""() => {
const root = document.documentElement;
const assistant = Array.from(
document.querySelectorAll('.aui-assistant-message-root')
);
const user = Array.from(
document.querySelectorAll('.aui-user-message-root')
);
if (assistant.length === 0 || user.length === 0) {
return { error: 'chat message roots are missing' };
}
const ua = navigator.userAgent.toLowerCase();
const role = (nodes) => {
const styles = nodes.map((node) => getComputedStyle(node));
return {
fontWeight: [...new Set(styles.map((style) => style.fontWeight))],
letterSpacing: [...new Set(styles.map((style) => style.letterSpacing))],
// The tracking is authored in em, so it only means anything next to the
// size it resolved against.
fontSize: [...new Set(styles.map((style) => style.fontSize))],
};
};
return {
// The scale the size tokens are multiplied by: index.css sets the 15px
// product default, and the appearance store overrides it inline as
// preference / 16 for any other size. Resolved through a length rather
// than read as a property: since #11648 it is a calc() of the size and
// interface scales, and an unregistered custom property reads back as
// that expression, not a number.
uiFontScale: (() => { const probe = document.createElement('div'); probe.style.cssText = 'position:absolute;visibility:hidden;width:calc(10000px * var(--ui-font-scale, 1))'; document.body.appendChild(probe); const px = parseFloat(getComputedStyle(probe).width); probe.remove(); return String(px / 10000); })(),
actualRenderLinux: root.classList.contains('render-linux'),
isDesktopLinux: ua.includes('linux') && !ua.includes('android'),
isDark: root.classList.contains('dark'),
usesBaselineTypography: (
root.classList.contains('no-font-smoothing') ||
root.hasAttribute('data-chat-font') ||
root.hasAttribute('data-ui-font')
),
assistant: role(assistant),
user: role(user),
};
}""",
)
_TEXT_UI_15P5_PX = 15.5
def assert_chat_typography(label, typography):
if typography.get("error"):
fail(typography["error"])
if typography["actualRenderLinux"] != typography["isDesktopLinux"]:
fail(f"desktop Linux detection mismatch: {typography!r}")
is_dark = typography["isDark"]
# Tracking is authored in em, so assert em and take the size from the element.
expected_em = 0.02 if is_dark else 0.01
if typography["isDesktopLinux"] and not typography["usesBaselineTypography"]:
expected_weight = "350" if is_dark else "390"
if is_dark:
expected_em = 0.023
else:
expected_weight = "410"
for role in ("assistant", "user"):
actual = typography[role]
if actual["fontWeight"] != [expected_weight]:
fail(
f"chat font weight {label}/{role}: expected {expected_weight}, "
f"got {actual['fontWeight']!r}"
)
if len(actual["fontSize"]) != 1:
fail(f"chat font size {label}/{role}: not uniform, got {actual['fontSize']!r}")
font_size = float(actual["fontSize"][0].removesuffix("px"))
# Pin the token: a range across every preference would admit the neighbouring tokens.
try:
ui_font_scale = float(typography.get("uiFontScale") or "1")
except ValueError:
ui_font_scale = None
if ui_font_scale is None:
fail(f"chat font size {label}/{role}: unreadable --ui-font-scale")
expected_size = _TEXT_UI_15P5_PX * ui_font_scale
if abs(font_size - expected_size) > 0.01:
fail(
f"chat font size {label}/{role}: expected text-ui-15p5, "
f"{_TEXT_UI_15P5_PX}px * {ui_font_scale} = {expected_size:g}px, "
f"got {font_size}px"
)
expected_spacing = expected_em * font_size
# float() raises on "normal", which is how zero tracking is reported.
spacings = [
0.0 if v.strip() == "normal" else float(v.removesuffix("px"))
for v in actual["letterSpacing"]
]
if len(spacings) != 1 or abs(spacings[0] - expected_spacing) > 0.005:
fail(
f"chat letter spacing {label}/{role}: expected {expected_em}em of "
f"{font_size}px = {expected_spacing:g}px, got {actual['letterSpacing']!r}"
)
# 9. Theme toggle cycles + computed background check.
acct = page.locator('button[aria-label$=" account menu"]').first
if acct.count() > 0:
step("theme toggle x3 with computed-color assertion")
observed = []
typography_states = []
def wait_menu_closed(timeout_ms):
try:
page.wait_for_function(
"""() => {
const m = document.querySelector('[role="menu"]');
if (!m) return true;
// Radix sets data-state="closed" during the
// close animation; treat that as already gone.
return m.getAttribute('data-state') === 'closed';
}""",
timeout = timeout_ms,
)
except Exception:
pass
for cycle in range(3):
wait_menu_closed(7_000)
# Retry once: the first click can be swallowed mid-view-transition.
opened = False
for attempt in range(2):
try:
click_forced(acct)
except Exception as exc:
if attempt == 1:
soft_fail(f"theme cycle {cycle + 1}: account-menu click failed ({exc!r})")
continue
try:
page.wait_for_selector(
'[role="menu"][data-state="open"]',
timeout = 5_000,
)
opened = True
break
except Exception:
page.keyboard.press("Escape")
wait_menu_closed(5_000)
if not opened:
soft_fail(f"theme cycle {cycle + 1}: account menu didn't open")
break
theme_item = page.get_by_role(
"menuitem",
name = re.compile(r"^(Light Mode|Dark Mode)$", re.I),
).first
if theme_item.count() == 0:
page.keyboard.press("Escape")
soft_fail(f"theme cycle {cycle + 1}: theme menuitem missing")
break
was_dark = robust_evaluate(
page, "() => document.documentElement.classList.contains('dark')"
)
# Small CI viewports can push the item off-screen, so fall back to scroll, then evaluate() click.
click_err = None
for click_attempt in range(3):
try:
if click_attempt == 0:
click_forced(theme_item, timeout = 3_000)
elif click_attempt == 1:
theme_item.scroll_into_view_if_needed(timeout = 2_000)
click_forced(theme_item, timeout = 3_000)
else:
theme_item.evaluate("el => el.click()")
click_err = None
break
except Exception as exc:
click_err = exc
page.wait_for_timeout(200)
if click_err is not None:
page.keyboard.press("Escape")
soft_fail(f"theme cycle {cycle + 1}: theme menuitem click failed ({click_err!r})")
break
# The .dark class is the ground truth. Wait for CSS transitions too: font-weight animates.
try:
page.wait_for_function(
"""([wasDark]) => {
const root = document.documentElement;
const msg = document.querySelector(
'.aui-assistant-message-root, .aui-user-message-root'
);
const ms = msg ? getComputedStyle(msg) : null;
const sig = [
getComputedStyle(document.body).backgroundColor,
ms ? ms.fontWeight + '/' + ms.letterSpacing : '',
].join('|');
const transitioning = document.getAnimations().some(
(a) => a instanceof CSSTransition && a.playState === 'running'
);
const s = (window.__themeSettle ||= { sig: null, same: 0 });
s.same = !transitioning && sig === s.sig ? s.same + 1 : 0;
s.sig = sig;
return root.classList.contains('dark') !== wasDark && s.same >= 3;
}""",
arg = [was_dark],
polling = "raf",
timeout = 10_000,
)
except Exception as exc:
info(f"WARN theme cycle {cycle + 1}: .dark did not flip: {type(exc).__name__}")
bg = robust_evaluate(
page,
"""() => {
const root = document.documentElement;
return {
cls: root.className,
isDark: root.classList.contains('dark'),
bg: getComputedStyle(document.body).backgroundColor,
rbg: getComputedStyle(root).backgroundColor,
};
}""",
)
observed.append(bg)
typography = read_chat_typography()
assert_chat_typography(f"theme-cycle-{cycle + 1}", typography)
typography_states.append(typography)
shoot(f"10-theme-cycle-{cycle + 1}")
info(f" cycle {cycle + 1}: dark={bg['isDark']} body bg={bg['bg']!r}")
# The theme item keeps the menu open on purpose; close it or the next cycle's first click is wasted.
page.keyboard.press("Escape")
wait_menu_closed(5_000)
rgbs = [parse_rgb(o["bg"]) for o in observed if parse_rgb(o["bg"])]
light_seen = any(min(r) > 220 for r in rgbs)
dark_seen = any(max(r) < 60 for r in rgbs)
if len(observed) < 3:
soft_fail(f"theme toggle ran only {len(observed)} cycle(s), expected 3")
# Not strict on both polarities: prefers-color-scheme with the "system" default can collapse them.
if light_seen and dark_seen:
info("OK light + dark computed background colors observed")
else:
info(
f"WARN observed only one polarity across {len(rgbs)} "
f"cycles: light_seen={light_seen}, dark_seen={dark_seen} "
"(toggle may not flip on this runner's color-scheme)"
)
if len(typography_states) != 3:
soft_fail(
f"chat typography observed {len(typography_states)} theme state(s), expected 3"
)
elif {state["isDark"] for state in typography_states} != {False, True}:
soft_fail(f"chat typography did not observe both themes: {typography_states!r}")
else:
info("OK chat typography platform and theme behavior")
else:
soft_fail("chat typography requires the account-menu theme control")
# 10. Sidebar nav.
def click_nav(label, expected_url_pat = None):
# The tooltip-derived accessible name can be empty on macOS with a collapsed sidebar.
candidates = [
page.get_by_role("button", name = re.compile(rf"^\s*{label}\s*$", re.I)).first,
page.locator(f'button:has-text("{label}")').first,
page.locator(f'a:has-text("{label}")').first,
page.locator(f'[data-sidebar="menu-button"]:has-text("{label}")').first,
]
btn = None
for c in candidates:
if c.count() > 0:
btn = c
break
if btn is None:
# Unpinned rows live in the hover-opened "More" flyout; a click would toggle it shut.
more_btn = page.get_by_role("button", name = re.compile(r"^\s*More\s*$", re.I)).first
if more_btn.count() > 0:
more_btn.hover()
items = page.get_by_role("menuitem", name = re.compile(label, re.I))
item = wait_for_first(items, timeout_ms = 5_000)
if item is None:
click_forced(more_btn)
item = wait_for_first(items, timeout_ms = 5_000)
if item is not None:
btn = item
if btn is None:
soft_fail(f"nav '{label}' not found")
return False
url_before = page.url
# force=True: the post-toggle view transition can briefly report <html> as topmost.
try:
click_forced(btn, timeout = 5_000)
except Exception as exc:
soft_fail(f"nav '{label}' click failed: {exc!r}")
return False
if expected_url_pat:
# Wait for a URL different from before the click, since the pattern alone may already match.
try:
page.wait_for_url(
lambda u: u != url_before and re.search(expected_url_pat, u) is not None,
timeout = 5_000,
)
except Exception:
pass
if expected_url_pat and not re.search(expected_url_pat, page.url):
soft_fail(
f"clicking '{label}' didn't change url to /{expected_url_pat}; current: {page.url}"
)
return False
return True
step("sidebar nav: New Chat -> Compare -> Search -> Recipes")
click_nav("New Chat", r"/chat")
shoot("11-new-chat")
plus_btn = page.get_by_role("button", name = re.compile(r"Tools and attachments", re.I)).first
if plus_btn.count() > 0:
click_forced(plus_btn)
compare_items = page.get_by_role("menuitem", name = re.compile(r"Compare chat", re.I))
# The menu renders items together, so once any item exists Compare's absence is real.
wait_for_first(page.get_by_role("menuitem"), timeout_ms = 5_000)
compare_item = compare_items.first
if compare_item.count() == 0:
more_trigger = page.get_by_role("menuitem", name = re.compile(r"^More$", re.I)).first
if more_trigger.count() > 0:
more_trigger.hover()
compare_item = wait_for_first(compare_items, timeout_ms = 5_000)
if compare_item is None:
click_forced(more_trigger)
compare_item = wait_for_first(compare_items, timeout_ms = 5_000)
compare_item = compare_items.first
if compare_item.count() > 0:
url_before = page.url
click_forced(compare_item)
try:
page.wait_for_url(
lambda u: u != url_before and re.search(r"/chat\?", u) is not None,
timeout = 10_000,
)
except Exception:
pass
if not re.search(r"/chat\?", page.url):
soft_fail(f"'Compare chat' didn't open compare; current: {page.url}")
else:
soft_fail("composer + menu: 'Compare chat' item not found")
else:
soft_fail("composer + menu: plus button not found")
shoot("12-compare")
search_btn = page.get_by_role("button", name = re.compile(r"^search$", re.I)).first
if search_btn.count() > 0:
search_btn.click()
search_dialog = page.get_by_role("dialog")
wait_for_first(search_dialog, timeout_ms = 10_000)
shoot("13-search-dialog")
page.keyboard.press("Escape")
try:
expect(search_dialog).to_have_count(0, timeout = 10_000)
except AssertionError:
pass
click_nav("Recipes", r"/data-recipes")
shoot("14-recipes")
page.goto(f"{BASE}/chat")
composer.wait_for(state = "visible", timeout = 60_000)
if acct.count() > 0:
step("Developer (API) tab via account menu")
acct.click()
wait_for_first(page.locator('[role="menu"][data-state="open"]'), timeout_ms = 5_000)
dev = page.get_by_role("menuitem", name = re.compile(r"developer|api", re.I)).first
if dev.count() > 0:
dev.click()
wait_for_first(page.get_by_role("dialog"), timeout_ms = 10_000)
shoot("15-developer-tab")
create_btn = page.get_by_role(
"button",
name = re.compile(r"create.*key|generate.*key|add.*key|new key", re.I),
).first
if create_btn.count() > 0:
info("OK 'create API key' affordance visible")
keys_section = page.get_by_text(
re.compile(r"api keys|developer", re.I),
).first
if keys_section.count() > 0:
info(f"OK API tab text: {(keys_section.text_content() or '').strip()[:80]!r}")
page.keyboard.press("Escape")
try:
expect(page.get_by_role("dialog")).to_have_count(0, timeout = 10_000)
except AssertionError:
pass
else:
page.keyboard.press("Escape")
# 11b. Recipes tab.
step("Recipes tab: cards render + click first card")
page.goto(f"{BASE}/data-recipes")
# The recipe list is read from IndexedDB after the route renders.
wait_for_first(page.locator('[data-tour="recipes-new"]'), timeout_ms = 15_000)
try:
expect(page.locator("main").get_by_text("Loading recipes", exact = True)).to_have_count(
0, timeout = 15_000
)
except AssertionError:
pass
headings = page.locator("main h2, main h3, [data-recipe], a[href*='/data-recipes/']")
n_cards = headings.count()
info(f"Recipes route headings/cards: {n_cards}")
shoot("15b-recipes-cards")
if n_cards > 0:
try:
headings.first.scroll_into_view_if_needed()
list_url = page.url
headings.first.click()
# Wait for the recipe URL: the list page's <main> would satisfy a settle check too early.
try:
page.wait_for_url(
lambda u: u != list_url and re.search(r"/data-recipes/[^/?#]+", u) is not None,
timeout = 10_000,
)
except Exception:
info(f"WARN recipe card click did not open a recipe route; current: {page.url}")
wait_for_settled(page.locator("main"))
shoot("15c-recipes-first-card")
info("OK clicked first recipe card")
except Exception as exc:
info(f"WARN click first recipe failed: {exc!r}")
page.goto(f"{BASE}/chat")
composer = page.locator('textarea[aria-label="Message input"]')
composer.wait_for(state = "visible", timeout = 60_000)
# 11c. Recents: reopen the thread persisted above.
step("Recents: click previous chat in sidebar")
open_recent_thread_with_our_prompts(
page,
["Reply with exactly: rapid-first", "Reply with exactly: rapid-second", *prompts, *extra],
shoot,
our_thread_id = our_thread_id,
)
page.goto(f"{BASE}/chat")
composer = page.locator('textarea[aria-label="Message input"]')
composer.wait_for(state = "visible", timeout = 60_000)
# 12. Attachment button reachable (model is text-only).
step("attachment widget reachable")
attach = page.locator('button[aria-label="Add Attachment"]').first
if attach.count() > 0:
# Only hover: clicking blocks on the native file dialog.
attach.hover()
wait_for_first(page.get_by_role("tooltip"), timeout_ms = 5_000)
shoot("16-attachment-hover")
# 13. Reload + session survives.
step("reload + session survives")
page.reload()
composer.wait_for(state = "visible", timeout = 60_000)
if "/login" in page.url:
fail(f"unexpected redirect to /login after reload: {page.url}")
shoot("17-after-reload")
# 14. /api/health stays healthy.
health = evaluate_fetch(
page,
f"{BASE}/api/health",
timeout_ms = FETCH_TIMEOUT_MS,
)
if health.get("error"):
fail(f"/api/health wedged: {health['error']!r}")
if health["status"] != 200:
fail(f"/api/health returned {health['status']}")
# 15. Negative-auth after UI rotation.
step("post-rotation auth check (after UI change-password)")
if (s_old := login_via_api(OLD)) != 401:
fail(f"old bootstrap pw should be 401, got {s_old}")
if (s_new := login_via_api(NEW)) != 200:
fail(f"rotated pw should be 200, got {s_new}")
info("OK old=401, new=200")
# 16. Out-of-band rotation via curl must revoke refresh tokens, so /api/auth/refresh fails too.
step("rotate password via subprocess(curl) -- the 'terminal' path")
login_proc = subprocess.run(
[
"curl",
"-fsS",
"-X",
"POST",
f"{BASE}/api/auth/login",
"-H",
"Content-Type: application/json",
"-d",
json.dumps({"username": "unsloth", "password": NEW}),
],
capture_output = True,
text = True,
timeout = 15,
)
if login_proc.returncode != 0:
fail(f"curl login failed: {login_proc.stderr!r}")
login_body = json.loads(login_proc.stdout)
cli_token = login_body.get("access_token")
if not cli_token:
fail(f"curl login returned no access_token: {login_body!r}")
info("CLI obtained an access token")
browser_refresh_token = robust_evaluate(
page,
"() => localStorage.getItem('unsloth_auth_refresh_token')",
)
if not browser_refresh_token:
fail("browser refresh token missing before CLI rotation")
change_proc = subprocess.run(
[
"curl",
"-fsS",
"-X",
"POST",
f"{BASE}/api/auth/change-password",
"-H",
"Content-Type: application/json",
"-H",
f"Authorization: Bearer {cli_token}",
"-d",
json.dumps({"current_password": NEW, "new_password": NEW2}),
],
capture_output = True,
text = True,
timeout = 15,
)
if change_proc.returncode != 0:
fail(
f"curl change-password failed: rc={change_proc.returncode} "
f"stderr={change_proc.stderr!r} stdout={change_proc.stdout!r}"
)
info("CLI rotated password NEW -> NEW2 successfully")
if (s_new1 := login_via_api(NEW)) != 401:
fail(f"after CLI rotation, NEW pw should be 401, got {s_new1}")
if (s_new2 := login_via_api(NEW2)) != 200:
fail(f"after CLI rotation, NEW2 pw should be 200, got {s_new2}")
info("OK after CLI rotation: NEW=401, NEW2=200 -- old studio creds dead")
refresh_proc = subprocess.run(
[
"curl",
"-sS",
"-o",
os.devnull,
"-w",
"%{http_code}",
"-X",
"POST",
f"{BASE}/api/auth/refresh",
"-H",
"Content-Type: application/json",
"-d",
json.dumps({"refresh_token": browser_refresh_token}),
],
capture_output = True,
text = True,
timeout = 15,
)
if refresh_proc.returncode != 0:
fail(
f"curl refresh-token check failed: rc={refresh_proc.returncode} "
f"stderr={refresh_proc.stderr!r} stdout={refresh_proc.stdout!r}"
)
try:
refresh_status = int(refresh_proc.stdout.strip())
except ValueError:
fail(f"curl refresh-token check returned invalid status: {refresh_proc.stdout!r}")
if refresh_status == 200:
fail(f"/api/auth/refresh should fail after CLI rotation; got 200")
info(
f"OK browser /api/auth/refresh now {refresh_status} "
"(refresh token revoked) -- old studio session can no longer renew"
)
# 17. Persisted monitor auth boundary, then shutdown. Own step: page.evaluate has no timeout here.
step("persisted monitor: reset the browser session and open a fresh page")
# Fresh page after CLI rotation, in the SAME context (keeps the init script); nothing runs in
# the stale page, whose evaluate can wedge with no timeout; storage writes go via a parked
# same-origin page; each call announces itself first
stale_pages.add(page)
info("closing the stale page")
try:
close_page_without_waiting(ctx, page)
except Exception as exc:
info(f"WARN closing the stale page failed: {exc!r}")
info("clearing stale session cookies")
try:
ctx.clear_cookies()
except Exception as exc:
info(f"WARN clearing stale session cookies failed: {exc!r}")
info("opening the fresh page")
_fresh_page = new_throttled_page(ctx)
_fresh_page.on("pageerror", lambda e: page_errors.append(str(e)))
_fresh_page.on("console", _on_console)
page = _fresh_page
info("parking the fresh page on /api/health to write localStorage")
page.goto(f"{BASE}/api/health", wait_until = "domcontentloaded", timeout = 30_000)
info("writing the monitor overlay and clearing the auth tokens")
robust_evaluate(
page,
"""() => {
localStorage.setItem(
"unsloth_monitor_overlay",
JSON.stringify({ state: { isOpen: true, isMinimized: false }, version: 0 })
);
["unsloth_auth_token", "unsloth_auth_refresh_token"].forEach(
(key) => localStorage.removeItem(key)
);
}""",
)
login_system_request_count = len(system_requests)
step("persisted monitor stays dormant on /login and resumes after auth", NO_STEP_CEILING)
_tolerated_nav = ("ERR_ABORTED", "interrupted by another navigation")
# A slow runner can time out this navigation with the server healthy, so retry the whole sequence.
wait_for_health(BASE, timeout = 30.0, info = info)
relogin_err: Exception | None = None
for _relogin_attempt in range(3):
try:
try:
page.goto(f"{BASE}/login", wait_until = "domcontentloaded", timeout = 60_000)
except Exception as exc:
if not any(t in str(exc) for t in _tolerated_nav):
raise
info(f"goto /login interrupted ({exc!r}); password-field wait will confirm /login")
pw_field = page.locator("#password")
pw_field.wait_for(state = "visible", timeout = 60_000)
page.keyboard.press("Control+,")
# No /api/system request may go out while /login is up; 5.5 s spans more than one monitor poll.
try:
wait_until(
lambda: len(system_requests) != login_system_request_count,
timeout_s = 5.5,
what = "an /api/system request while /login is active",
interval_s = 0.1,
page = page,
)
except TimeoutError:
pass
if len(system_requests) != login_system_request_count:
raise AssertionError(
"persisted monitor requested /api/system while /login was active"
)
if "/login" not in page.url:
raise AssertionError(f"login route reloaded or redirected unexpectedly: {page.url}")
pw_field.fill(NEW2)
status, _ = click_and_wait_for_response(
page,
url_substr = "/api/auth/login",
method = "POST",
do_click = lambda: page.locator('button[type="submit"]').click(),
timeout_ms = 30_000,
info = lambda m: print(f"[ui] {m}", flush = True),
)
if status is not None and status >= 400:
raise AssertionError(
f"login POST returned {status}; see console_errors={console_errors[:1]!r}"
)
relogin_err = None
break
except Exception as e:
relogin_err = e
try:
cur_url = page.url
except Exception:
cur_url = "<page closed>"
print(
f"[ui] re-login attempt {_relogin_attempt + 1} failed: "
f"{type(e).__name__}: {str(e)[:200]}; page.url={cur_url}; "
f"page_errors={len(page_errors)} console_errors={len(console_errors)}",
flush = True,
)
if console_errors:
print(
f"[ui] first console.error: {console_errors[0][:200]!r}",
flush = True,
)
if page_errors:
print(f"[ui] first pageerror: {page_errors[0][:200]!r}", flush = True)
try:
shoot(f"18-relogin-attempt-{_relogin_attempt + 1}-fail")
except Exception:
pass
if _relogin_attempt < 2:
# ERR_NO_BUFFER_SPACE needs the OS to recover socket buffers.
if "ERR_NO_BUFFER_SPACE" in str(e):
backoff_s = 5 if _relogin_attempt == 0 else 15
print(
f"[ui] ENOBUFS detected; sleeping {backoff_s}s "
f"before retry to let OS recover socket buffers...",
flush = True,
)
time.sleep(backoff_s)
old_page = page
page = recover_or_replace_page(
page,
ctx,
default_timeout_ms = 60_000,
info = lambda m: print(f"[ui] recovery: {m}", flush = True),
)
# A replacement page loses the pageerror/console listeners; re-attach them.
if page is not old_page:
page.on("pageerror", lambda e: page_errors.append(str(e)))
page.on("console", _on_console)
if relogin_err is not None:
raise relogin_err
# Outside the retry: re-goto /login after tokens are set would redirect to /chat and mask a slow mount.
composer = page.locator('textarea[aria-label="Message input"]')
composer.wait_for(state = "visible", timeout = 60_000)
monitor_deadline = time.time() + 10
while len(system_requests) == login_system_request_count and time.time() < monitor_deadline:
page.wait_for_timeout(100)
if len(system_requests) == login_system_request_count:
fail("persisted monitor did not resume /api/system polling after login")
if page.get_by_role("dialog", name = re.compile(r"^Settings$")).count() != 0:
fail("settings shortcut on /login left the dialog open after authentication")
info("OK persisted monitor stayed dormant on /login and resumed after authentication")
exercise_floating_monitor_geometry(page)
shoot("18-relogin-with-NEW2")
step("Shutdown via account menu")
acct_btn = page.locator('button[aria-label$=" account menu"]').first
if acct_btn.count() == 0:
fail("account menu button missing -- can't reach Shutdown")
acct_btn.click()
shutdown_item = wait_for_first(
page.get_by_role("menuitem", name = re.compile(r"^\s*Shutdown\s*$", re.I)),
timeout_ms = 10_000,
)
if shutdown_item is None:
fail("Shutdown menuitem not in account menu")
shutdown_item.click()
shoot("19-shutdown-dialog")
stop_btn = page.get_by_role(
"button",
name = re.compile(r"^\s*Stop server\s*$", re.I),
).first
stop_btn.wait_for(state = "visible", timeout = 5_000)
stop_btn.click()
try:
page.wait_for_function(
"""() => /Unsloth has stopped/.test(document.body.innerText)""",
timeout = 15_000,
)
shoot("20-shutdown-placeholder")
info("OK 'Unsloth has stopped' placeholder rendered")
except Exception as exc:
info(f"WARN shutdown placeholder didn't render: {exc!r}")
host = re.sub(r"^https?://", "", BASE).split(":")[0]
port = int(re.search(r":(\d+)", BASE).group(1)) if ":" in BASE else 80
deadline = time.time() + 15
while time.time() < deadline:
try:
with socket.create_connection((host, port), timeout = 1):
pass
time.sleep(0.5)
except (ConnectionRefusedError, OSError):
info("OK port closed -- server process is gone")
break
else:
try:
r = urllib.request.urlopen(f"{BASE}/api/health", timeout = 2)
fail(f"server still up after Shutdown click; /api/health={r.status}")
except urllib.error.URLError as exc:
info(f"OK /api/health unreachable: {exc!r}")
# The port closes as soon as shutdown starts; the process stays up while it stops its children
# (llama-server: up to 5 s SIGTERM + 5 s SIGKILL). When CI names the server (STUDIO_PID from
# boot-studio-api-only.sh), the cleanup must also finish: a closed port alone no longer shows it.
studio_pid = os.environ.get("STUDIO_PID", "").strip()
if studio_pid.isdigit() and os.name != "nt": # Windows os.kill(pid, 0) would terminate it
t0 = time.time()
while time.time() - t0 < 60:
try:
os.kill(int(studio_pid), 0)
if Path("/proc/self").exists(): # Linux: exited but not yet reaped is done
try:
stat = Path(f"/proc/{studio_pid}/stat").read_text(encoding = "utf-8")
except FileNotFoundError: # reaped between the probe and the read
raise ProcessLookupError from None
if stat.rsplit(")", 1)[1].split()[0] == "Z":
raise ProcessLookupError
except ProcessLookupError:
info(
f"OK server process {studio_pid} exited {time.time() - t0:.1f}s after the port closed"
)
break
except PermissionError:
break # not ours to signal: nothing to check
time.sleep(0.5)
else:
fail(f"server process {studio_pid} still running 60s after Shutdown closed its port")
# Some pageerrors are benign: chat-completions 422s (network-layer bubble-up, not a JS bug; per-turn flow already
# validates each turn) and fetch failures after Shutdown (server is dead by design). Full list in
# `_playwright_robust.BENIGN_PAGE_ERROR_PATTERNS`.
real_errors = [e for e in page_errors if not is_benign_page_error(e)]
real_console_errors = [e for e in console_errors if not is_benign_console_error(e)]
if page_errors:
info(
f"WARN page errors: {len(page_errors)} total "
f"({len(real_errors)} non-benign); first: {page_errors[0]!r}"
)
if real_errors:
fail(f"{len(real_errors)} non-benign pageerror events")
info(
f"console.error events: {len(console_errors)} total ({len(real_console_errors)} non-benign)"
)
info("PASS comprehensive UI flow")
_watchdog.cancel()
browser.close()