Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio CI: WebKit leg for the long-chat gate, report-only paint / CPU length check, recheck gap fixed (#17331)

Daniel Han3fe5adecf7c2
diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml
index 9ca55097561..21284218b0e 100644
--- a/.github/workflows/studio-frontend-ci.yml
+++ b/.github/workflows/studio-frontend-ci.yml
@@ -46,6 +46,9 @@ on:
- 'tests/studio/test_css_culprit.py'
- 'tests/studio/_area_matrix.py'
- 'tests/studio/test_area_matrix.py'
+ - 'tests/studio/_webkit_leg.py'
+ - 'tests/studio/test_webkit_leg.py'
+ - 'tests/studio/test_webkit_leg_ci.py'
- 'tests/studio/_appearance_coverage.py'
- 'tests/studio/test_appearance_coverage.py'
- 'tests/studio/_has_lint.py'
@@ -105,6 +108,9 @@ on:
- 'tests/studio/test_css_culprit.py'
- 'tests/studio/_area_matrix.py'
- 'tests/studio/test_area_matrix.py'
+ - 'tests/studio/_webkit_leg.py'
+ - 'tests/studio/test_webkit_leg.py'
+ - 'tests/studio/test_webkit_leg_ci.py'
- 'tests/studio/_appearance_coverage.py'
- 'tests/studio/test_appearance_coverage.py'
- 'tests/studio/_has_lint.py'
@@ -431,6 +437,8 @@ jobs:
tests/studio/test_code_block_flicker_contract.py \
tests/studio/test_css_culprit.py \
tests/studio/test_area_matrix.py \
+ tests/studio/test_webkit_leg.py \
+ tests/studio/test_webkit_leg_ci.py \
tests/studio/test_appearance_coverage.py \
tests/studio/test_has_lint.py -q
@@ -685,6 +693,72 @@ jobs:
path: studio/frontend/dist
retention-days: 1
+ # WebKit leg of the long-chat gate (tests/studio/_webkit_leg.py), in parallel with the build job so
+ # its WebKit install and run stay off that job's critical path. Times opening and closing the
+ # browser pane, Settings, shortcuts, palette, composer menu, sidebar and a thread scroll at 1 and 150
+ # turns in Playwright WebKit with a macOS and a Linux user agent (Safari, the macOS app, the Linux
+ # desktop app): fails when the long chat is over 2.5x the short one and over 200 ms, or has a frame
+ # over 250 ms and over 2.5x the short chat's, beyond the known costs pinned in that file.
+ webkit:
+ name: Long-chat gate (WebKit)
+ runs-on: ubuntu-latest
+ env:
+ UNSLOTH_DISABLE_UPDATE_CHECK: '1'
+ timeout-minutes: 40
+ defaults:
+ run:
+ working-directory: studio/frontend
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+ with:
+ node-version: '22'
+
+ - name: Upgrade npm to 11.x (allowScripts enforcement)
+ working-directory: ${{ github.workspace }}
+ run: |
+ npm install -g npm@^11 --no-fund --no-audit
+ V=$(npm -v)
+ case "$V" in
+ 11.1[6-9].*|11.[2-9][0-9].*|1[2-9].*) echo "npm $V has allowScripts" ;;
+ *) echo "::error::npm $V lacks allowScripts (need >=11.16)"; exit 1 ;;
+ esac
+
+ # Must run BEFORE npm ci, where a compromised tarball's install scripts run.
+ - name: Lockfile supply-chain audit (pre-install scan)
+ working-directory: ${{ github.workspace }}
+ run: python3 scripts/lockfile_supply_chain_audit.py
+
+ - name: Install dependencies
+ run: npm ci --strict-allow-scripts --no-fund --no-audit
+
+ - name: WebKit for the long-chat gate
+ working-directory: ${{ github.workspace }}
+ shell: bash
+ timeout-minutes: 10
+ env:
+ APT_ACQUIRE_RETRIES: '0'
+ run: |
+ python3 -m pip install 'playwright>=1.45,<2'
+ python3 -m playwright install webkit
+ if ! python3 -c "from playwright.sync_api import sync_playwright as s; p = s().start(); p.webkit.launch().close(); p.stop()"; then
+ bash .github/scripts/retry-with-apt-lock.sh python3 -m playwright install-deps webkit
+ fi
+
+ - name: Long-chat production build
+ run: npx vite build --config vite.long-thread.config.ts
+
+ - name: Browser gate for long-chat cost in WebKit
+ working-directory: ${{ github.workspace }}
+ shell: bash
+ timeout-minutes: 20
+ env:
+ LONG_THREAD_WEBKIT: '1'
+ run: python3 tests/studio/_webkit_leg.py
+
# The suite reads sources and resolves module URLs, so it can break only on Windows.
# .gitattributes pins LF for studio/frontend, so exact-substring assertions hold.
windows:
diff --git a/.github/workflows/workflow-trigger-lint.yml b/.github/workflows/workflow-trigger-lint.yml
index 49903e381b5..8bb5ad37223 100644
--- a/.github/workflows/workflow-trigger-lint.yml
+++ b/.github/workflows/workflow-trigger-lint.yml
@@ -75,6 +75,7 @@ jobs:
run: |
python3 -m pytest -q -n 4 \
tests/studio/test_absorbed_lanes_still_run.py \
+ tests/studio/test_webkit_leg_ci.py \
tests/studio/test_appimage_toolchain_contract.py \
tests/studio/test_lint_ci_scans_committed_files.py \
tests/studio/test_agent_guides_verdicts.py \
diff --git a/tests/studio/_area_matrix.py b/tests/studio/_area_matrix.py
index 6e486062a4e..044e73cd51f 100644
--- a/tests/studio/_area_matrix.py
+++ b/tests/studio/_area_matrix.py
@@ -38,6 +38,7 @@ import http.server
import json
import os
import re
+import statistics
import threading
import time
from pathlib import Path
@@ -60,6 +61,18 @@ GLASS = "liquid-3d"
LENGTH_RATIO = float(os.environ.get("LONG_THREAD_AREAS_LENGTH_RATIO", "2.0"))
# Counts below this many elements / layout objects at LONG turns never fail the length check.
COUNT_FLOOR = int(os.environ.get("LONG_THREAD_AREAS_COUNT_FLOOR", "3000"))
+# Paint and main-thread CPU, long over short chat (the cheapest long reading over the median short one,
+# each from the first reading and its rechecks):
+# fails over TIME_LENGTH_RATIO AND over TIME_LENGTH_FLOOR_MS more. A short reading below
+# TIME_BASE_FLOOR_MS counts as that, so a near-free area cannot make a large ratio.
+TIME_LENGTH_RATIO = float(os.environ.get("LONG_THREAD_AREAS_TIME_LENGTH_RATIO", "2.5"))
+# Report-only for now: the readings, ratios and a "would fail" list are printed, the exit code is left
+# alone. Main reads up to the ceilings below on GitHub runners (no margin yet) and the cost behind them
+# is message-level actions (click, menu, tool / reasoning toggles, source card), being worked on
+# separately. Set True (or LONG_THREAD_AREAS_LENGTH_TIME_BLOCKING=1) to make them fail the gate.
+LENGTH_TIME_BLOCKING = os.environ.get("LONG_THREAD_AREAS_LENGTH_TIME_BLOCKING", "0") == "1"
+TIME_LENGTH_FLOOR_MS = float(os.environ.get("LONG_THREAD_AREAS_TIME_LENGTH_FLOOR_MS", "50"))
+TIME_BASE_FLOOR_MS = float(os.environ.get("LONG_THREAD_AREAS_TIME_BASE_FLOOR_MS", "20"))
MAX_RATIO = float(os.environ.get("LONG_THREAD_AREAS_MAX_RATIO", "4.0"))
# Also judged on CPU growth with chat length: a streamed reply restyles only its own message, so its
# element / layout counts stay flat while per-token work over the thread grows.
@@ -87,6 +100,30 @@ KNOWN_DEFAULT = ",".join(
+ [f"{m}:thread_scroll=7" for m in _GLASS_MODES]
+ [f"{m}:settings_scroll=7" for m in _GLASS_MODES]
+ [f"{m}:type=6" for m in _GLASS_MODES]
+ # Main-thread CPU length (renderer main thread, 40 -> 200 turns, judged value, 3 GitHub-runner gate
+ # runs of main, default and Liquid 3D, probes that do not themselves scale with the chat). Presses
+ # that land inside a message (click, its menu, tool / reasoning toggles, source card) still read
+ # 2.5-2.9x; a few sidebar / composer surfaces touch 2.4-2.5x, inside the 0.4x run-to-run spread of
+ # the 2.5x rule. Ceilings: 3.5x over 2.5, 3.0x at 2.4-2.5. Everything else keeps the plain 2.5x.
+ # Fix work: per-interaction script cost that grows with the mounted thread (not yet assigned);
+ # remove each entry as its area drops under 2.5x.
+ + [f"cpu:{a}=3.5" for a in ("tool_toggle", "message_click", "message_menu")] # 2.0-2.9x
+ + [
+ f"cpu:{a}=3" # 1.6-2.5x
+ for a in (
+ "reasoning_toggle",
+ "source_card",
+ "chat_row_menu",
+ "sidebar_sections",
+ "run_settings",
+ "thread_scroll",
+ "permission_menu",
+ "chat_options",
+ "organize_chats",
+ "sidebar_toggle",
+ "api_monitor",
+ )
+ ]
)
KNOWN = os.environ.get("LONG_THREAD_AREAS_KNOWN", KNOWN_DEFAULT)
@@ -709,7 +746,17 @@ class Api:
"/api/hub/local": {"models": []},
"/api/hub/hidden-models": {"hidden": []},
"/api/settings/pinned-models": {"pinned": []},
- "/api/health": {"status": "ok"},
+ # A settled hardware verdict, as the real backend gives once detection ends. Without
+ # `version` / `chat_only` the app keeps capabilities "unknown" and the sidebar Train row
+ # spins for the whole run: an always-running animation, and Chromium then recomputes every
+ # IntersectionObserver target on each frame, so every action's cost grew with the chat.
+ "/api/health": {
+ "status": "healthy",
+ "version": "2026.11.1",
+ "device_type": "linux",
+ "chat_only": False,
+ "hardware_detecting": False,
+ },
"/api/inference/status": {
"active_model": STUB_MODEL,
"model_identifier": STUB_MODEL,
@@ -762,8 +809,14 @@ PROBE = (
"""() => {
const OVER = '%s';
const vis = (e) => { const r = e.getBoundingClientRect(); return r.width > 1 && r.height > 1; };
- window.__gateOverlays = () => [...document.querySelectorAll(OVER)].filter(vis).length;
- window.__gateVisible = (sel) => [...document.querySelectorAll(sel)].some(vis);
+ // Polled every frame inside the measured window, so never a document-wide query: that would cost the
+ // gate's own CPU in proportion to the chat. Message bodies (nearly all of a long chat) are skipped; no
+ // overlay or waited-for control renders inside one (portals sit under <body>).
+ const skip = { acceptNode: (e) => (e.hasAttribute('data-role') ? NodeFilter.FILTER_REJECT : NodeFilter.FILTER_ACCEPT) };
+ const outside = (sel) => { const out = []; const w = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, skip);
+ for (let e = w.nextNode(); e; e = w.nextNode()) if (e.matches(sel)) out.push(e); return out; };
+ window.__gateOverlays = () => outside(OVER).filter(vis).length;
+ window.__gateVisible = (sel) => outside(sel).some(vis);
window.__gateFrames = (n) => new Promise((r) => { let k = 0; const f = () => (++k >= n ? r() : requestAnimationFrame(f)); requestAnimationFrame(f); });
window.__gateFocusins = [];
document.addEventListener('focusin', (e) => window.__gateFocusins.push(e.target), true);
@@ -831,7 +884,9 @@ TABS = """async (timeoutMs) => {
return {count: tabs.length, missing, failed};
}"""
FIND = """(spec) => {
- document.querySelectorAll('[data-gate-target]').forEach((e) => e.removeAttribute('data-gate-target'));
+ // The target is kept by reference: a document-wide lookup would cost in proportion to the chat.
+ if (window.__gateTarget) window.__gateTarget.removeAttribute('data-gate-target');
+ window.__gateTarget = null;
const vis = (e) => { const r = e.getBoundingClientRect(); return r.width > 0 && r.height > 0; };
let el = null;
if (spec.startsWith('js|')) el = (0, eval)(spec.slice(3));
@@ -843,13 +898,13 @@ FIND = """(spec) => {
if (!el) return false;
// Instant: the thread viewport is scroll-smooth, so a plain scrollIntoView animates and returns while the
// target is still moving; a trusted pointer action then lands wherever the target has got to.
- el.setAttribute('data-gate-target', ''); el.scrollIntoView({block: 'nearest', behavior: 'instant'});
+ el.setAttribute('data-gate-target', ''); window.__gateTarget = el; el.scrollIntoView({block: 'nearest', behavior: 'instant'});
return true;
}"""
# The target must hold still and be hit at its centre: the floating composer can cover a row scrolled in from
# below. Only an occluded target is re-centred, so a clear one adds no scroll to the measured window.
STEADY = """async () => {
- const el = document.querySelector('[data-gate-target]');
+ const el = window.__gateTarget;
if (!el) return false;
const frame = () => new Promise((r) => requestAnimationFrame(() => r()));
const still = async () => {
@@ -867,7 +922,7 @@ STEADY = """async () => {
return hit();
}"""
CLICK = """(right) => {
- const el = document.querySelector('[data-gate-target]'); const r = el.getBoundingClientRect();
+ const el = window.__gateTarget; const r = el.getBoundingClientRect();
const at = {bubbles: true, cancelable: true, composed: true, clientX: r.x + r.width / 2, clientY: r.y + r.height / 2,
button: right ? 2 : 0, buttons: right ? 2 : 1, pointerType: 'mouse', isPrimary: true, view: window};
el.dispatchEvent(new PointerEvent('pointerdown', at)); el.dispatchEvent(new MouseEvent('mousedown', at));
@@ -1066,6 +1121,11 @@ class Page:
def close(self) -> None:
self.ctx.close()
+ def target(self):
+ # A handle, not a locator: Playwright's selector engine walks the whole document (for shadow
+ # roots) on every resolve, inside the measured window, at a cost that grows with the chat.
+ return self.page.evaluate_handle("() => window.__gateTarget").as_element()
+
def chord(self, arg) -> str:
if isinstance(arg, dict):
return arg["mac" if self.mac else "other"]
@@ -1114,12 +1174,12 @@ class Page:
self.last_error = "target never held still and clear of overlays under its centre"
return False
if kind == "hover":
- page.locator("[data-gate-target]").first.hover(timeout = 3_000, force = True)
+ self.target().hover(timeout = 3_000, force = True)
self.hovered = True
return True
if kind == "pclick":
# A trusted click (mouse move, press, native focus), for areas whose cost is the focus change.
- page.locator("[data-gate-target]").first.click(timeout = 3_000, force = True)
+ self.target().click(timeout = 3_000, force = True)
self.hovered = True
return True
# Clicks are dispatched in the page: the pointer stays parked, so no tooltip or :hover state
@@ -1383,6 +1443,10 @@ def split_trace(events: list) -> dict[str, dict]:
row["layout_objects"] += int((args.get("beginData") or {}).get("dirtyObjects") or 0)
elif name in ("Paint", "PrePaint", "Layerize", "PaintImage"):
row["paint_ms"] += dur
+ for row in out.values():
+ # The renderer's main thread alone: raster and compositor threads (busy on every frame under
+ # software compositing) are noise for a "does this area do more work in a longer chat" check.
+ row["main_ms"] = row["threads"].get("CrRendererMain", 0.0)
return out
@@ -1657,6 +1721,15 @@ def cheapest(cell: dict, key: str) -> float:
return min(vals)
+def typical(cell: dict, key: str) -> float:
+ """Median of the first reading and its rechecks: the yardstick a length ratio divides by. The
+ cheapest short reading would let one unusually quiet sample inflate every ratio taken over it."""
+ vals = [cell.get(key, 0.0)] + [
+ c.get(key, 0.0) for c in cell.get("again", []) if c.get("status") == "OK"
+ ]
+ return statistics.median(vals)
+
+
def judge(result: dict, known: str | None = None) -> list[dict]:
"""Verdict rows: kind length / appearance / broken; `fail` set on the failing ones."""
limits = known_limits(KNOWN if known is None else known)
@@ -1761,6 +1834,30 @@ def judge(result: dict, known: str | None = None) -> list[dict]:
lc, sc = settled(page["cells"].get(area, {})), settled(short.get(area, {}))
if lc.get("status") != "OK" or sc.get("status") != "OK":
continue
+ # Time grows where counts may not (paint behind an overlay grows with the thread while style
+ # and layout stay flat): paint and main-thread CPU, the cheapest long reading over the
+ # median short one (a single quiet short sample cannot inflate the ratio).
+ for metric, key in (("paint_ms", "paint"), ("main_ms", "cpu")):
+ lv, sv = cheapest(lc, metric), typical(sc, metric)
+ ratio = lv / max(sv, TIME_BASE_FLOOR_MS)
+ tlimit = limits.get(
+ (f"{key}@{mode}", area), limits.get((key, area), TIME_LENGTH_RATIO)
+ )
+ out.append(
+ {
+ "kind": "length",
+ "metric": metric,
+ "turns": turns,
+ "mode": mode,
+ "area": area,
+ "value": lv,
+ "base_value": sv,
+ "ratio": ratio,
+ "limit": tlimit,
+ "would_fail": (over := ratio > tlimit and lv - sv > TIME_LENGTH_FLOOR_MS),
+ "fail": LENGTH_TIME_BLOCKING and over,
+ }
+ )
limit = limits.get(("length", area), LENGTH_RATIO)
cap = limits.get(("count", area))
if area in CPU_LENGTH:
@@ -1880,6 +1977,29 @@ def thread_split(threads: dict[str, float]) -> str:
return ", ".join(f"{n or '?'} {v:.0f} ms" for n, v in top)
+def recheck_keys(verdicts: list[dict]) -> list[tuple[int, str, str]]:
+ """(turns, mode, area) to measure again: every failing CPU cell and broken area, and a blocking paint
+ / main-thread length failure at both lengths."""
+ keys = {
+ (v["turns"], v["mode"], v["area"])
+ for v in verdicts
+ if v["fail"]
+ and (
+ v["kind"] in ("appearance", "length")
+ and v["metric"] == "cpu_ms"
+ or v["kind"] == "broken"
+ and v["area"] != "*"
+ )
+ }
+ keys |= {
+ (t, v["mode"], v["area"])
+ for v in verdicts
+ if v["fail"] and v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms")
+ for t in (SHORT, LONG)
+ }
+ return sorted(keys)
+
+
def run(
dist: Path,
launch_args,
@@ -1890,22 +2010,9 @@ def run(
result = measure(dist, launch_args, log = log)
verdicts = judge(result)
# One reading per cell, so every CPU failure is measured again (fresh pages of the mode and its
- # base, RECHECKS times, the cheapest reading kept). Grouped by page; at most 4 mode pages.
- again = sorted(
- {
- (v["turns"], v["mode"], v["area"])
- for v in verdicts
- if v["fail"]
- and (
- v["kind"] in ("appearance", "length")
- and v["metric"] == "cpu_ms"
- or v["kind"] == "broken"
- and v["area"] != "*"
- )
- }
- )
- # Pages with the fewest failing cells first (a lone outlier is the usual noise); a mode that
- # fails many cells is one page anyway.
+ # base, RECHECKS times, the cheapest reading kept). Grouped by page; at most RECHECK_PAGES pages,
+ # those with the fewest failing cells first (a lone outlier is the usual noise).
+ again = recheck_keys(verdicts)
per_page: dict = {}
for t, m, _ in again:
per_page[(t, m)] = per_page.get((t, m), 0) + 1
@@ -1914,6 +2021,13 @@ def run(
if again and RECHECKS:
_in_thread(remeasure, result, again, log)
verdicts = judge(result)
+ # A recheck that lowers one side of a comparison (the short chat, or the base appearance) can
+ # make a failure appear only now, judged on a reading that was never measured again: those
+ # cells get the same fresh-page rechecks once.
+ late = sorted(set(recheck_keys(verdicts)) - set(again))
+ if late:
+ _in_thread(remeasure, result, late, log)
+ verdicts = judge(result)
failures, details = [], []
trig = (result["pages"].get((SHORT, "default")) or {}).get("triggers")
if trig is None:
@@ -1931,6 +2045,16 @@ def run(
continue
if v["kind"] == "broken":
failures.append(f"area {v['area']} ({v['mode']}, {v['turns']} turns): {v['why']}")
+ elif v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms"):
+ what = "paint" if v["metric"] == "paint_ms" else "main-thread CPU"
+ key = "paint" if v["metric"] == "paint_ms" else "cpu"
+ failures.append(
+ f"area {v['area']} ({v['mode']}) costs more as the chat grows: {what} {v['base_value']:,.0f} ms at "
+ f"{SHORT} turns -> {v['value']:,.0f} ms at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x over "
+ f"{TIME_LENGTH_FLOOR_MS:.0f} ms; LONG_THREAD_AREAS_KNOWN {key}:{v['area']}=... or {key}@{v['mode']}:{v['area']}=...). "
+ "Element and layout counts can stay flat while this grows: the thread behind the surface is repainted, "
+ "or script walks every message"
+ )
elif v["kind"] == "length":
bound = (
f"known ceiling {v['cap']:,.0f} elements, LONG_THREAD_AREAS_KNOWN count:{v['area']}=..."
@@ -2019,4 +2143,30 @@ def table(result: dict, verdicts: list[dict]) -> list[str]:
+ ("!" if lv and lv["fail"] else "")
)
lines.append(row)
+ timed = sorted(
+ (v for v in verdicts if v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms")),
+ key = lambda v: -(v["ratio"] / v["limit"]),
+ )
+ if timed:
+ lines.append(
+ f" paint / CPU growth {SHORT} -> {LONG} turns, highest against limit (! = over): "
+ + ", ".join(
+ f"{v['mode']}/{v['area']} {v['metric'][:-3]} {v['base_value']:.0f}->{v['value']:.0f} ms "
+ f"{v['ratio']:.1f}x/{v['limit']:.1f}{'!' if v.get('would_fail') else ''}"
+ for v in timed[:6]
+ )
+ )
+ would = [v for v in timed if v.get("would_fail")]
+ mode_note = "blocking" if LENGTH_TIME_BLOCKING else "report-only, not failing the gate"
+ lines.append(
+ f" paint / CPU growth would fail ({mode_note}): "
+ + (
+ ", ".join(
+ f"{v['mode']}/{v['area']} {v['metric'][:-3]} {v['base_value']:.0f}->{v['value']:.0f} ms "
+ f"{v['ratio']:.1f}x/{v['limit']:.1f}"
+ for v in would
+ )
+ or "none"
+ )
+ )
return lines
diff --git a/tests/studio/_webkit_leg.py b/tests/studio/_webkit_leg.py
new file mode 100644
index 00000000000..af71ffce84c
--- /dev/null
+++ b/tests/studio/_webkit_leg.py
@@ -0,0 +1,538 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""WebKit leg of the long-chat gate: open/close time and longest frame per surface, short vs long chat.
+
+The Chromium area matrix (_area_matrix.py) counts restyled elements and CPU from a trace, which WebKit
+cannot give. Here each surface is timed in the page itself: from just before the trigger to the first
+frame after the surface is visible (and back for closing), with the longest rAF frame inside that
+window. Playwright's WebKit runs twice: with a macOS Safari user agent and platform, so the app takes
+the macOS paths (the WebKitGTK long-thread windowing of #16854 stays off, as on a Mac), and with its
+own Linux user agent (the Linux desktop app's WebKitGTK paths). Same seeded chat as the matrix, served
+from the same production build with the API answered in-process.
+
+Per (engine, appearance, surface): the median of REPS runs on one page at SHORT and at LONG turns.
+Fails when the long chat is over RATIO x the short one AND over FLOOR_MS, or when its longest frame is
+over FRAME_MS AND over FRAME_GROWTH x the short chat's (WebKit on a CI runner already draws 300-600 ms
+frames mounting Settings at 1 turn: a mount cost, not a long-chat one); a failing cell is measured again
+on fresh pages and the lower reading kept. The two engines run in parallel, one browser each. Today's
+known WebKit costs are pinned in KNOWN (measured on main, with margin) so the gate holds that level
+and fails on anything worse; each names the fix work, and tightening is a one-line change.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import statistics
+import threading
+import time
+from pathlib import Path
+from urllib.parse import urlparse
+
+import _area_matrix as am
+
+SHORT = int(os.environ.get("LONG_THREAD_WEBKIT_SHORT", "1"))
+LONG = int(os.environ.get("LONG_THREAD_WEBKIT_LONG", "150"))
+REPS = int(os.environ.get("LONG_THREAD_WEBKIT_REPS", "3"))
+RATIO = float(os.environ.get("LONG_THREAD_WEBKIT_RATIO", "2.5"))
+FLOOR_MS = float(os.environ.get("LONG_THREAD_WEBKIT_FLOOR_MS", "200"))
+FRAME_MS = float(os.environ.get("LONG_THREAD_WEBKIT_FRAME_MS", "250"))
+FRAME_GROWTH = float(os.environ.get("LONG_THREAD_WEBKIT_FRAME_GROWTH", "2.5"))
+# A short-chat time below this counts as this, so a near-instant surface cannot make a large ratio.
+SHORT_FLOOR_MS = 100.0
+TIMEOUT_MS = 8_000
+SETTLE_MS = 150
+
+MAC_UA = (
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) "
+ "Version/18.0 Safari/605.1.15"
+)
+ENGINES = {
+ "webkit_mac": {"user_agent": MAC_UA, "platform": "MacIntel"},
+ "webkit_linux": {},
+}
+MODES = ("default", am.GLASS)
+# The surfaces Round 11 measured on real runners (plans/xplat_lag_status3.md), as _area_matrix AREAS.
+SURFACES = [
+ "browser_pane",
+ "settings",
+ "shortcuts_dialog",
+ "command_palette",
+ "composer_menu",
+ "sidebar_toggle",
+ "thread_scroll",
+]
+
+# Known WebKit long-chat costs on main, pinned so the gate holds today's level. Keys: engine:mode:surface:
+# metric (open / close = long/short ratio of the time; frame = the long chat's longest frame, ms).
+# Set from main readings with margin; lower each when its fix lands.
+KNOWN_DEFAULT = {
+ # Browser pane: opening or closing it narrows / widens the thread and WebKit re-lays out and repaints
+ # the rows still inside the #16986 content-visibility window (macos-15 Safari engine, Round 11:
+ # 460-562 ms at 150 turns vs 144-165 at 1). GitHub ubuntu runners, 6 readings: macOS UA open 1.5-3.0x
+ # (frame 267-335 ms), close 3.6-7.2x / 3.6x over the 100 ms floor (frame 235-300); Linux UA open up
+ # to 3.9x (Glass 6.4x, frame 327-453), close 2.9x (Glass 4.8x, frame 191-297). Fix work: branch
+ # studio-browser-pane-open.
+ "webkit_mac:default:browser_pane:open": 4.0,
+ "webkit_mac:default:browser_pane:close": 5.0,
+ "webkit_mac:default:browser_pane:open_frame": 450,
+ "webkit_mac:default:browser_pane:close_frame": 400,
+ "webkit_mac:liquid-3d:browser_pane:close": 7.0,
+ "webkit_mac:liquid-3d:browser_pane:close_frame": 520,
+ "webkit_linux:default:browser_pane:open": 5.0,
+ "webkit_linux:default:browser_pane:close": 4.0,
+ "webkit_linux:liquid-3d:browser_pane:open": 8.0,
+ "webkit_linux:liquid-3d:browser_pane:close": 6.5,
+ "webkit_linux:liquid-3d:browser_pane:open_frame": 600,
+ "webkit_linux:liquid-3d:browser_pane:close_frame": 400,
+ # Thread scroll: 20 scroll steps take 3.0-3.3x as long at 150 turns (macOS UA solid; Linux UA with
+ # Glass, the WebKitGTK Glass scroll of Round 11: 12-38 of 67-132 frames over 50 ms on Linux).
+ # Fix work: Linux WebKitGTK Glass scroll (GOAL_DETAILS open todo); macOS thread scroll paint.
+ "webkit_mac:default:thread_scroll:open": 4.5,
+ "webkit_linux:liquid-3d:thread_scroll:open": 4.5,
+ # Composer menu and command palette (macOS UA, solid): a 237-297 / 182-268 ms frame opening them at
+ # 150 turns: the paint behind the overlay grows with the thread (Round 11's Chromium probe: composer
+ # menu paint 10 -> 80 ms, Settings 15 -> 104). Fix work: Settings / overlay paint (GOAL_DETAILS todo).
+ "webkit_mac:default:composer_menu:open_frame": 400,
+ "webkit_mac:default:command_palette:open_frame": 350,
+}
+KNOWN = {
+ **KNOWN_DEFAULT,
+ **{
+ k: float(v)
+ for k, _, v in (
+ p.strip().partition("=")
+ for p in os.environ.get("LONG_THREAD_WEBKIT_KNOWN", "").split(",")
+ if p.strip()
+ )
+ },
+}
+
+ARM = """() => {
+ const w = window.__wk = {t0: performance.now(), frames: [], last: performance.now(), stop: false};
+ const tick = (t) => { w.frames.push([t, t - w.last]); w.last = t; if (!w.stop) requestAnimationFrame(tick); };
+ requestAnimationFrame(tick);
+}"""
+# Wait (in rAF steps) until `pred` holds, then one more frame: the time is that frame's, so it
+# includes painting the surface. Returns [ms since ARM, longest frame in the window] or null.
+UNTIL = """async ([pred, timeoutMs]) => {
+ const w = window.__wk; const test = (0, eval)(pred);
+ const frame = () => new Promise((r) => requestAnimationFrame(r));
+ while (!test()) { if (performance.now() - w.t0 > timeoutMs) { w.stop = true; return null; } await frame(); }
+ const t = await frame(); w.stop = true;
+ const long = w.frames.filter(([ts]) => ts <= t + 1).reduce((m, [, d]) => Math.max(m, d), 0);
+ return [t - w.t0, long];
+}"""
+# Thread scroll: the matrix's 20-step scroll; the time is the whole scroll, the frame its longest.
+SCROLLED = """() => { const w = window.__wk; w.stop = true; const t = performance.now();
+ return [t - w.t0, w.frames.reduce((m, [, d]) => Math.max(m, d), 0)]; }"""
+
+
+def pred_js(pred, base_overlays: int, attr_before) -> str:
+ kind = pred[0]
+ if kind == "overlay":
+ return f"() => window.__gateOverlays() > {base_overlays}"
+ if kind == "overlay_gone":
+ return f"() => window.__gateOverlays() <= {max(0, base_overlays - 1)} || window.__gateOverlays() === 0"
+ if kind == "sel":
+ return f"() => window.__gateVisible({json.dumps(pred[1])})"
+ if kind == "gone":
+ return f"() => !window.__gateVisible({json.dumps(pred[1])})"
+ if kind == "changed":
+ return (
+ f"() => {{ const v = window.__gateAttr({json.dumps(pred[1])}, {json.dumps(pred[2])}); "
+ f"return v !== null && v !== {json.dumps(attr_before)}; }}"
+ )
+ if kind == "restored":
+ return f"() => window.__gateAttr({json.dumps(pred[1])}, {json.dumps(pred[2])}) === {json.dumps(attr_before)}"
+ raise ValueError(f"no timing predicate for {kind}")
+
+
+class WebKitPage(am.Page):
+ """The matrix's page (same thread, API and controls) in WebKit, without Chromium tracing."""
+
+ def __init__(self, browser, base: str, turns: int, mode: dict, engine: str):
+ self.browser, self.turns, self.mode, self.tabs = browser, turns, mode, []
+ self.api = am.Api(turns)
+ extra = (
+ {"user_agent": ENGINES[engine]["user_agent"]} if "user_agent" in ENGINES[engine] else {}
+ )
+ self.ctx = browser.new_context(viewport = {"width": 1440, "height": 900}, **extra)
+ if ENGINES[engine].get("platform"):
+ self.ctx.add_init_script(
+ "Object.defineProperty(Navigator.prototype, 'platform', {get: () => %s});"
+ % json.dumps(ENGINES[engine]["platform"])
+ )
+ init = {
+ "unsloth_auth_token": am.auth_token(),
+ "unsloth_auth_refresh_token": "r",
+ "unsloth_window_material": am.material_storage(mode["set"]),
+ "theme": mode["set"]["theme"],
+ }
+ self.ctx.add_init_script(
+ "(() => { const v = %s; for (const k in v) localStorage.setItem(k, v[k]); })()"
+ % json.dumps(init)
+ )
+ self.ctx.route(lambda url: urlparse(url).path.startswith("/api/"), self.api.handle)
+ self.ctx.route(
+ lambda url: not url.startswith(base), lambda route: route.fulfill(status = 204, body = "")
+ )
+ self.page = page = self.ctx.new_page()
+ self.errors, self.last_error = [], ""
+ self.attr_before, self.focusins_before, self.hovered = None, 0, False
+ page.goto(f"{base}/chat?thread={self.api.rec['id']}", wait_until = "load", timeout = 180_000)
+ page.wait_for_function(
+ "(n) => document.querySelectorAll('[data-role]').length >= n",
+ arg = 2 * turns,
+ timeout = 180_000,
+ )
+ page.evaluate(am.PROBE)
+ page.wait_for_timeout(1_500)
+ self.mac = page.evaluate(
+ "() => /mac|iphone|ipad|ipod/i.test(`${navigator.platform || ''} ${navigator.userAgent || ''}`)"
+ )
+ page.mouse.move(1435, 450)
+
+ def timed(self, action, pred, base_overlays: int):
+ self.page.evaluate(ARM)
+ if not self.act(action):
+ self.page.evaluate("() => { window.__wk.stop = true; }")
+ return None
+ if pred is None:
+ return self.page.evaluate(SCROLLED)
+ return self.page.evaluate(
+ UNTIL, [pred_js(pred, base_overlays, self.attr_before), TIMEOUT_MS]
+ )
+
+ def surface(self, name: str) -> dict:
+ """One open (and close) of a surface: {open_ms, open_frame, close_ms, close_frame} or {error}."""
+ spec = am.AREAS[name]
+ page = self.page
+ self.reset()
+ page.wait_for_timeout(SETTLE_MS)
+ base = page.evaluate("() => window.__gateOverlays()")
+ ready = spec["ready"]
+ self.attr_before = (
+ page.evaluate("([s, a]) => window.__gateAttr(s, a)", [ready[1], ready[2]])
+ if ready[0] == "changed"
+ else None
+ )
+ scroll = spec["open"][0] == "scroll"
+ got = self.timed(spec["open"], None if scroll else ready, base)
+ if got is None:
+ return {"error": f"never reached {ready[0]}"}
+ out = {"open_ms": got[0], "open_frame": got[1]}
+ if spec.get("close"):
+ page.wait_for_timeout(SETTLE_MS)
+ gone = self.timed(spec["close"], spec["gone"], base + 1)
+ if gone is None:
+ return {**out, "error": f"opened but never reached {spec['gone'][0]}"}
+ out.update(close_ms = gone[0], close_frame = gone[1])
+ if spec.get("teardown"):
+ self.act(spec["teardown"])
+ return out
+
+ def measure(self, surfaces: list[str], reps: int) -> dict[str, dict]:
+ cells: dict[str, dict] = {}
+ for name in surfaces:
+ runs, error = [], None
+ for _ in range(reps):
+ try:
+ r = self.surface(name)
+ except Exception as exc: # noqa: BLE001 - reported as a NOT_RUN surface
+ r = {"error": f"{type(exc).__name__}: {str(exc)[:120]}"}
+ if "error" in r:
+ error = r["error"]
+ continue
+ runs.append(r)
+ if not runs:
+ cells[name] = {"status": f"NOT_RUN: {error}"}
+ continue
+ cell = {"status": "OK", "reps": len(runs)}
+ for key in ("open_ms", "open_frame", "close_ms", "close_frame"):
+ vals = [r[key] for r in runs if key in r]
+ if vals:
+ cell[key] = round(statistics.median(vals), 1)
+ cells[name] = cell
+ return cells
+
+
+def _engine_worker(engine, dist_base, modes, surfaces, reps, out, log) -> None:
+ from playwright.sync_api import sync_playwright
+ with sync_playwright() as pw:
+ browser = pw.webkit.launch()
+ try:
+ for mode in modes:
+ for turns in (SHORT, LONG):
+ t = time.time()
+ try:
+ page = WebKitPage(browser, dist_base, turns, mode, engine)
+ except Exception as exc: # noqa: BLE001
+ out[(engine, mode["name"], turns)] = {
+ "error": f"did not load: {str(exc)[:160]}"
+ }
+ continue
+ try:
+ out[(engine, mode["name"], turns)] = {"cells": page.measure(surfaces, reps)}
+ finally:
+ page.close()
+ log(
+ f" webkit: {engine} {mode['name']} {turns} turns ({time.time() - t:.0f} s)",
+ flush = True,
+ )
+ finally:
+ browser.close()
+
+
+def measure(
+ dist: Path,
+ surfaces = None,
+ modes = MODES,
+ engines = None,
+ reps = REPS,
+ log = print,
+) -> dict:
+ surfaces = surfaces or SURFACES
+ by_name = {m["name"]: m for m in am.app_modes()}
+ chosen = [by_name[m] for m in modes]
+ server, base = am.serve_app(dist)
+ out: dict = {}
+ try:
+ # One browser per engine (a browser per appearance as well doubles the frames on a 4-core runner).
+ crashed: dict = {}
+
+ def work(engine):
+ try:
+ _engine_worker(engine, base, chosen, surfaces, reps, out, log)
+ except Exception as exc: # noqa: BLE001 (recorded below: a crash must never read as a pass)
+ crashed[engine] = f"{type(exc).__name__}: {str(exc)[:160]}"
+
+ engines = engines or ENGINES
+ threads = [threading.Thread(target = work, args = (e,)) for e in engines]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ finally:
+ server.shutdown()
+ # Every planned page gets a result: one a worker never recorded is a load failure.
+ for e in engines:
+ for m in chosen:
+ for turns in (SHORT, LONG):
+ out.setdefault(
+ (e, m["name"], turns),
+ {
+ "error": f"not measured ({crashed.get(e, 'the engine worker stopped early')})"
+ },
+ )
+ return out
+
+
+def judge(result: dict, known: dict | None = None) -> list[dict]:
+ known = KNOWN if known is None else known
+ out = []
+ for (engine, mode, turns), page in sorted(result.items()):
+ if "error" in page:
+ out.append(
+ {
+ "engine": engine,
+ "mode": mode,
+ "surface": "*",
+ "metric": "load",
+ "fail": True,
+ "why": page["error"],
+ }
+ )
+ for (engine, mode, turns), page in sorted(result.items()):
+ if turns != LONG or "cells" not in page:
+ continue
+ short_page = result.get((engine, mode, SHORT)) or {}
+ if "cells" not in short_page:
+ continue # its load failure is reported above
+ short = short_page["cells"]
+ for name in sorted(set(page["cells"]) | set(short)):
+ lc, sc = page["cells"].get(name, {}), short.get(name, {})
+ if lc.get("status") != "OK" or sc.get("status") != "OK":
+ # A surface missing at either length is a gap in the gate, never a silent pass.
+ out.append(
+ {
+ "engine": engine,
+ "mode": mode,
+ "surface": name,
+ "metric": "run",
+ "fail": True,
+ "why": f"{lc.get('status', 'missing')} at {LONG} turns, {sc.get('status', 'missing')} at {SHORT}",
+ }
+ )
+ continue
+ for edge in ("open", "close"):
+ lv, sv = lc.get(f"{edge}_ms"), sc.get(f"{edge}_ms")
+ if lv is None or sv is None:
+ continue
+ ratio = lv / max(sv, SHORT_FLOOR_MS)
+ limit = known.get(f"{engine}:{mode}:{name}:{edge}", RATIO)
+ out.append(
+ {
+ "engine": engine,
+ "mode": mode,
+ "surface": name,
+ "metric": edge,
+ "value": lv,
+ "base_value": sv,
+ "ratio": ratio,
+ "limit": limit,
+ "fail": ratio > limit and lv > FLOOR_MS,
+ }
+ )
+ fv = lc.get(f"{edge}_frame")
+ if fv is not None:
+ flimit = known.get(f"{engine}:{mode}:{name}:{edge}_frame", FRAME_MS)
+ sf = sc.get(f"{edge}_frame") or 0.0
+ out.append(
+ {
+ "engine": engine,
+ "mode": mode,
+ "surface": name,
+ "metric": f"{edge}_frame",
+ "value": fv,
+ "base_value": sf,
+ "limit": flimit,
+ "fail": fv > flimit and fv > FRAME_GROWTH * sf,
+ }
+ )
+ return out
+
+
+def lower(first: dict, again: dict) -> dict:
+ """Merge a recheck: per cell, the lower of each reading (a recheck only ever lowers a reading)."""
+ merged = dict(first)
+ for key, page in again.items():
+ if "cells" not in page:
+ continue
+ if "cells" not in merged.get(key, {}):
+ merged[key] = page
+ continue
+ cells = dict(merged[key]["cells"])
+ for name, cell in page["cells"].items():
+ old = cells.get(name, {})
+ if cell.get("status") != "OK":
+ continue
+ if old.get("status") != "OK":
+ cells[name] = cell
+ continue
+ cells[name] = {
+ **old,
+ **{
+ k: min(old[k], v)
+ for k, v in cell.items()
+ if k.endswith(("_ms", "_frame")) and k in old
+ },
+ }
+ merged[key] = {"cells": cells}
+ return merged
+
+
+def run(dist: Path, log = print) -> tuple[list[str], list[str], dict]:
+ """(report lines, failures, result)."""
+ started = time.time()
+ result = measure(dist, log = log)
+ verdicts = judge(result)
+ failing = {
+ (v["engine"], v["mode"], v["surface"])
+ for v in verdicts
+ if v["fail"] and v["surface"] != "*"
+ }
+ if failing:
+ log(
+ f" webkit: {len(failing)} cell(s) over their limit; measuring them again on fresh pages",
+ flush = True,
+ )
+ for engine in sorted({f[0] for f in failing}):
+ mode_names = sorted({f[1] for f in failing if f[0] == engine})
+ surfaces = sorted({f[2] for f in failing if f[0] == engine})
+ again = measure(dist, surfaces = surfaces, modes = mode_names, engines = [engine], log = log)
+ result = lower(result, again)
+ verdicts = judge(result)
+ lines = table(result, verdicts)
+ lines.append(
+ f" webkit leg: {len(ENGINES)} engines x {len(MODES)} appearances x {SHORT}/{LONG} turns, {time.time() - started:.0f} s"
+ )
+ failures = []
+ for v in verdicts:
+ if not v["fail"]:
+ continue
+ where = f"{v['engine']}, {v['mode']}"
+ if v["metric"] in ("load", "run"):
+ failures.append(f"WebKit {where}: {v['surface']} {v['why']}")
+ elif v["metric"].endswith("frame"):
+ failures.append(
+ f"WebKit {where}: {v['surface']} ({v['metric'].replace('_', ' ')}) has a {v['value']:.0f} ms frame at "
+ f"{LONG} turns against {v['base_value']:.0f} ms at {SHORT} (limit {v['limit']:.0f} ms and {FRAME_GROWTH:g}x; LONG_THREAD_WEBKIT_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...)"
+ )
+ else:
+ failures.append(
+ f"WebKit {where}: {'opening' if v['metric'] == 'open' else 'closing'} {v['surface']} takes {v['base_value']:.0f} ms at {SHORT} turns -> "
+ f"{v['value']:.0f} ms at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x over {FLOOR_MS:.0f} ms; "
+ f"LONG_THREAD_WEBKIT_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...). It costs more "
+ "as the chat grows in WebKit (Safari, the macOS app, the Linux desktop app): usually re-layout or "
+ "repaint of the whole thread behind the surface"
+ )
+ return lines, failures, {"result": result, "verdicts": verdicts}
+
+
+def table(result: dict, verdicts: list[dict]) -> list[str]:
+ by = {(v["engine"], v["mode"], v["surface"], v["metric"]): v for v in verdicts}
+ lines = [
+ f" WebKit: open / close ms at {SHORT} -> {LONG} turns, longest frame at {LONG} (! = over its limit)"
+ ]
+ for (engine, mode, turns), page in sorted(result.items()):
+ if turns != LONG:
+ continue
+ if "error" in page:
+ lines.append(f" {engine:<13}{mode:<10} {page['error']}")
+ continue
+ short = (result.get((engine, mode, SHORT)) or {}).get("cells", {})
+ for name, lc in page["cells"].items():
+ sc = short.get(name, {})
+ if lc.get("status") != "OK":
+ lines.append(f" {engine:<13}{mode:<10}{name:<18}{lc.get('status')}")
+ continue
+ parts = []
+ for edge in ("open", "close"):
+ if f"{edge}_ms" not in lc:
+ continue
+ v = by.get((engine, mode, name, edge))
+ f = by.get((engine, mode, name, f"{edge}_frame"))
+ parts.append(
+ f"{edge} {sc.get(f'{edge}_ms', float('nan')):>5.0f} -> {lc[f'{edge}_ms']:>5.0f}"
+ + (f" ({v['ratio']:.1f}x{'!' if v['fail'] else ''})" if v else "")
+ + f" frame {lc.get(f'{edge}_frame', 0):>4.0f}{'!' if f and f['fail'] else ''}"
+ )
+ lines.append(f" {engine:<13}{mode:<10}{name:<18}" + " ".join(parts))
+ return lines
+
+
+def main() -> int:
+ """Frontend CI's WebKit job: the leg alone, on the long-thread production build."""
+ if os.environ.get("LONG_THREAD_WEBKIT", "0") != "1":
+ print("WebKit leg: off (set LONG_THREAD_WEBKIT=1)")
+ return 0
+ repo = Path(__file__).resolve().parents[2]
+ dist = Path(
+ os.environ.get("LONG_THREAD_DIST", repo / ".playwright-cli" / "long-thread-production")
+ )
+ if not (dist / "index.html").exists():
+ print(f"FAIL: no app build at {dist / 'index.html'} for the WebKit leg")
+ return 1
+ lines, fails, _ = run(dist)
+ for line in lines:
+ print(line)
+ for f in fails:
+ print(f"FAIL: {f}")
+ print("WebKit leg: " + (f"{len(fails)} failure(s)" if fails else "PASS"))
+ return 1 if fails else 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/studio/playwright_long_thread_gate.py b/tests/studio/playwright_long_thread_gate.py
index 0f16bd048c0..0e144203994 100644
--- a/tests/studio/playwright_long_thread_gate.py
+++ b/tests/studio/playwright_long_thread_gate.py
@@ -51,6 +51,7 @@ from playwright.sync_api import sync_playwright
sys.path.insert(0, str(Path(__file__).resolve().parent))
import _area_matrix as areas # noqa: E402
import _has_lint as has_lint # noqa: E402
+import _webkit_leg as webkit_leg # noqa: E402
from _css_culprit import explain, has_descendant, thread_anchor # noqa: E402
from _playwright_robust import chromium_launch_args # noqa: E402
@@ -80,6 +81,10 @@ AREAS = os.environ.get("LONG_THREAD_AREAS", "auto")
ONLY_AREAS = os.environ.get("LONG_THREAD_ONLY_AREAS") == "1"
# Static lint of the built CSS for new non-subject :has() rules (tests/studio/_has_lint.py).
HAS_LINT = os.environ.get("LONG_THREAD_HAS_LINT", "1") != "0"
+# The WebKit leg (_webkit_leg.py): surface open/close time and longest frame, short vs long chat, in
+# Playwright WebKit with a macOS and a Linux user agent. Opt-in (one Linux CI lane sets it), as it
+# needs the WebKit browser installed.
+WEBKIT = os.environ.get("LONG_THREAD_WEBKIT", "0") == "1"
HEALTHY = "healthy reference: 0.7-1.0 ms/key selector matching, 1.00x, 0.2 ms/key :has()"
KEYS = "the quick "
PRECHECK_KEYS = "ab"
@@ -726,6 +731,19 @@ def finish_areas(runs, failures, details, started) -> int:
default = str,
)
)
+ if WEBKIT:
+ if not (DIST / "index.html").exists():
+ failures.append(f"no app build at {DIST / 'index.html'} for the WebKit leg")
+ else:
+ try:
+ lines, fails, _ = webkit_leg.run(DIST)
+ except Exception as exc: # noqa: BLE001 - a leg that cannot start is a failure, not a pass
+ lines, fails = (
+ [],
+ [f"the WebKit leg did not run: {type(exc).__name__}: {str(exc)[:300]}"],
+ )
+ extra += lines
+ failures += fails
return report(runs, failures, details, started, extra)
diff --git a/tests/studio/test_area_matrix.py b/tests/studio/test_area_matrix.py
index 2f79f9e081c..8adfaf51c8b 100644
--- a/tests/studio/test_area_matrix.py
+++ b/tests/studio/test_area_matrix.py
@@ -83,6 +83,12 @@ def test_coverage_matcher():
assert am.uncovered(triggers) == ["dropdown-menu-trigger:Brand new menu"]
+def length_blocking(monkeypatch):
+ """The paint / main-thread length rule as it judges once LENGTH_TIME_BLOCKING is switched on."""
+ monkeypatch.setattr(am, "LENGTH_TIME_BLOCKING", True)
+
+
def _page(cells):
return {"cells": {a: {"status": "OK", **c} for a, c in cells.items()}}
@@ -336,6 +342,100 @@ def test_missing_default_page_fails_coverage(monkeypatch):
assert sum("measuring the page failed" in f for f in failures) == len(out)
+def test_a_length_failure_that_appears_after_a_recheck_is_measured_again(
+ monkeypatch, length_blocking
+):
+ # CI (message_menu, Liquid 3D): a 506 ms first short reading failed the appearance check, so only
+ # the short page was measured again (99, 90 ms). That made the length check fail for the first
+ # time, on a long reading (354 ms) nobody had measured again.
+ cell = lambda cpu, main: {
+ "elements": 10,
+ "layout_objects": 0,
+ "cpu_ms": cpu,
+ "main_ms": main,
+ "paint_ms": 5,
+ }
+ pages = {
+ (am.SHORT, "default"): _page({"type": cell(100, 90)}),
+ (am.SHORT, "frosted"): _page({"type": cell(2000, 506)}),
+ (am.LONG, "default"): _page({"type": cell(110, 100)}),
+ (am.LONG, "frosted"): _page({"type": cell(400, 354)}),
+ }
+ fake = {
+ "pages": pages,
+ "modes": {m["name"]: m for m in am.app_modes()},
+ "areas": ["type"],
+ "seconds": 0.0,
+ }
+ monkeypatch.setattr(am, "measure", lambda *a, **k: fake)
+ calls = []
+
+ def remeasure(result, items, log):
+ calls.append(sorted(items))
+ for turns, mode, area in items:
+ reading = {"status": "OK", "elements": 10, "layout_objects": 0, "paint_ms": 5}
+ reading.update(
+ {"cpu_ms": 200, "main_ms": 95}
+ if turns == am.SHORT
+ else {"cpu_ms": 220, "main_ms": 120}
+ )
+ result["pages"][(turns, mode)]["cells"][area].setdefault("again", []).extend(
+ [reading] * am.RECHECKS
+ )
+
+ monkeypatch.setattr(am, "remeasure", remeasure)
+ _, failures, _, _ = am.run(Path("unused"), [], log = lambda *a, **k: None)
+ assert calls[0] == [(am.SHORT, "frosted", "type")]
+ assert calls[1] == [(am.LONG, "frosted", "type")]
+ assert not [f for f in failures if "costs more as the chat grows" in f]
+
+
+def test_an_appearance_failure_that_appears_after_a_recheck_is_measured_again(monkeypatch):
+ # CI, main, Liquid 3D typing at 200 turns: one sweep of the 200-turn pages caught a background-
+ # thread burst in every look (default 1,266 ms CPU, Liquid 3D 1,005). Another mode's failure got
+ # the default page measured again (154 ms); Liquid 3D was then judged 1,005 / 154 = 6.5x against
+ # 6x on its never-rechecked first reading. Its own recheck reads 1.6x.
+ cell = lambda cpu: {
+ "elements": 70,
+ "layout_objects": 200,
+ "cpu_ms": cpu,
+ "main_ms": cpu / 2,
+ "paint_ms": 15,
+ }
+ pages = {
+ (am.SHORT, "default"): _page({"type": cell(150)}),
+ (am.SHORT, "frosted"): _page({"type": cell(160)}),
+ (am.SHORT, am.GLASS): _page({"type": cell(230)}),
+ (am.LONG, "default"): _page({"type": cell(1266)}),
+ (am.LONG, "frosted"): _page({"type": cell(9000)}),
+ (am.LONG, am.GLASS): _page({"type": cell(1005)}),
+ }
+ fake = {
+ "pages": pages,
+ "modes": {m["name"]: m for m in am.app_modes()},
+ "areas": ["type"],
+ "seconds": 0.0,
+ }
+ monkeypatch.setattr(am, "measure", lambda *a, **k: fake)
+ calm = {"default": 154, "frosted": 300, am.GLASS: 250}
+ calls = []
+
+ def remeasure(result, items, log):
+ calls.append(sorted(items))
+ for turns, mode, area in items:
+ for name in ("default", mode):
+ reading = {"status": "OK", **cell(calm[name])}
+ result["pages"][(turns, name)]["cells"][area].setdefault("again", []).extend(
+ [reading] * am.RECHECKS
+ )
+
+ monkeypatch.setattr(am, "remeasure", remeasure)
+ _, failures, _, _ = am.run(Path("unused"), [], log = lambda *a, **k: None)
+ assert calls[0] == [(am.LONG, "frosted", "type")]
+ assert calls[1] == [(am.LONG, am.GLASS, "type")]
+ assert not [f for f in failures if "makes area type cost" in f]
+
+
def test_every_area_opened_by_a_shortcut_or_toggle_proves_it_happened():
# A key press always "succeeds"; only a visible change shows the app acted on it.
for name, spec in am.AREAS.items():
@@ -467,6 +567,147 @@ def test_settings_ready_marker_is_set_from_inside_the_panel_suspense():
assert "data-settings-panel-ready=" in am.TABS and "data-settings-panel-failed" in am.TABS
+ "short, long_, fails",
+ [
+ (15, 104, True), # Round 11's Settings paint behind the overlay at 1 vs 150 turns
+ (100, 240, False), # under 2.5x
+ (10, 55, False), # over 2.5x, but under 50 ms more
+ (40, 140, True),
+ ],
+)
+def test_paint_length_check(short, long_, fails, length_blocking):
+ pages = {
+ (am.SHORT, "default"): _page(
+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 10, "paint_ms": short}}
+ ),
+ (am.LONG, "default"): _page(
+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 10, "paint_ms": long_}}
+ ),
+ }
+ rows = [v for v in am.judge(_result(pages), known = "") if v["metric"] == "paint_ms"]
+ assert [v["fail"] for v in rows] == [fails]
+ if fails:
+ assert not [v for v in am.judge(_result(pages), known = "paint:a=8") if v["fail"]]
+ assert not [v for v in am.judge(_result(pages), known = "paint@default:a=8") if v["fail"]]
+
+
+def test_paint_length_check_keeps_the_cheapest_reading():
+ # A recheck that reads lower clears a noisy first reading (the minimum of the sweeps is judged).
+ long_cell = {
+ "elements": 10,
+ "layout_objects": 0,
+ "cpu_ms": 10,
+ "paint_ms": 300,
+ "again": [{"status": "OK", "paint_ms": 30, "cpu_ms": 10}],
+ }
+ pages = {
+ (am.SHORT, "default"): _page(
+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 10, "paint_ms": 20}}
+ ),
+ (am.LONG, "default"): _page({"a": long_cell}),
+ }
+ assert not [v for v in am.judge(_result(pages), known = "") if v["fail"]]
+
+
[email protected]("metric", ["paint_ms", "main_ms"])
+def test_length_ratio_divides_by_the_median_short_reading(metric, length_blocking):
+ # CI, reasoning_toggle (Liquid 3D): short readings 23 / 34 / 49 ms, long 76 ms. The cheapest short
+ # reading (23) made it 3.3x; over the median (34) it is 2.2x and passes.
+ short_cell = {
+ "elements": 10,
+ "layout_objects": 0,
+ metric: 23,
+ "again": [{"status": "OK", metric: 49}, {"status": "OK", metric: 34}],
+ }
+ long_cell = {
+ "elements": 10,
+ "layout_objects": 0,
+ metric: 90,
+ "again": [{"status": "OK", metric: 76}],
+ }
+ pages = {
+ (am.SHORT, "default"): _page({"a": short_cell}),
+ (am.LONG, "default"): _page({"a": long_cell}),
+ }
+ rows = [
+ v
+ for v in am.judge(_result(pages), known = "")
+ if v["metric"] == metric and v["kind"] == "length"
+ ]
+ assert [(v["value"], v["base_value"], v["fail"]) for v in rows] == [(76, 34, False)]
+ # A real growth still fails over the same median: 34 -> 160 ms is 4.7x and 126 ms more.
+ long_cell = {
+ "elements": 10,
+ "layout_objects": 0,
+ metric: 170,
+ "again": [{"status": "OK", metric: 160}],
+ }
+ pages[(am.LONG, "default")] = _page({"a": long_cell})
+ rows = [
+ v
+ for v in am.judge(_result(pages), known = "")
+ if v["metric"] == metric and v["kind"] == "length"
+ ]
+ assert [(v["value"], v["base_value"], v["fail"]) for v in rows] == [(160, 34, True)]
+
+
+def test_length_time_checks_report_but_do_not_fail_by_default():
+ # Report-only until message-level actions stop growing with the chat (LENGTH_TIME_BLOCKING).
+ assert am.LENGTH_TIME_BLOCKING is False
+ pages = {
+ (am.SHORT, "default"): _page(
+ {
+ "a": {
+ "elements": 10,
+ "layout_objects": 0,
+ "cpu_ms": 80,
+ "main_ms": 80,
+ "paint_ms": 15,
+ }
+ }
+ ),
+ (am.LONG, "default"): _page(
+ {
+ "a": {
+ "elements": 10,
+ "layout_objects": 0,
+ "cpu_ms": 90,
+ "main_ms": 400,
+ "paint_ms": 104,
+ }
+ }
+ ),
+ }
+ result = _result(pages)
+ verdicts = am.judge(result, known = "")
+ rows = [v for v in verdicts if v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms")]
+ assert [(v["metric"], v["would_fail"], v["fail"]) for v in rows] == [
+ ("paint_ms", True, False),
+ ("main_ms", True, False),
+ ]
+ assert not am.recheck_keys(verdicts), "a report-only reading costs no recheck time"
+ text = "\n".join(am.table(result, verdicts))
+ assert "would fail (report-only, not failing the gate): default/a paint 15->104 ms" in text
+
+
+def test_cpu_length_check(length_blocking):
+ pages = {
+ (am.SHORT, "default"): _page(
+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 80, "main_ms": 80}}
+ ),
+ (am.LONG, "default"): _page(
+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 400, "main_ms": 400}}
+ ),
+ }
+ rows = [
+ v
+ for v in am.judge(_result(pages), known = "")
+ if v["metric"] == "main_ms" and v["kind"] == "length"
+ ]
+ assert [v["fail"] for v in rows] == [True]
+
+
def _stream_result(pages):
return {"pages": pages, "modes": {m["name"]: m for m in am.app_modes()}, "areas": ["stream"]}
@@ -508,6 +749,14 @@ def test_stream_area_is_last_sends_a_fixed_reply_and_waits_for_its_end():
assert am.stream_frames() == frames, "deterministic"
+def test_the_stub_health_settles_the_hardware_verdict():
+ # An unsettled verdict leaves the Train row spinner running, which made every measured action pay
+ # an IntersectionObserver pass over the whole chat on each frame (env.ts capabilitiesUnknown).
+ health = am.Api(2).bodies["/api/health"]
+ assert health.get("version") and health.get("chat_only") is False
+ assert not health.get("hardware_detecting") and not health.get("hardware_detection_deferred")
+
+
def test_the_stub_has_a_resident_model_and_saves_the_streamed_turn():
api = am.Api(2)
assert api.bodies["/api/inference/status"]["active_model"]
diff --git a/tests/studio/test_webkit_leg.py b/tests/studio/test_webkit_leg.py
new file mode 100644
index 00000000000..d5cee99600e
--- /dev/null
+++ b/tests/studio/test_webkit_leg.py
@@ -0,0 +1,161 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The long-chat gate's WebKit leg: verdicts, known ceilings and rechecks (CI wiring: test_webkit_leg_ci.py)."""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import _area_matrix as am # noqa: E402
+import _webkit_leg as wk # noqa: E402
+
+REPO = Path(__file__).resolve().parents[2]
+
+
+def _cell(
+ open_ms,
+ close_ms = 50.0,
+ frame = 40.0,
+):
+ return {
+ "status": "OK",
+ "reps": 3,
+ "open_ms": open_ms,
+ "open_frame": frame,
+ "close_ms": close_ms,
+ "close_frame": frame,
+ }
+
+
+def _result(
+ short,
+ long_,
+ engine = "webkit_mac",
+ mode = "default",
+ surface = "browser_pane",
+):
+ return {
+ (engine, mode, wk.SHORT): {"cells": {surface: short}},
+ (engine, mode, wk.LONG): {"cells": {surface: long_}},
+ }
+
+
+ "short, long_, fails",
+ [
+ (150, 500, True), # Round 11's macOS WebKit pane: 144-165 -> 460-562 ms
+ (150, 180, False),
+ (60, 190, False), # over 2.5x but under 200 ms
+ (300, 700, False), # under 2.5x
+ ],
+)
+def test_open_time_ratio(short, long_, fails):
+ rows = [
+ v for v in wk.judge(_result(_cell(short), _cell(long_)), known = {}) if v["metric"] == "open"
+ ]
+ assert [v["fail"] for v in rows] == [fails]
+ if fails:
+ known = {"webkit_mac:default:browser_pane:open": 4.0}
+ assert not [
+ v for v in wk.judge(_result(_cell(short), _cell(long_)), known = known) if v["fail"]
+ ]
+
+
+def test_long_frame_fails_and_can_be_pinned():
+ res = _result(_cell(100, frame = 30), _cell(120, frame = 300))
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == [
+ "open_frame",
+ "close_frame",
+ ]
+ known = {
+ "webkit_mac:default:browser_pane:open_frame": 350,
+ "webkit_mac:default:browser_pane:close_frame": 350,
+ }
+ assert not [v for v in wk.judge(res, known = known) if v["fail"]]
+
+
+def test_a_surface_that_stops_running_in_the_long_chat_fails():
+ res = _result(_cell(100), {"status": "NOT_RUN: never reached sel"})
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["run"]
+
+
+def test_a_long_page_that_does_not_load_fails():
+ res = {("webkit_mac", "default", wk.LONG): {"error": "did not load"}}
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["load"]
+
+
+def test_a_short_page_that_does_not_load_fails():
+ res = {
+ ("webkit_mac", "default", wk.SHORT): {"error": "did not load"},
+ ("webkit_mac", "default", wk.LONG): {"cells": {"browser_pane": _cell(10_000)}},
+ }
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["load"]
+
+
+def test_a_surface_without_a_short_baseline_fails():
+ res = _result({"status": "NOT_RUN: never reached sel"}, _cell(10_000))
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["run"]
+
+
+def test_a_surface_that_runs_at_neither_length_fails():
+ res = _result({"status": "NOT_RUN: a"}, {"status": "NOT_RUN: b"})
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["run"]
+
+
+def test_a_crashed_engine_worker_fails_every_planned_page(monkeypatch):
+ def boom(*args, **kwargs):
+ raise RuntimeError("webkit did not launch")
+
+ class _Server:
+ def shutdown(self):
+ pass
+
+ monkeypatch.setattr(wk, "_engine_worker", boom)
+ monkeypatch.setattr(wk.am, "serve_app", lambda dist: (_Server(), "http://x"))
+ res = wk.measure(Path("."), modes = ["default"], engines = ["webkit_mac"], log = lambda *a, **k: None)
+ assert set(res) == {("webkit_mac", "default", wk.SHORT), ("webkit_mac", "default", wk.LONG)}
+ assert all("webkit did not launch" in page["error"] for page in res.values())
+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["load", "load"]
+
+
+def test_short_floor_keeps_instant_surfaces_from_inflating_the_ratio():
+ v = [v for v in wk.judge(_result(_cell(5), _cell(210)), known = {}) if v["metric"] == "open"][0]
+ assert v["ratio"] == pytest.approx(210 / wk.SHORT_FLOOR_MS)
+
+
+def test_a_recheck_only_lowers_readings():
+ first = _result(_cell(150), _cell(600, frame = 300))
+ again = _result(_cell(140), _cell(400, frame = 320))
+ merged = wk.lower(first, again)
+ cell = merged[("webkit_mac", "default", wk.LONG)]["cells"]["browser_pane"]
+ assert cell["open_ms"] == 400 and cell["open_frame"] == 300
+
+
+def test_every_surface_is_a_matrix_area_with_a_timing_predicate():
+ for name in wk.SURFACES:
+ spec = am.AREAS[name]
+ if spec["open"][0] != "scroll":
+ wk.pred_js(spec["ready"], 0, "x")
+ if spec.get("close"):
+ wk.pred_js(spec["gone"], 1, "x")
+
+
+def test_known_ceilings_name_their_fix():
+ text = (Path(wk.__file__)).read_text(encoding = "utf-8")
+ block = text[text.index("KNOWN_DEFAULT = {") : text.index("KNOWN = {")]
+ for key in wk.KNOWN_DEFAULT:
+ engine, mode, surface, metric = key.split(":")
+ assert engine in wk.ENGINES and mode in wk.MODES and surface in wk.SURFACES
+ assert metric in ("open", "close", "open_frame", "close_frame")
+ if wk.KNOWN_DEFAULT:
+ assert "Fix work:" in block
+
+
+def test_a_frame_that_is_already_long_in_the_short_chat_is_a_mount_cost_not_a_long_chat_one():
+ # WebKit on a CI runner draws a 300-600 ms frame mounting Settings at 1 turn.
+ res = _result(_cell(600, frame = 400), _cell(620, frame = 450))
+ assert not [v for v in wk.judge(res, known = {}) if v["fail"]]
diff --git a/tests/studio/test_webkit_leg_ci.py b/tests/studio/test_webkit_leg_ci.py
new file mode 100644
index 00000000000..6af363d2e94
--- /dev/null
+++ b/tests/studio/test_webkit_leg_ci.py
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The WebKit leg's CI wiring. No harness imports, so the unfiltered workflow-trigger-lint job can run it."""
+
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+
+
+def test_frontend_ci_runs_the_webkit_leg_with_webkit_installed():
+ ci = (REPO / ".github" / "workflows" / "studio-frontend-ci.yml").read_text(encoding = "utf-8")
+ jobs = ci[ci.index("\njobs:") :]
+ build = jobs[jobs.index("\n build:") : jobs.index("\n webkit:")]
+ webkit = jobs[jobs.index("\n webkit:") : jobs.index("\n windows:")]
+ # Its own job, in parallel: the build job's critical path installs no WebKit and runs no leg.
+ assert "needs:" not in webkit
+ for needle in (
+ "install webkit",
+ "chromium webkit",
+ "p.webkit",
+ "LONG_THREAD_WEBKIT",
+ "python3 tests/studio/_webkit_leg.py",
+ ):
+ assert needle not in build
+ assert "python3 -m playwright install webkit" in webkit
+ assert "vite.long-thread.config.ts" in webkit
+ assert "LONG_THREAD_WEBKIT: '1'" in webkit
+ assert "python3 tests/studio/_webkit_leg.py" in webkit
+ assert "tests/studio/test_webkit_leg.py" in build
+ for path in (
+ "tests/studio/_webkit_leg.py",
+ "tests/studio/test_webkit_leg.py",
+ "tests/studio/test_webkit_leg_ci.py",
+ ):
+ assert ci[: ci.index("\njobs:")].count(f"- '{path}'") == 2