Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio CI: WebKit leg for the long-chat gate, report-only paint / CPU length check, recheck gap fixed (#17331)
Daniel Han3fe5adecf7c2
diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.ymlindex 9ca55097561..21284218b0e 100644--- a/.github/workflows/studio-frontend-ci.yml+++ b/.github/workflows/studio-frontend-ci.yml@@ -46,6 +46,9 @@ on:- 'tests/studio/test_css_culprit.py'- 'tests/studio/_area_matrix.py'- 'tests/studio/test_area_matrix.py'+ - 'tests/studio/_webkit_leg.py'+ - 'tests/studio/test_webkit_leg.py'+ - 'tests/studio/test_webkit_leg_ci.py'- 'tests/studio/_appearance_coverage.py'- 'tests/studio/test_appearance_coverage.py'- 'tests/studio/_has_lint.py'@@ -105,6 +108,9 @@ on:- 'tests/studio/test_css_culprit.py'- 'tests/studio/_area_matrix.py'- 'tests/studio/test_area_matrix.py'+ - 'tests/studio/_webkit_leg.py'+ - 'tests/studio/test_webkit_leg.py'+ - 'tests/studio/test_webkit_leg_ci.py'- 'tests/studio/_appearance_coverage.py'- 'tests/studio/test_appearance_coverage.py'- 'tests/studio/_has_lint.py'@@ -431,6 +437,8 @@ jobs:tests/studio/test_code_block_flicker_contract.py \tests/studio/test_css_culprit.py \tests/studio/test_area_matrix.py \+ tests/studio/test_webkit_leg.py \+ tests/studio/test_webkit_leg_ci.py \tests/studio/test_appearance_coverage.py \tests/studio/test_has_lint.py -q@@ -685,6 +693,72 @@ jobs:path: studio/frontend/distretention-days: 1+ # WebKit leg of the long-chat gate (tests/studio/_webkit_leg.py), in parallel with the build job so+ # its WebKit install and run stay off that job's critical path. Times opening and closing the+ # browser pane, Settings, shortcuts, palette, composer menu, sidebar and a thread scroll at 1 and 150+ # turns in Playwright WebKit with a macOS and a Linux user agent (Safari, the macOS app, the Linux+ # desktop app): fails when the long chat is over 2.5x the short one and over 200 ms, or has a frame+ # over 250 ms and over 2.5x the short chat's, beyond the known costs pinned in that file.+ webkit:+ name: Long-chat gate (WebKit)+ runs-on: ubuntu-latest+ env:+ UNSLOTH_DISABLE_UPDATE_CHECK: '1'+ timeout-minutes: 40+ defaults:+ run:+ working-directory: studio/frontend+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0+ with:+ node-version: '22'++ - name: Upgrade npm to 11.x (allowScripts enforcement)+ working-directory: ${{ github.workspace }}+ run: |+ npm install -g npm@^11 --no-fund --no-audit+ V=$(npm -v)+ case "$V" in+ 11.1[6-9].*|11.[2-9][0-9].*|1[2-9].*) echo "npm $V has allowScripts" ;;+ *) echo "::error::npm $V lacks allowScripts (need >=11.16)"; exit 1 ;;+ esac++ # Must run BEFORE npm ci, where a compromised tarball's install scripts run.+ - name: Lockfile supply-chain audit (pre-install scan)+ working-directory: ${{ github.workspace }}+ run: python3 scripts/lockfile_supply_chain_audit.py++ - name: Install dependencies+ run: npm ci --strict-allow-scripts --no-fund --no-audit++ - name: WebKit for the long-chat gate+ working-directory: ${{ github.workspace }}+ shell: bash+ timeout-minutes: 10+ env:+ APT_ACQUIRE_RETRIES: '0'+ run: |+ python3 -m pip install 'playwright>=1.45,<2'+ python3 -m playwright install webkit+ if ! python3 -c "from playwright.sync_api import sync_playwright as s; p = s().start(); p.webkit.launch().close(); p.stop()"; then+ bash .github/scripts/retry-with-apt-lock.sh python3 -m playwright install-deps webkit+ fi++ - name: Long-chat production build+ run: npx vite build --config vite.long-thread.config.ts++ - name: Browser gate for long-chat cost in WebKit+ working-directory: ${{ github.workspace }}+ shell: bash+ timeout-minutes: 20+ env:+ LONG_THREAD_WEBKIT: '1'+ run: python3 tests/studio/_webkit_leg.py+# The suite reads sources and resolves module URLs, so it can break only on Windows.# .gitattributes pins LF for studio/frontend, so exact-substring assertions hold.windows:diff --git a/.github/workflows/workflow-trigger-lint.yml b/.github/workflows/workflow-trigger-lint.ymlindex 49903e381b5..8bb5ad37223 100644--- a/.github/workflows/workflow-trigger-lint.yml+++ b/.github/workflows/workflow-trigger-lint.yml@@ -75,6 +75,7 @@ jobs:run: |python3 -m pytest -q -n 4 \tests/studio/test_absorbed_lanes_still_run.py \+ tests/studio/test_webkit_leg_ci.py \tests/studio/test_appimage_toolchain_contract.py \tests/studio/test_lint_ci_scans_committed_files.py \tests/studio/test_agent_guides_verdicts.py \diff --git a/tests/studio/_area_matrix.py b/tests/studio/_area_matrix.pyindex 6e486062a4e..044e73cd51f 100644--- a/tests/studio/_area_matrix.py+++ b/tests/studio/_area_matrix.py@@ -38,6 +38,7 @@ import http.serverimport jsonimport osimport re+import statisticsimport threadingimport timefrom pathlib import Path@@ -60,6 +61,18 @@ GLASS = "liquid-3d"LENGTH_RATIO = float(os.environ.get("LONG_THREAD_AREAS_LENGTH_RATIO", "2.0"))# Counts below this many elements / layout objects at LONG turns never fail the length check.COUNT_FLOOR = int(os.environ.get("LONG_THREAD_AREAS_COUNT_FLOOR", "3000"))+# Paint and main-thread CPU, long over short chat (the cheapest long reading over the median short one,+# each from the first reading and its rechecks):+# fails over TIME_LENGTH_RATIO AND over TIME_LENGTH_FLOOR_MS more. A short reading below+# TIME_BASE_FLOOR_MS counts as that, so a near-free area cannot make a large ratio.+TIME_LENGTH_RATIO = float(os.environ.get("LONG_THREAD_AREAS_TIME_LENGTH_RATIO", "2.5"))+# Report-only for now: the readings, ratios and a "would fail" list are printed, the exit code is left+# alone. Main reads up to the ceilings below on GitHub runners (no margin yet) and the cost behind them+# is message-level actions (click, menu, tool / reasoning toggles, source card), being worked on+# separately. Set True (or LONG_THREAD_AREAS_LENGTH_TIME_BLOCKING=1) to make them fail the gate.+LENGTH_TIME_BLOCKING = os.environ.get("LONG_THREAD_AREAS_LENGTH_TIME_BLOCKING", "0") == "1"+TIME_LENGTH_FLOOR_MS = float(os.environ.get("LONG_THREAD_AREAS_TIME_LENGTH_FLOOR_MS", "50"))+TIME_BASE_FLOOR_MS = float(os.environ.get("LONG_THREAD_AREAS_TIME_BASE_FLOOR_MS", "20"))MAX_RATIO = float(os.environ.get("LONG_THREAD_AREAS_MAX_RATIO", "4.0"))# Also judged on CPU growth with chat length: a streamed reply restyles only its own message, so its# element / layout counts stay flat while per-token work over the thread grows.@@ -87,6 +100,30 @@ KNOWN_DEFAULT = ",".join(+ [f"{m}:thread_scroll=7" for m in _GLASS_MODES]+ [f"{m}:settings_scroll=7" for m in _GLASS_MODES]+ [f"{m}:type=6" for m in _GLASS_MODES]+ # Main-thread CPU length (renderer main thread, 40 -> 200 turns, judged value, 3 GitHub-runner gate+ # runs of main, default and Liquid 3D, probes that do not themselves scale with the chat). Presses+ # that land inside a message (click, its menu, tool / reasoning toggles, source card) still read+ # 2.5-2.9x; a few sidebar / composer surfaces touch 2.4-2.5x, inside the 0.4x run-to-run spread of+ # the 2.5x rule. Ceilings: 3.5x over 2.5, 3.0x at 2.4-2.5. Everything else keeps the plain 2.5x.+ # Fix work: per-interaction script cost that grows with the mounted thread (not yet assigned);+ # remove each entry as its area drops under 2.5x.+ + [f"cpu:{a}=3.5" for a in ("tool_toggle", "message_click", "message_menu")] # 2.0-2.9x+ + [+ f"cpu:{a}=3" # 1.6-2.5x+ for a in (+ "reasoning_toggle",+ "source_card",+ "chat_row_menu",+ "sidebar_sections",+ "run_settings",+ "thread_scroll",+ "permission_menu",+ "chat_options",+ "organize_chats",+ "sidebar_toggle",+ "api_monitor",+ )+ ])KNOWN = os.environ.get("LONG_THREAD_AREAS_KNOWN", KNOWN_DEFAULT)@@ -709,7 +746,17 @@ class Api:"/api/hub/local": {"models": []},"/api/hub/hidden-models": {"hidden": []},"/api/settings/pinned-models": {"pinned": []},- "/api/health": {"status": "ok"},+ # A settled hardware verdict, as the real backend gives once detection ends. Without+ # `version` / `chat_only` the app keeps capabilities "unknown" and the sidebar Train row+ # spins for the whole run: an always-running animation, and Chromium then recomputes every+ # IntersectionObserver target on each frame, so every action's cost grew with the chat.+ "/api/health": {+ "status": "healthy",+ "version": "2026.11.1",+ "device_type": "linux",+ "chat_only": False,+ "hardware_detecting": False,+ },"/api/inference/status": {"active_model": STUB_MODEL,"model_identifier": STUB_MODEL,@@ -762,8 +809,14 @@ PROBE = ("""() => {const OVER = '%s';const vis = (e) => { const r = e.getBoundingClientRect(); return r.width > 1 && r.height > 1; };- window.__gateOverlays = () => [...document.querySelectorAll(OVER)].filter(vis).length;- window.__gateVisible = (sel) => [...document.querySelectorAll(sel)].some(vis);+ // Polled every frame inside the measured window, so never a document-wide query: that would cost the+ // gate's own CPU in proportion to the chat. Message bodies (nearly all of a long chat) are skipped; no+ // overlay or waited-for control renders inside one (portals sit under <body>).+ const skip = { acceptNode: (e) => (e.hasAttribute('data-role') ? NodeFilter.FILTER_REJECT : NodeFilter.FILTER_ACCEPT) };+ const outside = (sel) => { const out = []; const w = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, skip);+ for (let e = w.nextNode(); e; e = w.nextNode()) if (e.matches(sel)) out.push(e); return out; };+ window.__gateOverlays = () => outside(OVER).filter(vis).length;+ window.__gateVisible = (sel) => outside(sel).some(vis);window.__gateFrames = (n) => new Promise((r) => { let k = 0; const f = () => (++k >= n ? r() : requestAnimationFrame(f)); requestAnimationFrame(f); });window.__gateFocusins = [];document.addEventListener('focusin', (e) => window.__gateFocusins.push(e.target), true);@@ -831,7 +884,9 @@ TABS = """async (timeoutMs) => {return {count: tabs.length, missing, failed};}"""FIND = """(spec) => {- document.querySelectorAll('[data-gate-target]').forEach((e) => e.removeAttribute('data-gate-target'));+ // The target is kept by reference: a document-wide lookup would cost in proportion to the chat.+ if (window.__gateTarget) window.__gateTarget.removeAttribute('data-gate-target');+ window.__gateTarget = null;const vis = (e) => { const r = e.getBoundingClientRect(); return r.width > 0 && r.height > 0; };let el = null;if (spec.startsWith('js|')) el = (0, eval)(spec.slice(3));@@ -843,13 +898,13 @@ FIND = """(spec) => {if (!el) return false;// Instant: the thread viewport is scroll-smooth, so a plain scrollIntoView animates and returns while the// target is still moving; a trusted pointer action then lands wherever the target has got to.- el.setAttribute('data-gate-target', ''); el.scrollIntoView({block: 'nearest', behavior: 'instant'});+ el.setAttribute('data-gate-target', ''); window.__gateTarget = el; el.scrollIntoView({block: 'nearest', behavior: 'instant'});return true;}"""# The target must hold still and be hit at its centre: the floating composer can cover a row scrolled in from# below. Only an occluded target is re-centred, so a clear one adds no scroll to the measured window.STEADY = """async () => {- const el = document.querySelector('[data-gate-target]');+ const el = window.__gateTarget;if (!el) return false;const frame = () => new Promise((r) => requestAnimationFrame(() => r()));const still = async () => {@@ -867,7 +922,7 @@ STEADY = """async () => {return hit();}"""CLICK = """(right) => {- const el = document.querySelector('[data-gate-target]'); const r = el.getBoundingClientRect();+ const el = window.__gateTarget; const r = el.getBoundingClientRect();const at = {bubbles: true, cancelable: true, composed: true, clientX: r.x + r.width / 2, clientY: r.y + r.height / 2,button: right ? 2 : 0, buttons: right ? 2 : 1, pointerType: 'mouse', isPrimary: true, view: window};el.dispatchEvent(new PointerEvent('pointerdown', at)); el.dispatchEvent(new MouseEvent('mousedown', at));@@ -1066,6 +1121,11 @@ class Page:def close(self) -> None:self.ctx.close()+ def target(self):+ # A handle, not a locator: Playwright's selector engine walks the whole document (for shadow+ # roots) on every resolve, inside the measured window, at a cost that grows with the chat.+ return self.page.evaluate_handle("() => window.__gateTarget").as_element()+def chord(self, arg) -> str:if isinstance(arg, dict):return arg["mac" if self.mac else "other"]@@ -1114,12 +1174,12 @@ class Page:self.last_error = "target never held still and clear of overlays under its centre"return Falseif kind == "hover":- page.locator("[data-gate-target]").first.hover(timeout = 3_000, force = True)+ self.target().hover(timeout = 3_000, force = True)self.hovered = Truereturn Trueif kind == "pclick":# A trusted click (mouse move, press, native focus), for areas whose cost is the focus change.- page.locator("[data-gate-target]").first.click(timeout = 3_000, force = True)+ self.target().click(timeout = 3_000, force = True)self.hovered = Truereturn True# Clicks are dispatched in the page: the pointer stays parked, so no tooltip or :hover state@@ -1383,6 +1443,10 @@ def split_trace(events: list) -> dict[str, dict]:row["layout_objects"] += int((args.get("beginData") or {}).get("dirtyObjects") or 0)elif name in ("Paint", "PrePaint", "Layerize", "PaintImage"):row["paint_ms"] += dur+ for row in out.values():+ # The renderer's main thread alone: raster and compositor threads (busy on every frame under+ # software compositing) are noise for a "does this area do more work in a longer chat" check.+ row["main_ms"] = row["threads"].get("CrRendererMain", 0.0)return out@@ -1657,6 +1721,15 @@ def cheapest(cell: dict, key: str) -> float:return min(vals)+def typical(cell: dict, key: str) -> float:+ """Median of the first reading and its rechecks: the yardstick a length ratio divides by. The+ cheapest short reading would let one unusually quiet sample inflate every ratio taken over it."""+ vals = [cell.get(key, 0.0)] + [+ c.get(key, 0.0) for c in cell.get("again", []) if c.get("status") == "OK"+ ]+ return statistics.median(vals)++def judge(result: dict, known: str | None = None) -> list[dict]:"""Verdict rows: kind length / appearance / broken; `fail` set on the failing ones."""limits = known_limits(KNOWN if known is None else known)@@ -1761,6 +1834,30 @@ def judge(result: dict, known: str | None = None) -> list[dict]:lc, sc = settled(page["cells"].get(area, {})), settled(short.get(area, {}))if lc.get("status") != "OK" or sc.get("status") != "OK":continue+ # Time grows where counts may not (paint behind an overlay grows with the thread while style+ # and layout stay flat): paint and main-thread CPU, the cheapest long reading over the+ # median short one (a single quiet short sample cannot inflate the ratio).+ for metric, key in (("paint_ms", "paint"), ("main_ms", "cpu")):+ lv, sv = cheapest(lc, metric), typical(sc, metric)+ ratio = lv / max(sv, TIME_BASE_FLOOR_MS)+ tlimit = limits.get(+ (f"{key}@{mode}", area), limits.get((key, area), TIME_LENGTH_RATIO)+ )+ out.append(+ {+ "kind": "length",+ "metric": metric,+ "turns": turns,+ "mode": mode,+ "area": area,+ "value": lv,+ "base_value": sv,+ "ratio": ratio,+ "limit": tlimit,+ "would_fail": (over := ratio > tlimit and lv - sv > TIME_LENGTH_FLOOR_MS),+ "fail": LENGTH_TIME_BLOCKING and over,+ }+ )limit = limits.get(("length", area), LENGTH_RATIO)cap = limits.get(("count", area))if area in CPU_LENGTH:@@ -1880,6 +1977,29 @@ def thread_split(threads: dict[str, float]) -> str:return ", ".join(f"{n or '?'} {v:.0f} ms" for n, v in top)+def recheck_keys(verdicts: list[dict]) -> list[tuple[int, str, str]]:+ """(turns, mode, area) to measure again: every failing CPU cell and broken area, and a blocking paint+ / main-thread length failure at both lengths."""+ keys = {+ (v["turns"], v["mode"], v["area"])+ for v in verdicts+ if v["fail"]+ and (+ v["kind"] in ("appearance", "length")+ and v["metric"] == "cpu_ms"+ or v["kind"] == "broken"+ and v["area"] != "*"+ )+ }+ keys |= {+ (t, v["mode"], v["area"])+ for v in verdicts+ if v["fail"] and v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms")+ for t in (SHORT, LONG)+ }+ return sorted(keys)++def run(dist: Path,launch_args,@@ -1890,22 +2010,9 @@ def run(result = measure(dist, launch_args, log = log)verdicts = judge(result)# One reading per cell, so every CPU failure is measured again (fresh pages of the mode and its- # base, RECHECKS times, the cheapest reading kept). Grouped by page; at most 4 mode pages.- again = sorted(- {- (v["turns"], v["mode"], v["area"])- for v in verdicts- if v["fail"]- and (- v["kind"] in ("appearance", "length")- and v["metric"] == "cpu_ms"- or v["kind"] == "broken"- and v["area"] != "*"- )- }- )- # Pages with the fewest failing cells first (a lone outlier is the usual noise); a mode that- # fails many cells is one page anyway.+ # base, RECHECKS times, the cheapest reading kept). Grouped by page; at most RECHECK_PAGES pages,+ # those with the fewest failing cells first (a lone outlier is the usual noise).+ again = recheck_keys(verdicts)per_page: dict = {}for t, m, _ in again:per_page[(t, m)] = per_page.get((t, m), 0) + 1@@ -1914,6 +2021,13 @@ def run(if again and RECHECKS:_in_thread(remeasure, result, again, log)verdicts = judge(result)+ # A recheck that lowers one side of a comparison (the short chat, or the base appearance) can+ # make a failure appear only now, judged on a reading that was never measured again: those+ # cells get the same fresh-page rechecks once.+ late = sorted(set(recheck_keys(verdicts)) - set(again))+ if late:+ _in_thread(remeasure, result, late, log)+ verdicts = judge(result)failures, details = [], []trig = (result["pages"].get((SHORT, "default")) or {}).get("triggers")if trig is None:@@ -1931,6 +2045,16 @@ def run(continueif v["kind"] == "broken":failures.append(f"area {v['area']} ({v['mode']}, {v['turns']} turns): {v['why']}")+ elif v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms"):+ what = "paint" if v["metric"] == "paint_ms" else "main-thread CPU"+ key = "paint" if v["metric"] == "paint_ms" else "cpu"+ failures.append(+ f"area {v['area']} ({v['mode']}) costs more as the chat grows: {what} {v['base_value']:,.0f} ms at "+ f"{SHORT} turns -> {v['value']:,.0f} ms at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x over "+ f"{TIME_LENGTH_FLOOR_MS:.0f} ms; LONG_THREAD_AREAS_KNOWN {key}:{v['area']}=... or {key}@{v['mode']}:{v['area']}=...). "+ "Element and layout counts can stay flat while this grows: the thread behind the surface is repainted, "+ "or script walks every message"+ )elif v["kind"] == "length":bound = (f"known ceiling {v['cap']:,.0f} elements, LONG_THREAD_AREAS_KNOWN count:{v['area']}=..."@@ -2019,4 +2143,30 @@ def table(result: dict, verdicts: list[dict]) -> list[str]:+ ("!" if lv and lv["fail"] else ""))lines.append(row)+ timed = sorted(+ (v for v in verdicts if v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms")),+ key = lambda v: -(v["ratio"] / v["limit"]),+ )+ if timed:+ lines.append(+ f" paint / CPU growth {SHORT} -> {LONG} turns, highest against limit (! = over): "+ + ", ".join(+ f"{v['mode']}/{v['area']} {v['metric'][:-3]} {v['base_value']:.0f}->{v['value']:.0f} ms "+ f"{v['ratio']:.1f}x/{v['limit']:.1f}{'!' if v.get('would_fail') else ''}"+ for v in timed[:6]+ )+ )+ would = [v for v in timed if v.get("would_fail")]+ mode_note = "blocking" if LENGTH_TIME_BLOCKING else "report-only, not failing the gate"+ lines.append(+ f" paint / CPU growth would fail ({mode_note}): "+ + (+ ", ".join(+ f"{v['mode']}/{v['area']} {v['metric'][:-3]} {v['base_value']:.0f}->{v['value']:.0f} ms "+ f"{v['ratio']:.1f}x/{v['limit']:.1f}"+ for v in would+ )+ or "none"+ )+ )return linesdiff --git a/tests/studio/_webkit_leg.py b/tests/studio/_webkit_leg.pynew file mode 100644index 00000000000..af71ffce84c--- /dev/null+++ b/tests/studio/_webkit_leg.py@@ -0,0 +1,538 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""WebKit leg of the long-chat gate: open/close time and longest frame per surface, short vs long chat.++The Chromium area matrix (_area_matrix.py) counts restyled elements and CPU from a trace, which WebKit+cannot give. Here each surface is timed in the page itself: from just before the trigger to the first+frame after the surface is visible (and back for closing), with the longest rAF frame inside that+window. Playwright's WebKit runs twice: with a macOS Safari user agent and platform, so the app takes+the macOS paths (the WebKitGTK long-thread windowing of #16854 stays off, as on a Mac), and with its+own Linux user agent (the Linux desktop app's WebKitGTK paths). Same seeded chat as the matrix, served+from the same production build with the API answered in-process.++Per (engine, appearance, surface): the median of REPS runs on one page at SHORT and at LONG turns.+Fails when the long chat is over RATIO x the short one AND over FLOOR_MS, or when its longest frame is+over FRAME_MS AND over FRAME_GROWTH x the short chat's (WebKit on a CI runner already draws 300-600 ms+frames mounting Settings at 1 turn: a mount cost, not a long-chat one); a failing cell is measured again+on fresh pages and the lower reading kept. The two engines run in parallel, one browser each. Today's+known WebKit costs are pinned in KNOWN (measured on main, with margin) so the gate holds that level+and fails on anything worse; each names the fix work, and tightening is a one-line change.+"""++from __future__ import annotations++import json+import os+import statistics+import threading+import time+from pathlib import Path+from urllib.parse import urlparse++import _area_matrix as am++SHORT = int(os.environ.get("LONG_THREAD_WEBKIT_SHORT", "1"))+LONG = int(os.environ.get("LONG_THREAD_WEBKIT_LONG", "150"))+REPS = int(os.environ.get("LONG_THREAD_WEBKIT_REPS", "3"))+RATIO = float(os.environ.get("LONG_THREAD_WEBKIT_RATIO", "2.5"))+FLOOR_MS = float(os.environ.get("LONG_THREAD_WEBKIT_FLOOR_MS", "200"))+FRAME_MS = float(os.environ.get("LONG_THREAD_WEBKIT_FRAME_MS", "250"))+FRAME_GROWTH = float(os.environ.get("LONG_THREAD_WEBKIT_FRAME_GROWTH", "2.5"))+# A short-chat time below this counts as this, so a near-instant surface cannot make a large ratio.+SHORT_FLOOR_MS = 100.0+TIMEOUT_MS = 8_000+SETTLE_MS = 150++MAC_UA = (+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) "+ "Version/18.0 Safari/605.1.15"+)+ENGINES = {+ "webkit_mac": {"user_agent": MAC_UA, "platform": "MacIntel"},+ "webkit_linux": {},+}+MODES = ("default", am.GLASS)+# The surfaces Round 11 measured on real runners (plans/xplat_lag_status3.md), as _area_matrix AREAS.+SURFACES = [+ "browser_pane",+ "settings",+ "shortcuts_dialog",+ "command_palette",+ "composer_menu",+ "sidebar_toggle",+ "thread_scroll",+]++# Known WebKit long-chat costs on main, pinned so the gate holds today's level. Keys: engine:mode:surface:+# metric (open / close = long/short ratio of the time; frame = the long chat's longest frame, ms).+# Set from main readings with margin; lower each when its fix lands.+KNOWN_DEFAULT = {+ # Browser pane: opening or closing it narrows / widens the thread and WebKit re-lays out and repaints+ # the rows still inside the #16986 content-visibility window (macos-15 Safari engine, Round 11:+ # 460-562 ms at 150 turns vs 144-165 at 1). GitHub ubuntu runners, 6 readings: macOS UA open 1.5-3.0x+ # (frame 267-335 ms), close 3.6-7.2x / 3.6x over the 100 ms floor (frame 235-300); Linux UA open up+ # to 3.9x (Glass 6.4x, frame 327-453), close 2.9x (Glass 4.8x, frame 191-297). Fix work: branch+ # studio-browser-pane-open.+ "webkit_mac:default:browser_pane:open": 4.0,+ "webkit_mac:default:browser_pane:close": 5.0,+ "webkit_mac:default:browser_pane:open_frame": 450,+ "webkit_mac:default:browser_pane:close_frame": 400,+ "webkit_mac:liquid-3d:browser_pane:close": 7.0,+ "webkit_mac:liquid-3d:browser_pane:close_frame": 520,+ "webkit_linux:default:browser_pane:open": 5.0,+ "webkit_linux:default:browser_pane:close": 4.0,+ "webkit_linux:liquid-3d:browser_pane:open": 8.0,+ "webkit_linux:liquid-3d:browser_pane:close": 6.5,+ "webkit_linux:liquid-3d:browser_pane:open_frame": 600,+ "webkit_linux:liquid-3d:browser_pane:close_frame": 400,+ # Thread scroll: 20 scroll steps take 3.0-3.3x as long at 150 turns (macOS UA solid; Linux UA with+ # Glass, the WebKitGTK Glass scroll of Round 11: 12-38 of 67-132 frames over 50 ms on Linux).+ # Fix work: Linux WebKitGTK Glass scroll (GOAL_DETAILS open todo); macOS thread scroll paint.+ "webkit_mac:default:thread_scroll:open": 4.5,+ "webkit_linux:liquid-3d:thread_scroll:open": 4.5,+ # Composer menu and command palette (macOS UA, solid): a 237-297 / 182-268 ms frame opening them at+ # 150 turns: the paint behind the overlay grows with the thread (Round 11's Chromium probe: composer+ # menu paint 10 -> 80 ms, Settings 15 -> 104). Fix work: Settings / overlay paint (GOAL_DETAILS todo).+ "webkit_mac:default:composer_menu:open_frame": 400,+ "webkit_mac:default:command_palette:open_frame": 350,+}+KNOWN = {+ **KNOWN_DEFAULT,+ **{+ k: float(v)+ for k, _, v in (+ p.strip().partition("=")+ for p in os.environ.get("LONG_THREAD_WEBKIT_KNOWN", "").split(",")+ if p.strip()+ )+ },+}++ARM = """() => {+ const w = window.__wk = {t0: performance.now(), frames: [], last: performance.now(), stop: false};+ const tick = (t) => { w.frames.push([t, t - w.last]); w.last = t; if (!w.stop) requestAnimationFrame(tick); };+ requestAnimationFrame(tick);+}"""+# Wait (in rAF steps) until `pred` holds, then one more frame: the time is that frame's, so it+# includes painting the surface. Returns [ms since ARM, longest frame in the window] or null.+UNTIL = """async ([pred, timeoutMs]) => {+ const w = window.__wk; const test = (0, eval)(pred);+ const frame = () => new Promise((r) => requestAnimationFrame(r));+ while (!test()) { if (performance.now() - w.t0 > timeoutMs) { w.stop = true; return null; } await frame(); }+ const t = await frame(); w.stop = true;+ const long = w.frames.filter(([ts]) => ts <= t + 1).reduce((m, [, d]) => Math.max(m, d), 0);+ return [t - w.t0, long];+}"""+# Thread scroll: the matrix's 20-step scroll; the time is the whole scroll, the frame its longest.+SCROLLED = """() => { const w = window.__wk; w.stop = true; const t = performance.now();+ return [t - w.t0, w.frames.reduce((m, [, d]) => Math.max(m, d), 0)]; }"""+++def pred_js(pred, base_overlays: int, attr_before) -> str:+ kind = pred[0]+ if kind == "overlay":+ return f"() => window.__gateOverlays() > {base_overlays}"+ if kind == "overlay_gone":+ return f"() => window.__gateOverlays() <= {max(0, base_overlays - 1)} || window.__gateOverlays() === 0"+ if kind == "sel":+ return f"() => window.__gateVisible({json.dumps(pred[1])})"+ if kind == "gone":+ return f"() => !window.__gateVisible({json.dumps(pred[1])})"+ if kind == "changed":+ return (+ f"() => {{ const v = window.__gateAttr({json.dumps(pred[1])}, {json.dumps(pred[2])}); "+ f"return v !== null && v !== {json.dumps(attr_before)}; }}"+ )+ if kind == "restored":+ return f"() => window.__gateAttr({json.dumps(pred[1])}, {json.dumps(pred[2])}) === {json.dumps(attr_before)}"+ raise ValueError(f"no timing predicate for {kind}")+++class WebKitPage(am.Page):+ """The matrix's page (same thread, API and controls) in WebKit, without Chromium tracing."""++ def __init__(self, browser, base: str, turns: int, mode: dict, engine: str):+ self.browser, self.turns, self.mode, self.tabs = browser, turns, mode, []+ self.api = am.Api(turns)+ extra = (+ {"user_agent": ENGINES[engine]["user_agent"]} if "user_agent" in ENGINES[engine] else {}+ )+ self.ctx = browser.new_context(viewport = {"width": 1440, "height": 900}, **extra)+ if ENGINES[engine].get("platform"):+ self.ctx.add_init_script(+ "Object.defineProperty(Navigator.prototype, 'platform', {get: () => %s});"+ % json.dumps(ENGINES[engine]["platform"])+ )+ init = {+ "unsloth_auth_token": am.auth_token(),+ "unsloth_auth_refresh_token": "r",+ "unsloth_window_material": am.material_storage(mode["set"]),+ "theme": mode["set"]["theme"],+ }+ self.ctx.add_init_script(+ "(() => { const v = %s; for (const k in v) localStorage.setItem(k, v[k]); })()"+ % json.dumps(init)+ )+ self.ctx.route(lambda url: urlparse(url).path.startswith("/api/"), self.api.handle)+ self.ctx.route(+ lambda url: not url.startswith(base), lambda route: route.fulfill(status = 204, body = "")+ )+ self.page = page = self.ctx.new_page()+ self.errors, self.last_error = [], ""+ self.attr_before, self.focusins_before, self.hovered = None, 0, False+ page.goto(f"{base}/chat?thread={self.api.rec['id']}", wait_until = "load", timeout = 180_000)+ page.wait_for_function(+ "(n) => document.querySelectorAll('[data-role]').length >= n",+ arg = 2 * turns,+ timeout = 180_000,+ )+ page.evaluate(am.PROBE)+ page.wait_for_timeout(1_500)+ self.mac = page.evaluate(+ "() => /mac|iphone|ipad|ipod/i.test(`${navigator.platform || ''} ${navigator.userAgent || ''}`)"+ )+ page.mouse.move(1435, 450)++ def timed(self, action, pred, base_overlays: int):+ self.page.evaluate(ARM)+ if not self.act(action):+ self.page.evaluate("() => { window.__wk.stop = true; }")+ return None+ if pred is None:+ return self.page.evaluate(SCROLLED)+ return self.page.evaluate(+ UNTIL, [pred_js(pred, base_overlays, self.attr_before), TIMEOUT_MS]+ )++ def surface(self, name: str) -> dict:+ """One open (and close) of a surface: {open_ms, open_frame, close_ms, close_frame} or {error}."""+ spec = am.AREAS[name]+ page = self.page+ self.reset()+ page.wait_for_timeout(SETTLE_MS)+ base = page.evaluate("() => window.__gateOverlays()")+ ready = spec["ready"]+ self.attr_before = (+ page.evaluate("([s, a]) => window.__gateAttr(s, a)", [ready[1], ready[2]])+ if ready[0] == "changed"+ else None+ )+ scroll = spec["open"][0] == "scroll"+ got = self.timed(spec["open"], None if scroll else ready, base)+ if got is None:+ return {"error": f"never reached {ready[0]}"}+ out = {"open_ms": got[0], "open_frame": got[1]}+ if spec.get("close"):+ page.wait_for_timeout(SETTLE_MS)+ gone = self.timed(spec["close"], spec["gone"], base + 1)+ if gone is None:+ return {**out, "error": f"opened but never reached {spec['gone'][0]}"}+ out.update(close_ms = gone[0], close_frame = gone[1])+ if spec.get("teardown"):+ self.act(spec["teardown"])+ return out++ def measure(self, surfaces: list[str], reps: int) -> dict[str, dict]:+ cells: dict[str, dict] = {}+ for name in surfaces:+ runs, error = [], None+ for _ in range(reps):+ try:+ r = self.surface(name)+ except Exception as exc: # noqa: BLE001 - reported as a NOT_RUN surface+ r = {"error": f"{type(exc).__name__}: {str(exc)[:120]}"}+ if "error" in r:+ error = r["error"]+ continue+ runs.append(r)+ if not runs:+ cells[name] = {"status": f"NOT_RUN: {error}"}+ continue+ cell = {"status": "OK", "reps": len(runs)}+ for key in ("open_ms", "open_frame", "close_ms", "close_frame"):+ vals = [r[key] for r in runs if key in r]+ if vals:+ cell[key] = round(statistics.median(vals), 1)+ cells[name] = cell+ return cells+++def _engine_worker(engine, dist_base, modes, surfaces, reps, out, log) -> None:+ from playwright.sync_api import sync_playwright+ with sync_playwright() as pw:+ browser = pw.webkit.launch()+ try:+ for mode in modes:+ for turns in (SHORT, LONG):+ t = time.time()+ try:+ page = WebKitPage(browser, dist_base, turns, mode, engine)+ except Exception as exc: # noqa: BLE001+ out[(engine, mode["name"], turns)] = {+ "error": f"did not load: {str(exc)[:160]}"+ }+ continue+ try:+ out[(engine, mode["name"], turns)] = {"cells": page.measure(surfaces, reps)}+ finally:+ page.close()+ log(+ f" webkit: {engine} {mode['name']} {turns} turns ({time.time() - t:.0f} s)",+ flush = True,+ )+ finally:+ browser.close()+++def measure(+ dist: Path,+ surfaces = None,+ modes = MODES,+ engines = None,+ reps = REPS,+ log = print,+) -> dict:+ surfaces = surfaces or SURFACES+ by_name = {m["name"]: m for m in am.app_modes()}+ chosen = [by_name[m] for m in modes]+ server, base = am.serve_app(dist)+ out: dict = {}+ try:+ # One browser per engine (a browser per appearance as well doubles the frames on a 4-core runner).+ crashed: dict = {}++ def work(engine):+ try:+ _engine_worker(engine, base, chosen, surfaces, reps, out, log)+ except Exception as exc: # noqa: BLE001 (recorded below: a crash must never read as a pass)+ crashed[engine] = f"{type(exc).__name__}: {str(exc)[:160]}"++ engines = engines or ENGINES+ threads = [threading.Thread(target = work, args = (e,)) for e in engines]+ for t in threads:+ t.start()+ for t in threads:+ t.join()+ finally:+ server.shutdown()+ # Every planned page gets a result: one a worker never recorded is a load failure.+ for e in engines:+ for m in chosen:+ for turns in (SHORT, LONG):+ out.setdefault(+ (e, m["name"], turns),+ {+ "error": f"not measured ({crashed.get(e, 'the engine worker stopped early')})"+ },+ )+ return out+++def judge(result: dict, known: dict | None = None) -> list[dict]:+ known = KNOWN if known is None else known+ out = []+ for (engine, mode, turns), page in sorted(result.items()):+ if "error" in page:+ out.append(+ {+ "engine": engine,+ "mode": mode,+ "surface": "*",+ "metric": "load",+ "fail": True,+ "why": page["error"],+ }+ )+ for (engine, mode, turns), page in sorted(result.items()):+ if turns != LONG or "cells" not in page:+ continue+ short_page = result.get((engine, mode, SHORT)) or {}+ if "cells" not in short_page:+ continue # its load failure is reported above+ short = short_page["cells"]+ for name in sorted(set(page["cells"]) | set(short)):+ lc, sc = page["cells"].get(name, {}), short.get(name, {})+ if lc.get("status") != "OK" or sc.get("status") != "OK":+ # A surface missing at either length is a gap in the gate, never a silent pass.+ out.append(+ {+ "engine": engine,+ "mode": mode,+ "surface": name,+ "metric": "run",+ "fail": True,+ "why": f"{lc.get('status', 'missing')} at {LONG} turns, {sc.get('status', 'missing')} at {SHORT}",+ }+ )+ continue+ for edge in ("open", "close"):+ lv, sv = lc.get(f"{edge}_ms"), sc.get(f"{edge}_ms")+ if lv is None or sv is None:+ continue+ ratio = lv / max(sv, SHORT_FLOOR_MS)+ limit = known.get(f"{engine}:{mode}:{name}:{edge}", RATIO)+ out.append(+ {+ "engine": engine,+ "mode": mode,+ "surface": name,+ "metric": edge,+ "value": lv,+ "base_value": sv,+ "ratio": ratio,+ "limit": limit,+ "fail": ratio > limit and lv > FLOOR_MS,+ }+ )+ fv = lc.get(f"{edge}_frame")+ if fv is not None:+ flimit = known.get(f"{engine}:{mode}:{name}:{edge}_frame", FRAME_MS)+ sf = sc.get(f"{edge}_frame") or 0.0+ out.append(+ {+ "engine": engine,+ "mode": mode,+ "surface": name,+ "metric": f"{edge}_frame",+ "value": fv,+ "base_value": sf,+ "limit": flimit,+ "fail": fv > flimit and fv > FRAME_GROWTH * sf,+ }+ )+ return out+++def lower(first: dict, again: dict) -> dict:+ """Merge a recheck: per cell, the lower of each reading (a recheck only ever lowers a reading)."""+ merged = dict(first)+ for key, page in again.items():+ if "cells" not in page:+ continue+ if "cells" not in merged.get(key, {}):+ merged[key] = page+ continue+ cells = dict(merged[key]["cells"])+ for name, cell in page["cells"].items():+ old = cells.get(name, {})+ if cell.get("status") != "OK":+ continue+ if old.get("status") != "OK":+ cells[name] = cell+ continue+ cells[name] = {+ **old,+ **{+ k: min(old[k], v)+ for k, v in cell.items()+ if k.endswith(("_ms", "_frame")) and k in old+ },+ }+ merged[key] = {"cells": cells}+ return merged+++def run(dist: Path, log = print) -> tuple[list[str], list[str], dict]:+ """(report lines, failures, result)."""+ started = time.time()+ result = measure(dist, log = log)+ verdicts = judge(result)+ failing = {+ (v["engine"], v["mode"], v["surface"])+ for v in verdicts+ if v["fail"] and v["surface"] != "*"+ }+ if failing:+ log(+ f" webkit: {len(failing)} cell(s) over their limit; measuring them again on fresh pages",+ flush = True,+ )+ for engine in sorted({f[0] for f in failing}):+ mode_names = sorted({f[1] for f in failing if f[0] == engine})+ surfaces = sorted({f[2] for f in failing if f[0] == engine})+ again = measure(dist, surfaces = surfaces, modes = mode_names, engines = [engine], log = log)+ result = lower(result, again)+ verdicts = judge(result)+ lines = table(result, verdicts)+ lines.append(+ f" webkit leg: {len(ENGINES)} engines x {len(MODES)} appearances x {SHORT}/{LONG} turns, {time.time() - started:.0f} s"+ )+ failures = []+ for v in verdicts:+ if not v["fail"]:+ continue+ where = f"{v['engine']}, {v['mode']}"+ if v["metric"] in ("load", "run"):+ failures.append(f"WebKit {where}: {v['surface']} {v['why']}")+ elif v["metric"].endswith("frame"):+ failures.append(+ f"WebKit {where}: {v['surface']} ({v['metric'].replace('_', ' ')}) has a {v['value']:.0f} ms frame at "+ f"{LONG} turns against {v['base_value']:.0f} ms at {SHORT} (limit {v['limit']:.0f} ms and {FRAME_GROWTH:g}x; LONG_THREAD_WEBKIT_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...)"+ )+ else:+ failures.append(+ f"WebKit {where}: {'opening' if v['metric'] == 'open' else 'closing'} {v['surface']} takes {v['base_value']:.0f} ms at {SHORT} turns -> "+ f"{v['value']:.0f} ms at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x over {FLOOR_MS:.0f} ms; "+ f"LONG_THREAD_WEBKIT_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...). It costs more "+ "as the chat grows in WebKit (Safari, the macOS app, the Linux desktop app): usually re-layout or "+ "repaint of the whole thread behind the surface"+ )+ return lines, failures, {"result": result, "verdicts": verdicts}+++def table(result: dict, verdicts: list[dict]) -> list[str]:+ by = {(v["engine"], v["mode"], v["surface"], v["metric"]): v for v in verdicts}+ lines = [+ f" WebKit: open / close ms at {SHORT} -> {LONG} turns, longest frame at {LONG} (! = over its limit)"+ ]+ for (engine, mode, turns), page in sorted(result.items()):+ if turns != LONG:+ continue+ if "error" in page:+ lines.append(f" {engine:<13}{mode:<10} {page['error']}")+ continue+ short = (result.get((engine, mode, SHORT)) or {}).get("cells", {})+ for name, lc in page["cells"].items():+ sc = short.get(name, {})+ if lc.get("status") != "OK":+ lines.append(f" {engine:<13}{mode:<10}{name:<18}{lc.get('status')}")+ continue+ parts = []+ for edge in ("open", "close"):+ if f"{edge}_ms" not in lc:+ continue+ v = by.get((engine, mode, name, edge))+ f = by.get((engine, mode, name, f"{edge}_frame"))+ parts.append(+ f"{edge} {sc.get(f'{edge}_ms', float('nan')):>5.0f} -> {lc[f'{edge}_ms']:>5.0f}"+ + (f" ({v['ratio']:.1f}x{'!' if v['fail'] else ''})" if v else "")+ + f" frame {lc.get(f'{edge}_frame', 0):>4.0f}{'!' if f and f['fail'] else ''}"+ )+ lines.append(f" {engine:<13}{mode:<10}{name:<18}" + " ".join(parts))+ return lines+++def main() -> int:+ """Frontend CI's WebKit job: the leg alone, on the long-thread production build."""+ if os.environ.get("LONG_THREAD_WEBKIT", "0") != "1":+ print("WebKit leg: off (set LONG_THREAD_WEBKIT=1)")+ return 0+ repo = Path(__file__).resolve().parents[2]+ dist = Path(+ os.environ.get("LONG_THREAD_DIST", repo / ".playwright-cli" / "long-thread-production")+ )+ if not (dist / "index.html").exists():+ print(f"FAIL: no app build at {dist / 'index.html'} for the WebKit leg")+ return 1+ lines, fails, _ = run(dist)+ for line in lines:+ print(line)+ for f in fails:+ print(f"FAIL: {f}")+ print("WebKit leg: " + (f"{len(fails)} failure(s)" if fails else "PASS"))+ return 1 if fails else 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tests/studio/playwright_long_thread_gate.py b/tests/studio/playwright_long_thread_gate.pyindex 0f16bd048c0..0e144203994 100644--- a/tests/studio/playwright_long_thread_gate.py+++ b/tests/studio/playwright_long_thread_gate.py@@ -51,6 +51,7 @@ from playwright.sync_api import sync_playwrightsys.path.insert(0, str(Path(__file__).resolve().parent))import _area_matrix as areas # noqa: E402import _has_lint as has_lint # noqa: E402+import _webkit_leg as webkit_leg # noqa: E402from _css_culprit import explain, has_descendant, thread_anchor # noqa: E402from _playwright_robust import chromium_launch_args # noqa: E402@@ -80,6 +81,10 @@ AREAS = os.environ.get("LONG_THREAD_AREAS", "auto")ONLY_AREAS = os.environ.get("LONG_THREAD_ONLY_AREAS") == "1"# Static lint of the built CSS for new non-subject :has() rules (tests/studio/_has_lint.py).HAS_LINT = os.environ.get("LONG_THREAD_HAS_LINT", "1") != "0"+# The WebKit leg (_webkit_leg.py): surface open/close time and longest frame, short vs long chat, in+# Playwright WebKit with a macOS and a Linux user agent. Opt-in (one Linux CI lane sets it), as it+# needs the WebKit browser installed.+WEBKIT = os.environ.get("LONG_THREAD_WEBKIT", "0") == "1"HEALTHY = "healthy reference: 0.7-1.0 ms/key selector matching, 1.00x, 0.2 ms/key :has()"KEYS = "the quick "PRECHECK_KEYS = "ab"@@ -726,6 +731,19 @@ def finish_areas(runs, failures, details, started) -> int:default = str,))+ if WEBKIT:+ if not (DIST / "index.html").exists():+ failures.append(f"no app build at {DIST / 'index.html'} for the WebKit leg")+ else:+ try:+ lines, fails, _ = webkit_leg.run(DIST)+ except Exception as exc: # noqa: BLE001 - a leg that cannot start is a failure, not a pass+ lines, fails = (+ [],+ [f"the WebKit leg did not run: {type(exc).__name__}: {str(exc)[:300]}"],+ )+ extra += lines+ failures += failsreturn report(runs, failures, details, started, extra)diff --git a/tests/studio/test_area_matrix.py b/tests/studio/test_area_matrix.pyindex 2f79f9e081c..8adfaf51c8b 100644--- a/tests/studio/test_area_matrix.py+++ b/tests/studio/test_area_matrix.py@@ -83,6 +83,12 @@ def test_coverage_matcher():assert am.uncovered(triggers) == ["dropdown-menu-trigger:Brand new menu"]+def length_blocking(monkeypatch):+ """The paint / main-thread length rule as it judges once LENGTH_TIME_BLOCKING is switched on."""+ monkeypatch.setattr(am, "LENGTH_TIME_BLOCKING", True)++def _page(cells):return {"cells": {a: {"status": "OK", **c} for a, c in cells.items()}}@@ -336,6 +342,100 @@ def test_missing_default_page_fails_coverage(monkeypatch):assert sum("measuring the page failed" in f for f in failures) == len(out)+def test_a_length_failure_that_appears_after_a_recheck_is_measured_again(+ monkeypatch, length_blocking+):+ # CI (message_menu, Liquid 3D): a 506 ms first short reading failed the appearance check, so only+ # the short page was measured again (99, 90 ms). That made the length check fail for the first+ # time, on a long reading (354 ms) nobody had measured again.+ cell = lambda cpu, main: {+ "elements": 10,+ "layout_objects": 0,+ "cpu_ms": cpu,+ "main_ms": main,+ "paint_ms": 5,+ }+ pages = {+ (am.SHORT, "default"): _page({"type": cell(100, 90)}),+ (am.SHORT, "frosted"): _page({"type": cell(2000, 506)}),+ (am.LONG, "default"): _page({"type": cell(110, 100)}),+ (am.LONG, "frosted"): _page({"type": cell(400, 354)}),+ }+ fake = {+ "pages": pages,+ "modes": {m["name"]: m for m in am.app_modes()},+ "areas": ["type"],+ "seconds": 0.0,+ }+ monkeypatch.setattr(am, "measure", lambda *a, **k: fake)+ calls = []++ def remeasure(result, items, log):+ calls.append(sorted(items))+ for turns, mode, area in items:+ reading = {"status": "OK", "elements": 10, "layout_objects": 0, "paint_ms": 5}+ reading.update(+ {"cpu_ms": 200, "main_ms": 95}+ if turns == am.SHORT+ else {"cpu_ms": 220, "main_ms": 120}+ )+ result["pages"][(turns, mode)]["cells"][area].setdefault("again", []).extend(+ [reading] * am.RECHECKS+ )++ monkeypatch.setattr(am, "remeasure", remeasure)+ _, failures, _, _ = am.run(Path("unused"), [], log = lambda *a, **k: None)+ assert calls[0] == [(am.SHORT, "frosted", "type")]+ assert calls[1] == [(am.LONG, "frosted", "type")]+ assert not [f for f in failures if "costs more as the chat grows" in f]+++def test_an_appearance_failure_that_appears_after_a_recheck_is_measured_again(monkeypatch):+ # CI, main, Liquid 3D typing at 200 turns: one sweep of the 200-turn pages caught a background-+ # thread burst in every look (default 1,266 ms CPU, Liquid 3D 1,005). Another mode's failure got+ # the default page measured again (154 ms); Liquid 3D was then judged 1,005 / 154 = 6.5x against+ # 6x on its never-rechecked first reading. Its own recheck reads 1.6x.+ cell = lambda cpu: {+ "elements": 70,+ "layout_objects": 200,+ "cpu_ms": cpu,+ "main_ms": cpu / 2,+ "paint_ms": 15,+ }+ pages = {+ (am.SHORT, "default"): _page({"type": cell(150)}),+ (am.SHORT, "frosted"): _page({"type": cell(160)}),+ (am.SHORT, am.GLASS): _page({"type": cell(230)}),+ (am.LONG, "default"): _page({"type": cell(1266)}),+ (am.LONG, "frosted"): _page({"type": cell(9000)}),+ (am.LONG, am.GLASS): _page({"type": cell(1005)}),+ }+ fake = {+ "pages": pages,+ "modes": {m["name"]: m for m in am.app_modes()},+ "areas": ["type"],+ "seconds": 0.0,+ }+ monkeypatch.setattr(am, "measure", lambda *a, **k: fake)+ calm = {"default": 154, "frosted": 300, am.GLASS: 250}+ calls = []++ def remeasure(result, items, log):+ calls.append(sorted(items))+ for turns, mode, area in items:+ for name in ("default", mode):+ reading = {"status": "OK", **cell(calm[name])}+ result["pages"][(turns, name)]["cells"][area].setdefault("again", []).extend(+ [reading] * am.RECHECKS+ )++ monkeypatch.setattr(am, "remeasure", remeasure)+ _, failures, _, _ = am.run(Path("unused"), [], log = lambda *a, **k: None)+ assert calls[0] == [(am.LONG, "frosted", "type")]+ assert calls[1] == [(am.LONG, am.GLASS, "type")]+ assert not [f for f in failures if "makes area type cost" in f]++def test_every_area_opened_by_a_shortcut_or_toggle_proves_it_happened():# A key press always "succeeds"; only a visible change shows the app acted on it.for name, spec in am.AREAS.items():@@ -467,6 +567,147 @@ def test_settings_ready_marker_is_set_from_inside_the_panel_suspense():assert "data-settings-panel-ready=" in am.TABS and "data-settings-panel-failed" in am.TABS+ "short, long_, fails",+ [+ (15, 104, True), # Round 11's Settings paint behind the overlay at 1 vs 150 turns+ (100, 240, False), # under 2.5x+ (10, 55, False), # over 2.5x, but under 50 ms more+ (40, 140, True),+ ],+)+def test_paint_length_check(short, long_, fails, length_blocking):+ pages = {+ (am.SHORT, "default"): _page(+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 10, "paint_ms": short}}+ ),+ (am.LONG, "default"): _page(+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 10, "paint_ms": long_}}+ ),+ }+ rows = [v for v in am.judge(_result(pages), known = "") if v["metric"] == "paint_ms"]+ assert [v["fail"] for v in rows] == [fails]+ if fails:+ assert not [v for v in am.judge(_result(pages), known = "paint:a=8") if v["fail"]]+ assert not [v for v in am.judge(_result(pages), known = "paint@default:a=8") if v["fail"]]+++def test_paint_length_check_keeps_the_cheapest_reading():+ # A recheck that reads lower clears a noisy first reading (the minimum of the sweeps is judged).+ long_cell = {+ "elements": 10,+ "layout_objects": 0,+ "cpu_ms": 10,+ "paint_ms": 300,+ "again": [{"status": "OK", "paint_ms": 30, "cpu_ms": 10}],+ }+ pages = {+ (am.SHORT, "default"): _page(+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 10, "paint_ms": 20}}+ ),+ (am.LONG, "default"): _page({"a": long_cell}),+ }+ assert not [v for v in am.judge(_result(pages), known = "") if v["fail"]]++[email protected]("metric", ["paint_ms", "main_ms"])+def test_length_ratio_divides_by_the_median_short_reading(metric, length_blocking):+ # CI, reasoning_toggle (Liquid 3D): short readings 23 / 34 / 49 ms, long 76 ms. The cheapest short+ # reading (23) made it 3.3x; over the median (34) it is 2.2x and passes.+ short_cell = {+ "elements": 10,+ "layout_objects": 0,+ metric: 23,+ "again": [{"status": "OK", metric: 49}, {"status": "OK", metric: 34}],+ }+ long_cell = {+ "elements": 10,+ "layout_objects": 0,+ metric: 90,+ "again": [{"status": "OK", metric: 76}],+ }+ pages = {+ (am.SHORT, "default"): _page({"a": short_cell}),+ (am.LONG, "default"): _page({"a": long_cell}),+ }+ rows = [+ v+ for v in am.judge(_result(pages), known = "")+ if v["metric"] == metric and v["kind"] == "length"+ ]+ assert [(v["value"], v["base_value"], v["fail"]) for v in rows] == [(76, 34, False)]+ # A real growth still fails over the same median: 34 -> 160 ms is 4.7x and 126 ms more.+ long_cell = {+ "elements": 10,+ "layout_objects": 0,+ metric: 170,+ "again": [{"status": "OK", metric: 160}],+ }+ pages[(am.LONG, "default")] = _page({"a": long_cell})+ rows = [+ v+ for v in am.judge(_result(pages), known = "")+ if v["metric"] == metric and v["kind"] == "length"+ ]+ assert [(v["value"], v["base_value"], v["fail"]) for v in rows] == [(160, 34, True)]+++def test_length_time_checks_report_but_do_not_fail_by_default():+ # Report-only until message-level actions stop growing with the chat (LENGTH_TIME_BLOCKING).+ assert am.LENGTH_TIME_BLOCKING is False+ pages = {+ (am.SHORT, "default"): _page(+ {+ "a": {+ "elements": 10,+ "layout_objects": 0,+ "cpu_ms": 80,+ "main_ms": 80,+ "paint_ms": 15,+ }+ }+ ),+ (am.LONG, "default"): _page(+ {+ "a": {+ "elements": 10,+ "layout_objects": 0,+ "cpu_ms": 90,+ "main_ms": 400,+ "paint_ms": 104,+ }+ }+ ),+ }+ result = _result(pages)+ verdicts = am.judge(result, known = "")+ rows = [v for v in verdicts if v["kind"] == "length" and v["metric"] in ("paint_ms", "main_ms")]+ assert [(v["metric"], v["would_fail"], v["fail"]) for v in rows] == [+ ("paint_ms", True, False),+ ("main_ms", True, False),+ ]+ assert not am.recheck_keys(verdicts), "a report-only reading costs no recheck time"+ text = "\n".join(am.table(result, verdicts))+ assert "would fail (report-only, not failing the gate): default/a paint 15->104 ms" in text+++def test_cpu_length_check(length_blocking):+ pages = {+ (am.SHORT, "default"): _page(+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 80, "main_ms": 80}}+ ),+ (am.LONG, "default"): _page(+ {"a": {"elements": 10, "layout_objects": 0, "cpu_ms": 400, "main_ms": 400}}+ ),+ }+ rows = [+ v+ for v in am.judge(_result(pages), known = "")+ if v["metric"] == "main_ms" and v["kind"] == "length"+ ]+ assert [v["fail"] for v in rows] == [True]++def _stream_result(pages):return {"pages": pages, "modes": {m["name"]: m for m in am.app_modes()}, "areas": ["stream"]}@@ -508,6 +749,14 @@ def test_stream_area_is_last_sends_a_fixed_reply_and_waits_for_its_end():assert am.stream_frames() == frames, "deterministic"+def test_the_stub_health_settles_the_hardware_verdict():+ # An unsettled verdict leaves the Train row spinner running, which made every measured action pay+ # an IntersectionObserver pass over the whole chat on each frame (env.ts capabilitiesUnknown).+ health = am.Api(2).bodies["/api/health"]+ assert health.get("version") and health.get("chat_only") is False+ assert not health.get("hardware_detecting") and not health.get("hardware_detection_deferred")++def test_the_stub_has_a_resident_model_and_saves_the_streamed_turn():api = am.Api(2)assert api.bodies["/api/inference/status"]["active_model"]diff --git a/tests/studio/test_webkit_leg.py b/tests/studio/test_webkit_leg.pynew file mode 100644index 00000000000..d5cee99600e--- /dev/null+++ b/tests/studio/test_webkit_leg.py@@ -0,0 +1,161 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The long-chat gate's WebKit leg: verdicts, known ceilings and rechecks (CI wiring: test_webkit_leg_ci.py)."""++import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parent))++import _area_matrix as am # noqa: E402+import _webkit_leg as wk # noqa: E402++REPO = Path(__file__).resolve().parents[2]+++def _cell(+ open_ms,+ close_ms = 50.0,+ frame = 40.0,+):+ return {+ "status": "OK",+ "reps": 3,+ "open_ms": open_ms,+ "open_frame": frame,+ "close_ms": close_ms,+ "close_frame": frame,+ }+++def _result(+ short,+ long_,+ engine = "webkit_mac",+ mode = "default",+ surface = "browser_pane",+):+ return {+ (engine, mode, wk.SHORT): {"cells": {surface: short}},+ (engine, mode, wk.LONG): {"cells": {surface: long_}},+ }+++ "short, long_, fails",+ [+ (150, 500, True), # Round 11's macOS WebKit pane: 144-165 -> 460-562 ms+ (150, 180, False),+ (60, 190, False), # over 2.5x but under 200 ms+ (300, 700, False), # under 2.5x+ ],+)+def test_open_time_ratio(short, long_, fails):+ rows = [+ v for v in wk.judge(_result(_cell(short), _cell(long_)), known = {}) if v["metric"] == "open"+ ]+ assert [v["fail"] for v in rows] == [fails]+ if fails:+ known = {"webkit_mac:default:browser_pane:open": 4.0}+ assert not [+ v for v in wk.judge(_result(_cell(short), _cell(long_)), known = known) if v["fail"]+ ]+++def test_long_frame_fails_and_can_be_pinned():+ res = _result(_cell(100, frame = 30), _cell(120, frame = 300))+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == [+ "open_frame",+ "close_frame",+ ]+ known = {+ "webkit_mac:default:browser_pane:open_frame": 350,+ "webkit_mac:default:browser_pane:close_frame": 350,+ }+ assert not [v for v in wk.judge(res, known = known) if v["fail"]]+++def test_a_surface_that_stops_running_in_the_long_chat_fails():+ res = _result(_cell(100), {"status": "NOT_RUN: never reached sel"})+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["run"]+++def test_a_long_page_that_does_not_load_fails():+ res = {("webkit_mac", "default", wk.LONG): {"error": "did not load"}}+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["load"]+++def test_a_short_page_that_does_not_load_fails():+ res = {+ ("webkit_mac", "default", wk.SHORT): {"error": "did not load"},+ ("webkit_mac", "default", wk.LONG): {"cells": {"browser_pane": _cell(10_000)}},+ }+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["load"]+++def test_a_surface_without_a_short_baseline_fails():+ res = _result({"status": "NOT_RUN: never reached sel"}, _cell(10_000))+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["run"]+++def test_a_surface_that_runs_at_neither_length_fails():+ res = _result({"status": "NOT_RUN: a"}, {"status": "NOT_RUN: b"})+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["run"]+++def test_a_crashed_engine_worker_fails_every_planned_page(monkeypatch):+ def boom(*args, **kwargs):+ raise RuntimeError("webkit did not launch")++ class _Server:+ def shutdown(self):+ pass++ monkeypatch.setattr(wk, "_engine_worker", boom)+ monkeypatch.setattr(wk.am, "serve_app", lambda dist: (_Server(), "http://x"))+ res = wk.measure(Path("."), modes = ["default"], engines = ["webkit_mac"], log = lambda *a, **k: None)+ assert set(res) == {("webkit_mac", "default", wk.SHORT), ("webkit_mac", "default", wk.LONG)}+ assert all("webkit did not launch" in page["error"] for page in res.values())+ assert [v["metric"] for v in wk.judge(res, known = {}) if v["fail"]] == ["load", "load"]+++def test_short_floor_keeps_instant_surfaces_from_inflating_the_ratio():+ v = [v for v in wk.judge(_result(_cell(5), _cell(210)), known = {}) if v["metric"] == "open"][0]+ assert v["ratio"] == pytest.approx(210 / wk.SHORT_FLOOR_MS)+++def test_a_recheck_only_lowers_readings():+ first = _result(_cell(150), _cell(600, frame = 300))+ again = _result(_cell(140), _cell(400, frame = 320))+ merged = wk.lower(first, again)+ cell = merged[("webkit_mac", "default", wk.LONG)]["cells"]["browser_pane"]+ assert cell["open_ms"] == 400 and cell["open_frame"] == 300+++def test_every_surface_is_a_matrix_area_with_a_timing_predicate():+ for name in wk.SURFACES:+ spec = am.AREAS[name]+ if spec["open"][0] != "scroll":+ wk.pred_js(spec["ready"], 0, "x")+ if spec.get("close"):+ wk.pred_js(spec["gone"], 1, "x")+++def test_known_ceilings_name_their_fix():+ text = (Path(wk.__file__)).read_text(encoding = "utf-8")+ block = text[text.index("KNOWN_DEFAULT = {") : text.index("KNOWN = {")]+ for key in wk.KNOWN_DEFAULT:+ engine, mode, surface, metric = key.split(":")+ assert engine in wk.ENGINES and mode in wk.MODES and surface in wk.SURFACES+ assert metric in ("open", "close", "open_frame", "close_frame")+ if wk.KNOWN_DEFAULT:+ assert "Fix work:" in block+++def test_a_frame_that_is_already_long_in_the_short_chat_is_a_mount_cost_not_a_long_chat_one():+ # WebKit on a CI runner draws a 300-600 ms frame mounting Settings at 1 turn.+ res = _result(_cell(600, frame = 400), _cell(620, frame = 450))+ assert not [v for v in wk.judge(res, known = {}) if v["fail"]]diff --git a/tests/studio/test_webkit_leg_ci.py b/tests/studio/test_webkit_leg_ci.pynew file mode 100644index 00000000000..6af363d2e94--- /dev/null+++ b/tests/studio/test_webkit_leg_ci.py@@ -0,0 +1,36 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The WebKit leg's CI wiring. No harness imports, so the unfiltered workflow-trigger-lint job can run it."""++from pathlib import Path++REPO = Path(__file__).resolve().parents[2]+++def test_frontend_ci_runs_the_webkit_leg_with_webkit_installed():+ ci = (REPO / ".github" / "workflows" / "studio-frontend-ci.yml").read_text(encoding = "utf-8")+ jobs = ci[ci.index("\njobs:") :]+ build = jobs[jobs.index("\n build:") : jobs.index("\n webkit:")]+ webkit = jobs[jobs.index("\n webkit:") : jobs.index("\n windows:")]+ # Its own job, in parallel: the build job's critical path installs no WebKit and runs no leg.+ assert "needs:" not in webkit+ for needle in (+ "install webkit",+ "chromium webkit",+ "p.webkit",+ "LONG_THREAD_WEBKIT",+ "python3 tests/studio/_webkit_leg.py",+ ):+ assert needle not in build+ assert "python3 -m playwright install webkit" in webkit+ assert "vite.long-thread.config.ts" in webkit+ assert "LONG_THREAD_WEBKIT: '1'" in webkit+ assert "python3 tests/studio/_webkit_leg.py" in webkit+ assert "tests/studio/test_webkit_leg.py" in build+ for path in (+ "tests/studio/_webkit_leg.py",+ "tests/studio/test_webkit_leg.py",+ "tests/studio/test_webkit_leg_ci.py",+ ):+ assert ci[: ci.index("\njobs:")].count(f"- '{path}'") == 2