Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Long-chat gate: Firefox leg for dialogs, menus, browser pane and typing (#17361)

Daniel Han56fa8af5136a
diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml
index 21284218b0e..50277d50570 100644
--- a/.github/workflows/studio-frontend-ci.yml
+++ b/.github/workflows/studio-frontend-ci.yml
@@ -49,6 +49,9 @@ on:
- 'tests/studio/_webkit_leg.py'
- 'tests/studio/test_webkit_leg.py'
- 'tests/studio/test_webkit_leg_ci.py'
+ - 'tests/studio/_firefox_leg.py'
+ - 'tests/studio/test_firefox_leg.py'
+ - 'tests/studio/test_firefox_leg_ci.py'
- 'tests/studio/_appearance_coverage.py'
- 'tests/studio/test_appearance_coverage.py'
- 'tests/studio/_has_lint.py'
@@ -111,6 +114,9 @@ on:
- 'tests/studio/_webkit_leg.py'
- 'tests/studio/test_webkit_leg.py'
- 'tests/studio/test_webkit_leg_ci.py'
+ - 'tests/studio/_firefox_leg.py'
+ - 'tests/studio/test_firefox_leg.py'
+ - 'tests/studio/test_firefox_leg_ci.py'
- 'tests/studio/_appearance_coverage.py'
- 'tests/studio/test_appearance_coverage.py'
- 'tests/studio/_has_lint.py'
@@ -439,6 +445,8 @@ jobs:
tests/studio/test_area_matrix.py \
tests/studio/test_webkit_leg.py \
tests/studio/test_webkit_leg_ci.py \
+ tests/studio/test_firefox_leg.py \
+ tests/studio/test_firefox_leg_ci.py \
tests/studio/test_appearance_coverage.py \
tests/studio/test_has_lint.py -q
@@ -759,6 +767,72 @@ jobs:
LONG_THREAD_WEBKIT: '1'
run: python3 tests/studio/_webkit_leg.py
+ # Firefox leg of the long-chat gate (tests/studio/_firefox_leg.py: the WebKit leg's timing in
+ # Playwright Firefox), in parallel like the WebKit job. Times opening and closing Settings, the
+ # composer and tools menus, chat options, the palette, search, the browser pane and typing at 1 and
+ # 150 turns, default look and Liquid 3D: fails when the long chat is over 2.5x the short one and over
+ # 200 ms, or has a frame over 250 ms and over 2.5x the short chat's (a dialog or menu used to cost
+ # Firefox about 250 ms more at 150 turns, which the Chromium and WebKit legs could not see).
+ firefox:
+ name: Long-chat gate (Firefox)
+ runs-on: ubuntu-latest
+ env:
+ UNSLOTH_DISABLE_UPDATE_CHECK: '1'
+ timeout-minutes: 40
+ defaults:
+ run:
+ working-directory: studio/frontend
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
+ with:
+ node-version: '22'
+
+ - name: Upgrade npm to 11.x (allowScripts enforcement)
+ working-directory: ${{ github.workspace }}
+ run: |
+ npm install -g npm@^11 --no-fund --no-audit
+ V=$(npm -v)
+ case "$V" in
+ 11.1[6-9].*|11.[2-9][0-9].*|1[2-9].*) echo "npm $V has allowScripts" ;;
+ *) echo "::error::npm $V lacks allowScripts (need >=11.16)"; exit 1 ;;
+ esac
+
+ # Must run BEFORE npm ci, where a compromised tarball's install scripts run.
+ - name: Lockfile supply-chain audit (pre-install scan)
+ working-directory: ${{ github.workspace }}
+ run: python3 scripts/lockfile_supply_chain_audit.py
+
+ - name: Install dependencies
+ run: npm ci --strict-allow-scripts --no-fund --no-audit
+
+ - name: Firefox for the long-chat gate
+ working-directory: ${{ github.workspace }}
+ shell: bash
+ timeout-minutes: 10
+ env:
+ APT_ACQUIRE_RETRIES: '0'
+ run: |
+ python3 -m pip install 'playwright>=1.45,<2'
+ python3 -m playwright install firefox
+ if ! python3 -c "from playwright.sync_api import sync_playwright as s; p = s().start(); p.firefox.launch().close(); p.stop()"; then
+ bash .github/scripts/retry-with-apt-lock.sh python3 -m playwright install-deps firefox
+ fi
+
+ - name: Long-chat production build
+ run: npx vite build --config vite.long-thread.config.ts
+
+ - name: Browser gate for long-chat cost in Firefox
+ working-directory: ${{ github.workspace }}
+ shell: bash
+ timeout-minutes: 20
+ env:
+ LONG_THREAD_FIREFOX: '1'
+ run: python3 tests/studio/_firefox_leg.py
+
# The suite reads sources and resolves module URLs, so it can break only on Windows.
# .gitattributes pins LF for studio/frontend, so exact-substring assertions hold.
windows:
diff --git a/.github/workflows/workflow-trigger-lint.yml b/.github/workflows/workflow-trigger-lint.yml
index 8bb5ad37223..9bd2417e4e7 100644
--- a/.github/workflows/workflow-trigger-lint.yml
+++ b/.github/workflows/workflow-trigger-lint.yml
@@ -76,6 +76,7 @@ jobs:
python3 -m pytest -q -n 4 \
tests/studio/test_absorbed_lanes_still_run.py \
tests/studio/test_webkit_leg_ci.py \
+ tests/studio/test_firefox_leg_ci.py \
tests/studio/test_appimage_toolchain_contract.py \
tests/studio/test_lint_ci_scans_committed_files.py \
tests/studio/test_agent_guides_verdicts.py \
diff --git a/tests/studio/_firefox_leg.py b/tests/studio/_firefox_leg.py
new file mode 100644
index 00000000000..704295d1800
--- /dev/null
+++ b/tests/studio/_firefox_leg.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Firefox leg of the long-chat gate: open/close time and longest frame per surface, short vs long chat.
+
+The same leg as _webkit_leg.py (in-page timing from just before the trigger to the first frame with the
+surface visible, the longest rAF frame inside that window, median of REPS per page, failing cells
+measured again on fresh pages, a crashed worker or a surface that stops running fails), run in
+Playwright's Firefox. Firefox lays out and paints differently: opening a dialog or menu used to cost it
+about 250 ms more at 150 turns (the modal lock's thread cover re-laid out the whole thread, fixed by a
+Firefox-only rule in index.css), a cost the Chromium matrix and the WebKit leg could not see.
+
+Surfaces: the modal ones (Radix dialogs and modal menus), the browser pane and typing (a "settle"
+surface: the last content mutation after typing four characters). Limits as in the WebKit leg, set from
+main readings with margin (LONG_THREAD_FIREFOX_RATIO / _FLOOR_MS / _FRAME_MS / _FRAME_GROWTH / _KNOWN).
+"""
+
+from __future__ import annotations
+
+import os
+
+import _area_matrix as am
+import _webkit_leg as wk
+
+SURFACES = [
+ "settings",
+ "composer_menu",
+ "tools_menu",
+ "chat_options",
+ "command_palette",
+ "search_chats",
+ "browser_pane",
+ "type",
+]
+
+# Known Firefox long-chat costs on main, pinned so the gate holds today's level (engine:mode:surface:
+# metric, as in the WebKit leg). None today.
+KNOWN_DEFAULT: dict = {}
+
+FIREFOX = wk.Leg(
+ name = "Firefox",
+ browser = "firefox",
+ env = "LONG_THREAD_FIREFOX",
+ engines = {"firefox": {}},
+ surfaces = SURFACES,
+ known = wk.leg_known("LONG_THREAD_FIREFOX", KNOWN_DEFAULT),
+ who = "Firefox",
+ modes = ("default", am.GLASS),
+ short = int(os.environ.get("LONG_THREAD_FIREFOX_SHORT", "1")),
+ long = int(os.environ.get("LONG_THREAD_FIREFOX_LONG", "150")),
+ reps = int(os.environ.get("LONG_THREAD_FIREFOX_REPS", "3")),
+ ratio = float(os.environ.get("LONG_THREAD_FIREFOX_RATIO", "2.5")),
+ floor_ms = float(os.environ.get("LONG_THREAD_FIREFOX_FLOOR_MS", "200")),
+ frame_ms = float(os.environ.get("LONG_THREAD_FIREFOX_FRAME_MS", "250")),
+ frame_growth = float(os.environ.get("LONG_THREAD_FIREFOX_FRAME_GROWTH", "2.5")),
+)
+
+
+def judge(result: dict, known: dict | None = None) -> list[dict]:
+ return wk.judge(result, known = known, leg = FIREFOX)
+
+
+def measure(dist, **kwargs) -> dict:
+ return wk.measure(dist, leg = FIREFOX, **kwargs)
+
+
+def run(dist, log = print):
+ return wk.run(dist, log = log, leg = FIREFOX)
+
+
+def main() -> int:
+ """Frontend CI's Firefox job: the leg alone, on the long-thread production build."""
+ return wk.main(FIREFOX)
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/studio/_webkit_leg.py b/tests/studio/_webkit_leg.py
index af71ffce84c..6afbd621288 100644
--- a/tests/studio/_webkit_leg.py
+++ b/tests/studio/_webkit_leg.py
@@ -27,6 +27,7 @@ import os
import statistics
import threading
import time
+from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlparse
@@ -109,6 +110,54 @@ KNOWN = {
},
}
+
+@dataclass
+class Leg:
+ """One browser engine family's leg: what it runs, its limits and how its failures read. The WebKit
+ leg is WEBKIT below; _firefox_leg.py builds FIREFOX the same way."""
+
+ name: str # in reports: "WebKit"
+ browser: str # Playwright browser type: "webkit", "firefox"
+ env: str # environment prefix: LONG_THREAD_WEBKIT (=1 runs it, _KNOWN, _REPS, ...)
+ engines: dict # engine name -> {"user_agent": ..., "platform": ...} (empty: the browser's own)
+ surfaces: list
+ known: dict
+ who: str # who pays a failure: "WebKit (Safari, the macOS app, the Linux desktop app)"
+ modes: tuple = MODES
+ short: int = SHORT
+ long: int = LONG
+ reps: int = REPS
+ ratio: float = RATIO
+ floor_ms: float = FLOOR_MS
+ frame_ms: float = FRAME_MS
+ frame_growth: float = FRAME_GROWTH
+
+
+def leg_known(env: str, default: dict) -> dict:
+ """KNOWN ceilings: the pinned defaults plus <env>_KNOWN=engine:mode:surface:metric=value,..."""
+ return {
+ **default,
+ **{
+ k: float(v)
+ for k, _, v in (
+ p.strip().partition("=")
+ for p in os.environ.get(f"{env}_KNOWN", "").split(",")
+ if p.strip()
+ )
+ },
+ }
+
+
+WEBKIT = Leg(
+ name = "WebKit",
+ browser = "webkit",
+ env = "LONG_THREAD_WEBKIT",
+ engines = ENGINES,
+ surfaces = SURFACES,
+ known = KNOWN,
+ who = "WebKit (Safari, the macOS app, the Linux desktop app)",
+)
+
ARM = """() => {
const w = window.__wk = {t0: performance.now(), frames: [], last: performance.now(), stop: false};
const tick = (t) => { w.frames.push([t, t - w.last]); w.last = t; if (!w.stop) requestAnimationFrame(tick); };
@@ -124,6 +173,30 @@ UNTIL = """async ([pred, timeoutMs]) => {
const long = w.frames.filter(([ts]) => ts <= t + 1).reduce((m, [, d]) => Math.max(m, d), 0);
return [t - w.t0, long];
}"""
+# Surfaces judged on "settle" (typing): the frame clock plus a MutationObserver armed before the action,
+# ignoring the autoscroll follow flag and the Glass motion hold flags (state, not content). The time
+# is the last content mutation, once the page has been quiet for QUIET_MS.
+QUIET_MS = 300
+SETTLE_IGNORED = ["data-aui-following", "data-glass-moving", "data-glass-settling"]
+ARM_SETTLE = """(ignored) => {
+ const w = window.__wk = {t0: performance.now(), frames: [], last: performance.now(), stop: false, lastMut: null};
+ const tick = (t) => { w.frames.push([t, t - w.last]); w.last = t; if (!w.stop) requestAnimationFrame(tick); };
+ requestAnimationFrame(tick);
+ w.mo = new MutationObserver((recs) => { for (const r of recs) {
+ if (r.type === 'attributes' && ignored.includes(r.attributeName)) continue; w.lastMut = performance.now(); } });
+ w.mo.observe(document.documentElement, {subtree: true, childList: true, attributes: true, characterData: true});
+}"""
+SETTLED = """async ([quietMs, timeoutMs]) => {
+ const w = window.__wk; const frame = () => new Promise((r) => requestAnimationFrame(r));
+ while (performance.now() - (w.lastMut ?? w.t0) < quietMs) {
+ if (performance.now() - w.t0 > timeoutMs) { w.mo.disconnect(); w.stop = true; return null; }
+ await frame();
+ }
+ w.mo.disconnect(); w.stop = true; const end = w.lastMut ?? w.t0;
+ // Frames that started by the last mutation, so the one rendering it counts even when it ends long after.
+ const long = w.frames.filter(([ts, d]) => ts - d <= end + 20).reduce((m, [, d]) => Math.max(m, d), 0);
+ return [end - w.t0, long];
+}"""
# Thread scroll: the matrix's 20-step scroll; the time is the whole scroll, the frame its longest.
SCROLLED = """() => { const w = window.__wk; w.stop = true; const t = performance.now();
return [t - w.t0, w.frames.reduce((m, [, d]) => Math.max(m, d), 0)]; }"""
@@ -150,19 +223,26 @@ def pred_js(pred, base_overlays: int, attr_before) -> str:
class WebKitPage(am.Page):
- """The matrix's page (same thread, API and controls) in WebKit, without Chromium tracing."""
-
- def __init__(self, browser, base: str, turns: int, mode: dict, engine: str):
+ """The matrix's page (same thread, API and controls) in a non-Chromium browser, without tracing."""
+
+ def __init__(
+ self,
+ browser,
+ base: str,
+ turns: int,
+ mode: dict,
+ engine: str,
+ engines = None,
+ ):
self.browser, self.turns, self.mode, self.tabs = browser, turns, mode, []
self.api = am.Api(turns)
- extra = (
- {"user_agent": ENGINES[engine]["user_agent"]} if "user_agent" in ENGINES[engine] else {}
- )
+ spec = (engines or ENGINES).get(engine, {})
+ extra = {"user_agent": spec["user_agent"]} if "user_agent" in spec else {}
self.ctx = browser.new_context(viewport = {"width": 1440, "height": 900}, **extra)
- if ENGINES[engine].get("platform"):
+ if spec.get("platform"):
self.ctx.add_init_script(
"Object.defineProperty(Navigator.prototype, 'platform', {get: () => %s});"
- % json.dumps(ENGINES[engine]["platform"])
+ % json.dumps(spec["platform"])
)
init = {
"unsloth_auth_token": am.auth_token(),
@@ -195,12 +275,18 @@ class WebKitPage(am.Page):
page.mouse.move(1435, 450)
def timed(self, action, pred, base_overlays: int):
- self.page.evaluate(ARM)
+ settle = pred is not None and pred[0] == "settle"
+ if settle:
+ self.page.evaluate(ARM_SETTLE, SETTLE_IGNORED)
+ else:
+ self.page.evaluate(ARM)
if not self.act(action):
- self.page.evaluate("() => { window.__wk.stop = true; }")
+ self.page.evaluate("() => { window.__wk.stop = true; window.__wk.mo?.disconnect(); }")
return None
if pred is None:
return self.page.evaluate(SCROLLED)
+ if settle:
+ return self.page.evaluate(SETTLED, [QUIET_MS, TIMEOUT_MS])
return self.page.evaluate(
UNTIL, [pred_js(pred, base_overlays, self.attr_before), TIMEOUT_MS]
)
@@ -211,6 +297,10 @@ class WebKitPage(am.Page):
page = self.page
self.reset()
page.wait_for_timeout(SETTLE_MS)
+ if spec.get("setup"): # typing: focus the composer first, untimed
+ if not self.act(spec["setup"]):
+ return {"error": "setup did not run"}
+ page.wait_for_timeout(SETTLE_MS)
base = page.evaluate("() => window.__gateOverlays()")
ready = spec["ready"]
self.attr_before = (
@@ -258,16 +348,26 @@ class WebKitPage(am.Page):
return cells
-def _engine_worker(engine, dist_base, modes, surfaces, reps, out, log) -> None:
+def _engine_worker(
+ engine,
+ dist_base,
+ modes,
+ surfaces,
+ reps,
+ out,
+ log,
+ leg = None,
+) -> None:
from playwright.sync_api import sync_playwright
+ leg = leg or WEBKIT
with sync_playwright() as pw:
- browser = pw.webkit.launch()
+ browser = getattr(pw, leg.browser).launch()
try:
for mode in modes:
- for turns in (SHORT, LONG):
+ for turns in (leg.short, leg.long):
t = time.time()
try:
- page = WebKitPage(browser, dist_base, turns, mode, engine)
+ page = WebKitPage(browser, dist_base, turns, mode, engine, leg.engines)
except Exception as exc: # noqa: BLE001
out[(engine, mode["name"], turns)] = {
"error": f"did not load: {str(exc)[:160]}"
@@ -278,7 +378,7 @@ def _engine_worker(engine, dist_base, modes, surfaces, reps, out, log) -> None:
finally:
page.close()
log(
- f" webkit: {engine} {mode['name']} {turns} turns ({time.time() - t:.0f} s)",
+ f" {leg.name.lower()}: {engine} {mode['name']} {turns} turns ({time.time() - t:.0f} s)",
flush = True,
)
finally:
@@ -288,12 +388,16 @@ def _engine_worker(engine, dist_base, modes, surfaces, reps, out, log) -> None:
def measure(
dist: Path,
surfaces = None,
- modes = MODES,
+ modes = None,
engines = None,
- reps = REPS,
+ reps = None,
log = print,
+ leg = None,
) -> dict:
- surfaces = surfaces or SURFACES
+ leg = leg or WEBKIT
+ surfaces = surfaces or leg.surfaces
+ modes = modes or leg.modes
+ reps = reps or leg.reps
by_name = {m["name"]: m for m in am.app_modes()}
chosen = [by_name[m] for m in modes]
server, base = am.serve_app(dist)
@@ -304,11 +408,11 @@ def measure(
def work(engine):
try:
- _engine_worker(engine, base, chosen, surfaces, reps, out, log)
+ _engine_worker(engine, base, chosen, surfaces, reps, out, log, leg = leg)
except Exception as exc: # noqa: BLE001 (recorded below: a crash must never read as a pass)
crashed[engine] = f"{type(exc).__name__}: {str(exc)[:160]}"
- engines = engines or ENGINES
+ engines = engines or leg.engines
threads = [threading.Thread(target = work, args = (e,)) for e in engines]
for t in threads:
t.start()
@@ -319,7 +423,7 @@ def measure(
# Every planned page gets a result: one a worker never recorded is a load failure.
for e in engines:
for m in chosen:
- for turns in (SHORT, LONG):
+ for turns in (leg.short, leg.long):
out.setdefault(
(e, m["name"], turns),
{
@@ -329,8 +433,14 @@ def measure(
return out
-def judge(result: dict, known: dict | None = None) -> list[dict]:
- known = KNOWN if known is None else known
+def judge(
+ result: dict,
+ known: dict | None = None,
+ leg = None,
+) -> list[dict]:
+ leg = leg or WEBKIT
+ known = leg.known if known is None else known
+ SHORT, LONG = leg.short, leg.long # noqa: N806 - the leg's lengths
out = []
for (engine, mode, turns), page in sorted(result.items()):
if "error" in page:
@@ -371,7 +481,7 @@ def judge(result: dict, known: dict | None = None) -> list[dict]:
if lv is None or sv is None:
continue
ratio = lv / max(sv, SHORT_FLOOR_MS)
- limit = known.get(f"{engine}:{mode}:{name}:{edge}", RATIO)
+ limit = known.get(f"{engine}:{mode}:{name}:{edge}", leg.ratio)
out.append(
{
"engine": engine,
@@ -382,12 +492,12 @@ def judge(result: dict, known: dict | None = None) -> list[dict]:
"base_value": sv,
"ratio": ratio,
"limit": limit,
- "fail": ratio > limit and lv > FLOOR_MS,
+ "fail": ratio > limit and lv > leg.floor_ms,
}
)
fv = lc.get(f"{edge}_frame")
if fv is not None:
- flimit = known.get(f"{engine}:{mode}:{name}:{edge}_frame", FRAME_MS)
+ flimit = known.get(f"{engine}:{mode}:{name}:{edge}_frame", leg.frame_ms)
sf = sc.get(f"{edge}_frame") or 0.0
out.append(
{
@@ -398,7 +508,7 @@ def judge(result: dict, known: dict | None = None) -> list[dict]:
"value": fv,
"base_value": sf,
"limit": flimit,
- "fail": fv > flimit and fv > FRAME_GROWTH * sf,
+ "fail": fv > flimit and fv > leg.frame_growth * sf,
}
)
return out
@@ -433,11 +543,17 @@ def lower(first: dict, again: dict) -> dict:
return merged
-def run(dist: Path, log = print) -> tuple[list[str], list[str], dict]:
+def run(
+ dist: Path,
+ log = print,
+ leg = None,
+) -> tuple[list[str], list[str], dict]:
"""(report lines, failures, result)."""
+ leg = leg or WEBKIT
+ SHORT, LONG = leg.short, leg.long # noqa: N806 - the leg's lengths
started = time.time()
- result = measure(dist, log = log)
- verdicts = judge(result)
+ result = measure(dist, log = log, leg = leg)
+ verdicts = judge(result, leg = leg)
failing = {
(v["engine"], v["mode"], v["surface"])
for v in verdicts
@@ -445,18 +561,20 @@ def run(dist: Path, log = print) -> tuple[list[str], list[str], dict]:
}
if failing:
log(
- f" webkit: {len(failing)} cell(s) over their limit; measuring them again on fresh pages",
+ f" {leg.name.lower()}: {len(failing)} cell(s) over their limit; measuring them again on fresh pages",
flush = True,
)
for engine in sorted({f[0] for f in failing}):
mode_names = sorted({f[1] for f in failing if f[0] == engine})
surfaces = sorted({f[2] for f in failing if f[0] == engine})
- again = measure(dist, surfaces = surfaces, modes = mode_names, engines = [engine], log = log)
+ again = measure(
+ dist, surfaces = surfaces, modes = mode_names, engines = [engine], log = log, leg = leg
+ )
result = lower(result, again)
- verdicts = judge(result)
- lines = table(result, verdicts)
+ verdicts = judge(result, leg = leg)
+ lines = table(result, verdicts, leg = leg)
lines.append(
- f" webkit leg: {len(ENGINES)} engines x {len(MODES)} appearances x {SHORT}/{LONG} turns, {time.time() - started:.0f} s"
+ f" {leg.name.lower()} leg: {len(leg.engines)} engine(s) x {len(leg.modes)} appearances x {SHORT}/{LONG} turns, {time.time() - started:.0f} s"
)
failures = []
for v in verdicts:
@@ -464,27 +582,33 @@ def run(dist: Path, log = print) -> tuple[list[str], list[str], dict]:
continue
where = f"{v['engine']}, {v['mode']}"
if v["metric"] in ("load", "run"):
- failures.append(f"WebKit {where}: {v['surface']} {v['why']}")
+ failures.append(f"{leg.name} {where}: {v['surface']} {v['why']}")
elif v["metric"].endswith("frame"):
failures.append(
- f"WebKit {where}: {v['surface']} ({v['metric'].replace('_', ' ')}) has a {v['value']:.0f} ms frame at "
- f"{LONG} turns against {v['base_value']:.0f} ms at {SHORT} (limit {v['limit']:.0f} ms and {FRAME_GROWTH:g}x; LONG_THREAD_WEBKIT_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...)"
+ f"{leg.name} {where}: {v['surface']} ({v['metric'].replace('_', ' ')}) has a {v['value']:.0f} ms frame at "
+ f"{LONG} turns against {v['base_value']:.0f} ms at {SHORT} (limit {v['limit']:.0f} ms and {leg.frame_growth:g}x; {leg.env}_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...)"
)
else:
failures.append(
- f"WebKit {where}: {'opening' if v['metric'] == 'open' else 'closing'} {v['surface']} takes {v['base_value']:.0f} ms at {SHORT} turns -> "
- f"{v['value']:.0f} ms at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x over {FLOOR_MS:.0f} ms; "
- f"LONG_THREAD_WEBKIT_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...). It costs more "
- "as the chat grows in WebKit (Safari, the macOS app, the Linux desktop app): usually re-layout or "
+ f"{leg.name} {where}: {'opening' if v['metric'] == 'open' else 'closing'} {v['surface']} takes {v['base_value']:.0f} ms at {SHORT} turns -> "
+ f"{v['value']:.0f} ms at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x over {leg.floor_ms:.0f} ms; "
+ f"{leg.env}_KNOWN {v['engine']}:{v['mode']}:{v['surface']}:{v['metric']}=...). It costs more "
+ f"as the chat grows in {leg.who}: usually re-layout or "
"repaint of the whole thread behind the surface"
)
return lines, failures, {"result": result, "verdicts": verdicts}
-def table(result: dict, verdicts: list[dict]) -> list[str]:
+def table(
+ result: dict,
+ verdicts: list[dict],
+ leg = None,
+) -> list[str]:
+ leg = leg or WEBKIT
+ SHORT, LONG = leg.short, leg.long # noqa: N806 - the leg's lengths
by = {(v["engine"], v["mode"], v["surface"], v["metric"]): v for v in verdicts}
lines = [
- f" WebKit: open / close ms at {SHORT} -> {LONG} turns, longest frame at {LONG} (! = over its limit)"
+ f" {leg.name}: open / close ms at {SHORT} -> {LONG} turns, longest frame at {LONG} (! = over its limit)"
]
for (engine, mode, turns), page in sorted(result.items()):
if turns != LONG:
@@ -513,24 +637,25 @@ def table(result: dict, verdicts: list[dict]) -> list[str]:
return lines
-def main() -> int:
- """Frontend CI's WebKit job: the leg alone, on the long-thread production build."""
- if os.environ.get("LONG_THREAD_WEBKIT", "0") != "1":
- print("WebKit leg: off (set LONG_THREAD_WEBKIT=1)")
+def main(leg = None) -> int:
+ """Frontend CI's WebKit (or Firefox) job: the leg alone, on the long-thread production build."""
+ leg = leg or WEBKIT
+ if os.environ.get(leg.env, "0") != "1":
+ print(f"{leg.name} leg: off (set {leg.env}=1)")
return 0
repo = Path(__file__).resolve().parents[2]
dist = Path(
os.environ.get("LONG_THREAD_DIST", repo / ".playwright-cli" / "long-thread-production")
)
if not (dist / "index.html").exists():
- print(f"FAIL: no app build at {dist / 'index.html'} for the WebKit leg")
+ print(f"FAIL: no app build at {dist / 'index.html'} for the {leg.name} leg")
return 1
- lines, fails, _ = run(dist)
+ lines, fails, _ = run(dist, leg = leg)
for line in lines:
print(line)
for f in fails:
print(f"FAIL: {f}")
- print("WebKit leg: " + (f"{len(fails)} failure(s)" if fails else "PASS"))
+ print(f"{leg.name} leg: " + (f"{len(fails)} failure(s)" if fails else "PASS"))
return 1 if fails else 0
diff --git a/tests/studio/test_firefox_leg.py b/tests/studio/test_firefox_leg.py
new file mode 100644
index 00000000000..9ff8526744c
--- /dev/null
+++ b/tests/studio/test_firefox_leg.py
@@ -0,0 +1,169 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The long-chat gate's Firefox leg: its limits, surfaces and fail-closed runs (CI wiring: test_firefox_leg_ci.py)."""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import _area_matrix as am # noqa: E402
+import _firefox_leg as ff # noqa: E402
+import _webkit_leg as wk # noqa: E402
+
+LEG = ff.FIREFOX
+
+
+def _cell(
+ open_ms,
+ close_ms = 60.0,
+ frame = 40.0,
+ close_frame = None,
+):
+ return {
+ "status": "OK",
+ "reps": 3,
+ "open_ms": open_ms,
+ "open_frame": frame,
+ "close_ms": close_ms,
+ "close_frame": frame if close_frame is None else close_frame,
+ }
+
+
+def _result(
+ short,
+ long_,
+ mode = "default",
+ surface = "composer_menu",
+):
+ return {
+ ("firefox", mode, LEG.short): {"cells": {surface: short}},
+ ("firefox", mode, LEG.long): {"cells": {surface: long_}},
+ }
+
+
+def _fails(res, known = None):
+ return [
+ (v["surface"], v["metric"])
+ for v in ff.judge(res, known = {} if known is None else known)
+ if v["fail"]
+ ]
+
+
+def test_the_leg_runs_firefox_on_its_own_engine_with_its_own_switch():
+ assert LEG.browser == "firefox" and list(LEG.engines) == ["firefox"]
+ assert LEG.env == "LONG_THREAD_FIREFOX" and LEG.name == "Firefox"
+ assert LEG.modes == ("default", am.GLASS)
+
+
+def test_the_modal_cover_regression_fails():
+ # Main before the Firefox-only cover rule, composer menu at 150 turns: 349-425 ms open with a
+ # 160-340 ms frame, against about 110 ms and a 40 ms frame at 1 turn.
+ res = _result(_cell(110, close_ms = 120, frame = 40), _cell(380, close_ms = 390, frame = 280))
+ assert set(_fails(res)) >= {("composer_menu", "open"), ("composer_menu", "open_frame")}
+
+
+def test_the_fixed_reading_passes():
+ # With the cover rule: 133-250 ms open at 150 turns, frames 33-130 ms.
+ res = _result(_cell(110, close_ms = 120, frame = 40), _cell(185, close_ms = 200, frame = 110))
+ assert not _fails(res)
+
+
+ "short, long_, fails",
+ [
+ (110, 380, True),
+ (110, 260, False), # under 2.5x
+ (60, 190, False), # over 2.5x but under 200 ms
+ ],
+)
+def test_open_time_rule(short, long_, fails):
+ rows = [
+ v for v in ff.judge(_result(_cell(short), _cell(long_)), known = {}) if v["metric"] == "open"
+ ]
+ assert [v["fail"] for v in rows] == [fails]
+
+
+def test_a_known_ceiling_holds_a_cost_and_fails_anything_worse():
+ res = _result(_cell(110), _cell(380))
+ key = "firefox:default:composer_menu:open"
+ assert not [f for f in _fails(res, {key: 4.0}) if f[1] == "open"]
+ assert ("composer_menu", "open") in _fails(_result(_cell(110), _cell(500)), {key: 4.0})
+
+
+def test_load_and_run_failures_fail_closed():
+ assert _fails({("firefox", "default", LEG.long): {"error": "did not load"}}) == [("*", "load")]
+ assert _fails(_result(_cell(100), {"status": "NOT_RUN: never reached overlay"})) == [
+ ("composer_menu", "run")
+ ]
+
+
+def test_a_crashed_firefox_worker_fails_every_planned_page(monkeypatch):
+ def boom(*args, **kwargs):
+ raise RuntimeError("firefox did not launch")
+
+ class _Server:
+ def shutdown(self):
+ pass
+
+ monkeypatch.setattr(wk, "_engine_worker", boom)
+ monkeypatch.setattr(wk.am, "serve_app", lambda dist: (_Server(), "http://x"))
+ res = ff.measure(Path("."), modes = ["default"], log = lambda *a, **k: None)
+ assert set(res) == {("firefox", "default", LEG.short), ("firefox", "default", LEG.long)}
+ assert all("firefox did not launch" in page["error"] for page in res.values())
+ assert _fails(res) == [("*", "load"), ("*", "load")]
+
+
+def test_every_surface_is_a_matrix_area_the_leg_can_time():
+ for name in LEG.surfaces:
+ spec = am.AREAS[name]
+ for pred in (spec["ready"], spec.get("gone")):
+ if pred is None or pred[0] == "settle":
+ continue # settle: timed by the MutationObserver window, no predicate
+ wk.pred_js(pred, 1, "x")
+ # The modal surfaces the cover regression hit are all measured.
+ assert {
+ "settings",
+ "composer_menu",
+ "tools_menu",
+ "chat_options",
+ "command_palette",
+ "search_chats",
+ } <= set(LEG.surfaces)
+ assert "browser_pane" in LEG.surfaces and "type" in LEG.surfaces
+
+
+def test_settle_surfaces_ignore_state_flags_that_are_not_content():
+ for flag in ("data-aui-following", "data-glass-moving", "data-glass-settling"):
+ assert flag in wk.SETTLE_IGNORED
+
+
+def test_the_webkit_leg_keeps_its_own_limits():
+ assert wk.WEBKIT.browser == "webkit" and wk.WEBKIT.engines is wk.ENGINES
+ assert wk.WEBKIT.ratio == wk.RATIO and wk.WEBKIT.known is wk.KNOWN
+ assert "type" not in wk.WEBKIT.surfaces
+
+
+def test_settled_counts_the_frame_that_renders_the_last_mutation():
+ # A slow layout / paint after the final typing mutation ends its frame long after the mutation; that
+ # frame must still be the "longest frame" (last mutation at 160 ms, a 316 ms frame ending at 460 ms).
+ import shutil
+ import subprocess
+
+ node = shutil.which("node")
+ if node is None:
+ pytest.skip("node not available")
+ script = (
+ "globalThis.performance = {now: () => 10000};"
+ "globalThis.requestAnimationFrame = (cb) => setTimeout(() => cb(10000), 0);"
+ "globalThis.window = {__wk: {t0: 0, lastMut: 160, stop: false, mo: {disconnect() {}},"
+ " frames: [[128, 16], [144, 16], [460, 316], [476, 16]]}};"
+ f"const settled = ({wk.SETTLED});"
+ "settled([300, 60000]).then((r) => console.log(JSON.stringify(r)));"
+ )
+ out = subprocess.run([node, "-e", script], capture_output = True, text = True, check = True).stdout
+ assert json.loads(out) == [160, 316]
diff --git a/tests/studio/test_firefox_leg_ci.py b/tests/studio/test_firefox_leg_ci.py
new file mode 100644
index 00000000000..3dbb5298902
--- /dev/null
+++ b/tests/studio/test_firefox_leg_ci.py
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The Firefox leg's CI wiring. No harness imports, so the unfiltered workflow-trigger-lint job can run it."""
+
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+
+
+def test_frontend_ci_runs_the_firefox_leg_in_its_own_job():
+ ci = (REPO / ".github" / "workflows" / "studio-frontend-ci.yml").read_text(encoding = "utf-8")
+ jobs = ci[ci.index("\njobs:") :]
+ build = jobs[jobs.index("\n build:") : jobs.index("\n webkit:")]
+ firefox = jobs[jobs.index("\n firefox:") : jobs.index("\n windows:")]
+ # In parallel with the build job: none of its install or run sits on that job's critical path.
+ assert "needs:" not in firefox
+ for needle in (
+ "LONG_THREAD_FIREFOX",
+ "python3 tests/studio/_firefox_leg.py",
+ "p.firefox.launch",
+ ):
+ assert needle not in build
+ assert "python3 -m playwright install firefox" in firefox
+ assert "vite.long-thread.config.ts" in firefox
+ assert "LONG_THREAD_FIREFOX: '1'" in firefox
+ assert "python3 tests/studio/_firefox_leg.py" in firefox
+ assert "tests/studio/test_firefox_leg.py" in build
+ for path in (
+ "tests/studio/_firefox_leg.py",
+ "tests/studio/test_firefox_leg.py",
+ "tests/studio/test_firefox_leg_ci.py",
+ ):
+ assert ci[: ci.index("\njobs:")].count(f"- '{path}'") == 2
+
+
+def test_the_wiring_test_runs_in_the_unfiltered_trigger_lint_job():
+ lint = (REPO / ".github" / "workflows" / "workflow-trigger-lint.yml").read_text(
+ encoding = "utf-8"
+ )
+ assert "tests/studio/test_firefox_leg_ci.py" in lint