Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Stall and frame-time curves over thread content size; measures only and never gates on timing."""
from __future__ import annotations
import json
import os
import re
import sys
from pathlib import Path
from playwright.sync_api import sync_playwright
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _playwright_robust import ( # noqa: E402
chromium_launch_args,
start_vite,
stop_process,
wait_for_smoke_page,
)
PORT = int(os.environ.get("SMOKE_PORT", "5215"))
# rstrip("/"): a trailing slash makes the anchored /api/ route regex never match, hitting live HTTP.
_EXTERNAL = os.environ.get("SMOKE_BASE_URL", "").strip().rstrip("/")
BASE = _EXTERNAL or f"http://127.0.0.1:{PORT}"
OWNS_SERVER = not _EXTERNAL
LABEL = os.environ.get("SMOKE_LABEL", "tree")
OUT = Path(os.environ.get("PW_ART_DIR", "logs/playwright-heavy-thread"))
OUT.mkdir(parents = True, exist_ok = True)
# Sorted: the growth check reads the first and last entries as smallest and largest.
SIZES = sorted(
int(n) for n in os.environ.get("SMOKE_HEAVY_CHARS", "25000,100000,300000").split(",")
)
ENGINES = [
e.strip()
for e in os.environ.get("SMOKE_HEAVY_ENGINES", "chromium,webkit,firefox").split(",")
if e.strip()
]
# Median, because the first repetition is systematically slowest (cold Shiki cache, cold layout).
REPEATS = int(os.environ.get("SMOKE_HEAVY_REPEATS", "3"))
# Throttling is CDP-only, so enabling it makes the chromium column incomparable.
CPU_THROTTLE_RATE = float(os.environ.get("SMOKE_CPU_THROTTLE", "1"))
KEYSTROKES = int(os.environ.get("SMOKE_KEYSTROKES", "5"))
SCROLL_STEPS = int(os.environ.get("SMOKE_SCROLL_STEPS", "20"))
SCROLL_STEP_PX = int(os.environ.get("SMOKE_SCROLL_STEP_PX", "400"))
SEED_TIMEOUT_MS = int(os.environ.get("SMOKE_SEED_TIMEOUT_MS", "300000"))
ACTION_TIMEOUT_MS = int(os.environ.get("SMOKE_ACTION_TIMEOUT_MS", "120000"))
SETTLE_TIMEOUT_MS = int(os.environ.get("SMOKE_SETTLE_TIMEOUT_MS", "120000"))
# Four 250ms polls: a gate releasing after one lull released mid-rebuild. Not charged to the action.
HIGHLIGHT_GRACE_MS = int(os.environ.get("SMOKE_HIGHLIGHT_GRACE_MS", "1000"))
# Counting tokens is O(nodes), so per frame it would cost time inside the timed region.
HIGHLIGHT_PROBE_MS = int(os.environ.get("SMOKE_HIGHLIGHT_PROBE_MS", "100"))
ACTIONS = ("keystroke", "scroll", "jump", "menu", "delete", "reopen")
# The rAF wrapper counts, it does not pump: a fixed timer would destroy time-to-paint numbers.
# Harness waits use the unwrapped reference so __rafCount counts only page frames.
RECORDER_INIT = """
(() => {
const nativeRaf = window.requestAnimationFrame.bind(window);
window.__nativeRaf = nativeRaf;
window.__rafCount = 0;
window.requestAnimationFrame = (cb) =>
nativeRaf((t) => {
window.__rafCount += 1;
cb(t);
});
// Counted, because every one of these is a double rAF and therefore a ~33ms vsync floor inside
// whatever is being timed across it. GROWTH_AXES used to declare that count by hand per axis
// and defaulted it to 0 for every generated axis, which left the floor in both ends of those
// ratios. begin() zeroes this and end() reports it, so a window's floor is measured rather
// than asserted, and waits taken OUTSIDE a recorder window (ACTION_SETUPS, for one) are
// excluded by construction rather than by remembering to exclude them.
window.__paintWaits = 0;
window.__nextPaint = () => {
window.__paintWaits += 1;
return new Promise((resolve) => nativeRaf(() => nativeRaf(() => resolve())));
};
// Chromium-only, recorded as a cross-check on the portable stall number and never as the
// headline.
//
// Support is read from supportedEntryTypes, NOT from whether observe() throws. Measured on
// this tree: PerformanceObserver.observe({ type: "longtask" }) throws on neither WebKit 26.5
// nor Firefox 153 -- it is accepted and then never fires. A try/catch therefore reports both
// as supported and both then report zero long tasks, which reads as "no jank on the engine
// Unsloth Desktop ships on macOS and Linux" rather than as "this API does not exist there".
// supportedEntryTypes lists `longtask` on Chromium alone.
window.__longTasks = [];
window.__longTaskSupported = Boolean(
(PerformanceObserver.supportedEntryTypes || []).includes("longtask"),
);
if (window.__longTaskSupported) {
try {
new PerformanceObserver((list) => {
for (const entry of list.getEntries()) {
window.__longTasks.push({ start: entry.startTime, duration: entry.duration });
}
}).observe({ type: "longtask", buffered: true });
} catch (e) {
window.__longTaskSupported = false;
}
}
const recorder = {
running: false,
// Which begin() a scheduled callback belongs to. `running` alone cannot answer that: when the
// next action starts before the previous action's already-scheduled rAF has fired -- and it
// does, because the actions are issued back to back over a CDP round trip that is shorter
// than one frame -- that stale callback wakes to `running === true`, pushes the whole
// between-action gap into the NEW arrays as if it were a frame, and schedules a second
// recursive loop that then runs alongside the real one for the rest of the run. Measured on
// the node harness in test_heavy_thread_measurement_integrity.py: without this token the
// second action records the inter-action gap as its worst frame and counts double the frames.
generation: 0,
frames: [],
stalls: [],
// When each sample was taken, so end() can close the window at an earlier instant than the
// one it was called at without the samples from after that instant still being in the arrays.
frameAt: [],
stallAt: [],
startedAt: 0,
begin() {
this.running = true;
this.generation += 1;
const generation = this.generation;
this.frames = [];
this.stalls = [];
this.frameAt = [];
this.stallAt = [];
this.startedAt = performance.now();
window.__paintWaits = 0;
let lastFrame = performance.now();
const frame = () => {
if (generation !== this.generation) return;
const now = performance.now();
this.frames.push(now - lastFrame);
this.frameAt.push(now);
lastFrame = now;
if (this.running) nativeRaf(frame);
};
nativeRaf(frame);
// 1ms setTimeout, not a MessageChannel ping-pong. Measured: the MessageChannel version
// ticks ~150k/s and halves Firefox's frame rate before any app code runs; this one ticks
// ~150/s and costs nothing on any engine. The clamp puts a ~4ms floor under the
// resolution, which is far below any stall a user can feel.
let lastStall = performance.now();
const stall = () => {
if (generation !== this.generation) return;
const now = performance.now();
this.stalls.push(now - lastStall);
this.stallAt.push(now);
lastStall = now;
if (this.running) setTimeout(stall, 1);
};
setTimeout(stall, 1);
},
/**
* Close the window. `untilMs` closes it at an earlier instant than "now" -- used by the
* settle loops, which have to keep watching for a while after the page went quiet in order
* to know that it stayed quiet, and must not charge that watching to the action.
*/
end(untilMs) {
this.running = false;
// Retire this generation as well as stopping it, so the one callback already in flight
// cannot append to the array a later begin() is about to hand out.
this.generation += 1;
const cutoff = untilMs === undefined ? Infinity : untilMs;
const wallMs = (untilMs === undefined ? performance.now() : untilMs) - this.startedAt;
// Trimmed, not just clocked: a settle loop that watches an idle page for another second
// would otherwise add sixty fast frames to the count and drag median_frame_ms down with
// them, at every size equally, which is the same constant-offset trap as the wall clock.
const frames = this.frames.filter((_, i) => this.frameAt[i] <= cutoff);
const stalls = this.stalls.filter((_, i) => this.stallAt[i] <= cutoff);
const sorted = frames.slice().sort((a, b) => a - b);
return {
wall_ms: Math.round(wallMs * 10) / 10,
// How many double-rAF waits this window was clocked across. Not trimmed to `cutoff`:
// a wait after the cutoff still happened inside the window that produced wall_ms, and
// wall_ms is the number this count is subtracted from.
paint_waits: window.__paintWaits,
frames: frames.length,
// The first entry spans begin() to the first callback, so it is the wait for the next
// vsync as much as a rendered frame. Kept: at these timescales it is ~16ms, well under
// the 33ms threshold, and dropping it would also drop a genuine stall that landed there.
worst_frame_ms: Math.round(Math.max(0, ...frames) * 10) / 10,
median_frame_ms:
sorted.length === 0 ? 0 : Math.round(sorted[Math.floor(sorted.length / 2)] * 10) / 10,
frames_over_33: frames.filter((ms) => ms > 33).length,
stall_ticks: stalls.length,
longest_stall_ms: Math.round(Math.max(0, ...stalls) * 10) / 10,
stalls_over_33: stalls.filter((ms) => ms > 33).length,
};
},
/**
* Time to settle, measured from the START of the action, not from the end of the input.
*
* Measured from the end of the gesture it reads ~50ms at every size on every engine, because
* the answer is then "three frames", which is the minimum this loop can return. From the
* start of the action it is what a user actually waits: the input, plus everything the page
* does afterwards before it is calm again.
*/
async quiet(timeoutMs) {
const started = performance.now();
let calm = 0;
let last = performance.now();
while (performance.now() - started < timeoutMs) {
await new Promise((resolve) => nativeRaf(() => resolve()));
const now = performance.now();
calm = now - last > 33 ? 0 : calm + 1;
last = now;
if (calm >= 3) return performance.now() - this.startedAt;
}
return null;
},
/**
* Settle for an action that also restarts the SYNTAX HIGHLIGHTER, which quiet() cannot see.
*
* quiet() declares an action settled after three sub-33ms frames. Shiki highlights each fence
* on its own task, and the lull between two of those batches is longer than that -- the same
* lull wait_for_highlighting_settled() exists for, where a two-read gate released at 577
* highlighted tokens out of the 3216 a finished thread holds. So on re-open, which rebuilds
* every fence from nothing, quiet() stops the clock partway through the rebuild.
*
* Settled here means: no frame over 33ms AND no new highlighted token, for graceMs.
*
* The returned time is the time of the LAST activity, not the time the grace window expired.
* graceMs is a fixed cost that every size would pay equally, and a constant added to both
* ends of a ratio drags it towards 1 -- the same trap the paint floor is subtracted for.
*
* `probe` counts highlighted tokens, which is a document-wide query, so it runs on an
* INTERVAL and not once per frame. An O(nodes) query inside the window being timed would cost
* more the bigger the thread is, which is to say it would grow like the signal -- the same
* reason DELETE_JS polls isConnected on a captured node instead of re-counting [data-role].
* The price is that the returned time is quantised to probeEveryMs, which is far below the
* differences this axis exists to show.
*/
async quietUntilIdle(timeoutMs, graceMs, probe, probeEveryMs) {
const started = performance.now();
let lastActivity = performance.now();
let lastCount = probe();
let lastProbeAt = performance.now();
let last = performance.now();
while (performance.now() - started < timeoutMs) {
await new Promise((resolve) => nativeRaf(() => resolve()));
const now = performance.now();
let changed = false;
if (now - lastProbeAt >= probeEveryMs) {
const count = probe();
changed = count !== lastCount;
lastCount = count;
lastProbeAt = now;
}
if (now - last > 33 || changed) lastActivity = now;
last = now;
if (now - lastActivity >= graceMs) {
return { settleMs: lastActivity - this.startedAt, at: lastActivity };
}
}
return { settleMs: null, at: performance.now() };
},
};
window.__hv = recorder;
})();
"""
def info(message: str) -> None:
print(f"[heavy-thread] {message}", flush = True)
# In-page actions bracket themselves with __hv.begin()/__hv.end(), excluding CDP round trips.
# Resolved on the second rAF, the frame that painted the character.
KEYSTROKE_JS = """
async (count) => {
const api = window.__heavyThread;
const input = api.composer();
if (!input) return null;
input.focus();
const setValue = Object.getOwnPropertyDescriptor(
HTMLTextAreaElement.prototype, "value",
).set;
const samples = [];
window.__hv.begin();
for (let i = 0; i < count; i += 1) {
await window.__nextPaint();
const started = performance.now();
setValue.call(input, input.value + "a");
input.dispatchEvent(new Event("input", { bubbles: true }));
await window.__nextPaint();
samples.push(performance.now() - started);
}
const metrics = window.__hv.end();
const sorted = samples.slice().sort((a, b) => a - b);
// domText is what the harness itself wrote; runtimeText is what the runtime received. Only
// the second can show the keystroke reached React rather than just the DOM node.
return {
samples: samples.map((s) => Math.round(s * 10) / 10),
// The first sample is systematically a cold outlier, which is why the headline is a median.
median_sample_ms: Math.round(sorted[Math.floor(sorted.length / 2)] * 10) / 10,
worst_sample_ms: Math.round(sorted[sorted.length - 1] * 10) / 10,
domText: input.value,
runtimeText: api.composerText(),
metrics,
};
}
"""
SCROLL_JS = """
async ([steps, stepPx, settleMs]) => {
const api = window.__heavyThread;
const viewport = api.viewport();
if (!viewport) return null;
// The viewport carries `scroll-smooth`, so each scrollTop write starts an animation and the
// next read lands mid-flight. Stepping from a tracked target with an explicit instant
// behaviour is what a wheel gesture actually does, and it is the only way the gesture moves
// the distance it asks for.
const bottom = viewport.scrollHeight - viewport.clientHeight;
viewport.scrollTo({ top: bottom, behavior: "instant" });
await window.__nextPaint();
let target = viewport.scrollTop;
// Reverse at either end rather than stopping. A small thread runs out of travel long before a
// large one does, and a gesture that covers 2600px at 25K chars and 8000px at 300K is not the
// same gesture, so the two columns would not be comparable.
let direction = -1;
let travelled = 0;
window.__hv.begin();
for (let i = 0; i < steps; i += 1) {
if (direction < 0 && target <= 0) direction = 1;
else if (direction > 0 && target >= bottom) direction = -1;
const next = Math.min(bottom, Math.max(0, target + direction * stepPx));
// The wheel event is what the app's own scroll listeners key off; the scrollTo is what
// moves the viewport in a headless run with no compositor input.
viewport.dispatchEvent(
new WheelEvent("wheel", { deltaY: direction * stepPx, bubbles: true, cancelable: true }),
);
viewport.scrollTo({ top: next, behavior: "instant" });
await window.__nextPaint();
travelled += Math.abs(next - target);
target = next;
}
const gestureMs = performance.now() - window.__hv.startedAt;
const settleMsTaken = await window.__hv.quiet(settleMs);
const metrics = window.__hv.end();
return {
scrolledPx: travelled,
gestureMs: Math.round(gestureMs * 10) / 10,
settleMs: settleMsTaken === null ? null : Math.round(settleMsTaken * 10) / 10,
metrics,
};
}
"""
# Wheel gestures are nearly free once laid out, so also time the jump to an unpainted region.
JUMP_JS = """
async (settleMs) => {
const api = window.__heavyThread;
const viewport = api.viewport();
if (!viewport) return null;
const bottom = viewport.scrollHeight - viewport.clientHeight;
viewport.scrollTo({ top: bottom, behavior: "instant" });
await window.__nextPaint();
window.__hv.begin();
const started = performance.now();
// The wheel event first, and it is load-bearing. Unsloth replaces assistant-ui's autoscroll
// with an intent-aware one, and a bare scrollTo from the bottom is read as programmatic and
// snapped straight back: measured, the jump landed at the bottom it started from and the whole
// column timed nothing. A wheel is what says a person did this.
viewport.dispatchEvent(
new WheelEvent("wheel", { deltaY: -bottom, bubbles: true, cancelable: true }),
);
viewport.scrollTo({ top: 0, behavior: "instant" });
await window.__nextPaint();
const paintedMs = performance.now() - started;
const settleMsTaken = await window.__hv.quiet(settleMs);
const metrics = window.__hv.end();
const landedAt = viewport.scrollTop;
viewport.scrollTo({ top: bottom, behavior: "instant" });
await window.__nextPaint();
return {
paintedMs: Math.round(paintedMs * 10) / 10,
settleMs: settleMsTaken === null ? null : Math.round(settleMsTaken * 10) / 10,
travelledPx: bottom,
landedAt,
metrics,
};
}
"""
# The trigger opens on pointerdown; element.click() leaves the menu shut and reads zero.
# The window's scans are a fixed count, measured at ~0.08% of the number at both sizes.
MENU_JS = """
async (timeoutMs) => {
const api = window.__heavyThread;
const trigger = api.actionButton("More");
if (!trigger) return null;
// A MutationObserver flag, not a querySelector per frame. The menu content is portaled to the
// end of document.body, so polling for it walks the whole message list and finds nothing for
// the entire open latency -- a cost that grows like the signal being measured.
let open = Boolean(document.querySelector(".aui-action-bar-more-content"));
const watcher = new MutationObserver(() => {
open = Boolean(document.querySelector(".aui-action-bar-more-content"));
});
watcher.observe(document.body, { childList: true, subtree: false });
const settle = async (want) => {
const started = performance.now();
while (performance.now() - started < timeoutMs) {
if (open === want) return performance.now() - started;
await window.__nextPaint();
}
return null;
};
const pointer = {
bubbles: true, cancelable: true, composed: true,
button: 0, pointerId: 1, pointerType: "mouse", isPrimary: true,
};
window.__hv.begin();
const openStarted = performance.now();
trigger.dispatchEvent(new PointerEvent("pointerdown", { ...pointer, buttons: 1 }));
trigger.dispatchEvent(new PointerEvent("pointerup", { ...pointer, buttons: 0 }));
const opened = await settle(true);
const openMs = opened === null ? null : performance.now() - openStarted;
const bodyPointerEvents = getComputedStyle(document.body).pointerEvents;
const itemsWhileOpen = api.openMenuItemCount();
// Counted here, under the pointer, not in the resting-state census. An autohidden bar is
// absent at rest by design; one that never mounts at all is a broken page, and only a hovered
// count tells the two apart.
const triggersWhileHovered = document.querySelectorAll('[data-slot="tooltip-trigger"]').length;
// The clock starts BEFORE the dispatch. Radix dismisses synchronously inside it -- layer
// teardown, focus restore, the body coming off the modal layer and the re-render that follows
// -- which is the fan-out being measured. Starting it after the dispatch excluded exactly the
// part worth timing.
const closeStarted = performance.now();
document.dispatchEvent(
new KeyboardEvent("keydown", { key: "Escape", bubbles: true, cancelable: true }),
);
const closed = await settle(false);
const metrics = window.__hv.end();
watcher.disconnect();
const closeMs = closed === null ? null : performance.now() - closeStarted;
return {
openMs: openMs === null ? null : Math.round(openMs * 10) / 10,
closeMs: closeMs === null ? null : Math.round(closeMs * 10) / 10,
open_close_ms:
openMs === null || closeMs === null ? null : Math.round((openMs + closeMs) * 10) / 10,
bodyPointerEvents,
bodyPointerEventsAfterClose: getComputedStyle(document.body).pointerEvents,
itemsWhileOpen,
triggersWhileHovered,
metrics,
};
}
"""
# Delete is an action-bar button again since 13615 (12735 had moved it to the More menu); it is
# found before the clock starts, so the window holds only the message leaving.
DELETE_JS = """
async (timeoutMs) => {
const api = window.__heavyThread;
const button = api.actionButton("Delete");
if (!button) return null;
// The last assistant message. That is the cheapest delete on the React side -- one subtree
// unmounts -- so this column under-measures reconciliation, though the export/rebuild/import
// half is O(messages) wherever the target sits.
const target = api.lastAssistantMessage();
const before = api.messageCount();
window.__hv.begin();
const started = performance.now();
button.click();
let ms = null;
// isConnected on the captured node is O(1). Re-counting [data-role] every frame would put an
// O(messages) query inside the window being timed, growing like the signal.
while (performance.now() - started < timeoutMs) {
if (target === null || !target.isConnected) {
ms = performance.now() - started;
break;
}
await window.__nextPaint();
}
const metrics = window.__hv.end();
return { ms, before, after: api.messageCount(), metrics };
}
"""
# Reopen rebuilds the whole Thread subtree. Completed tool cards remount collapsed as in the app;
# measured at ~0.45% of the tree at both sizes, so leaving them shut does not move the ratio.
REOPEN_JS = """
async ([timeoutMs, settleMs, graceMs, probeEveryMs]) => {
const api = window.__heavyThread;
const before = api.messageCount();
if (!before) return null;
window.__hv.begin();
const started = performance.now();
api.closeThread();
// Unmount first, or "already back" is indistinguishable from "never left".
// `paintWaits` below is REPORTED, not subtracted. Reopening is driven by a React state update,
// so the count check immediately after openThread() always still sees the unmounted tree and
// the loop always pays at least one __nextPaint() before it can observe the rebuilt messages.
// That one wait is the observation floor GROWTH_AXES removes (REOPEN_OBSERVATION_FLOOR), the
// same floor already subtracted from jump and delete. Any wait beyond it is a frame the
// APPLICATION spent committing rows, so it stays in the number. Reporting the count lets the
// harness verify it paid at least the floor it removes, and puts the progressive mount's frame
// count in the table as a diagnostic in its own right.
let closedMs = null;
let closePaintWaits = 0;
while (performance.now() - started < timeoutMs) {
if (api.messageCount() === 0) { closedMs = performance.now() - started; break; }
closePaintWaits += 1;
await window.__nextPaint();
}
const reopenStarted = performance.now();
api.openThread();
let ms = null;
let paintWaits = 0;
while (performance.now() - reopenStarted < timeoutMs) {
if (api.messageCount() >= before) { ms = performance.now() - reopenStarted; break; }
paintWaits += 1;
await window.__nextPaint();
}
// Not quiet(): re-open is the action whose whole cost is re-highlighting, and three calm frames
// land inside the lull between two Shiki batches.
const settled = await window.__hv.quietUntilIdle(
settleMs,
graceMs,
() => api.highlightedTokenCount(),
probeEveryMs,
);
const metrics = window.__hv.end(settled.at);
return {
ms,
paintWaits,
closePaintWaits,
closedMs: closedMs === null ? null : Math.round(closedMs * 10) / 10,
settleMs: settled.settleMs === null ? null : Math.round(settled.settleMs * 10) / 10,
before,
after: api.messageCount(),
metrics,
};
}
"""
# Two rAFs take at least two vsyncs (~33ms), so the floor is recorded per cell and subtracted.
PAINT_FLOOR_JS = """
async (samples) => {
const values = [];
for (let i = 0; i < samples; i += 1) {
await window.__nextPaint();
const started = performance.now();
await window.__nextPaint();
values.push(performance.now() - started);
}
values.sort((a, b) => a - b);
return values[Math.floor(values.length / 2)];
}
"""
def median(values: list[float | None]) -> float | None:
"""A missing repetition makes the median None; dropping it would hide a broken action."""
if not values or any(v is None for v in values):
return None
ordered = sorted(values)
middle = len(ordered) // 2
if len(ordered) % 2:
return round(ordered[middle], 1)
return round((ordered[middle - 1] + ordered[middle]) / 2, 1)
def cdp_metrics(cdp) -> dict[str, float]:
if cdp is None:
return {}
got = cdp.send("Performance.getMetrics")
return {m["name"]: m["value"] for m in got["metrics"]}
def cdp_counters(before: dict[str, float], after: dict[str, float]) -> dict[str, float | None]:
"""CHROMIUM-ONLY. Empty dicts off Chromium, and every consumer prints `-` for that."""
if not before or not after:
return {
"layout_count": None,
"recalc_style_count": None,
"layout_ms": None,
"recalc_style_ms": None,
"task_ms": None,
}
def d(name: str) -> float:
return after.get(name, 0.0) - before.get(name, 0.0)
return {
"layout_count": round(d("LayoutCount"), 1),
"recalc_style_count": round(d("RecalcStyleCount"), 1),
"layout_ms": round(d("LayoutDuration") * 1000, 1),
"recalc_style_ms": round(d("RecalcStyleDuration") * 1000, 1),
"task_ms": round(d("TaskDuration") * 1000, 1),
}
def long_task_summary(page) -> dict[str, float | None]:
"""CHROMIUM-ONLY. `supported` is the point: without it, an engine with no Long Tasks API
reports zero jank in exactly the same shape as an engine that had none."""
# PerformanceObserver delivers on a later task; yield once or the tail long task is dropped.
got = page.evaluate(
"""async () => {
await new Promise((r) => setTimeout(r, 0));
return { supported: window.__longTaskSupported, tasks: window.__longTasks };
}"""
)
if not got["supported"]:
return {"long_tasks": None, "long_task_ms": None, "worst_long_task_ms": None}
tasks = got["tasks"]
return {
"long_tasks": len(tasks),
"long_task_ms": round(sum(t["duration"] for t in tasks), 1),
"worst_long_task_ms": round(max((t["duration"] for t in tasks), default = 0.0), 1),
}
def reset_long_tasks(page) -> None:
page.evaluate("window.__longTasks.length = 0")
def wait_for_highlighting_settled(page, timeout_ms: int) -> None:
"""Requires five stable reads: two adjacent reads land in a lull between async Shiki batches."""
page.evaluate("() => { window.__hvTokens = undefined; }")
page.wait_for_function(
"""() => {
const n = window.__heavyThread.highlightedTokenCount();
const state = window.__hvTokens || { value: -1, stable: 0 };
if (n === state.value && n > 0) state.stable += 1;
else { state.value = n; state.stable = 0; }
window.__hvTokens = state;
return state.stable >= 5;
}""",
polling = 250,
timeout = timeout_ms,
)
def run_action(page, cdp, name: str, script: str, arg) -> dict:
"""One action, with the portable recorder inside it and the CDP counters bracketing it."""
reset_long_tasks(page)
before = cdp_metrics(cdp)
raw = page.evaluate(script, arg)
after = cdp_metrics(cdp)
if raw is None:
return {"name": name, "ran": False, **cdp_counters({}, {}), **long_task_summary(page)}
out = {"name": name, "ran": True}
out.update(raw.pop("metrics"))
out.update(raw)
out.update(cdp_counters(before, after))
out.update(long_task_summary(page))
return out
# Gate on codeExecutionPanes, not collapsibleOutputs: Radix keeps the content element in the tree
# while the card is shut, so that count is satisfied by closed cards.
EXPANDED_PANES_GATE_JS = "(n) => window.__heavyThread.counts().codeExecutionPanes >= n"
def build_fixture(page) -> None:
"""Expand tool panes before waiting on the highlighter; expanding mounts new unhighlighted code."""
expanded = page.evaluate("() => window.__heavyThread.expandTools()")
if expanded:
page.wait_for_function(
EXPANDED_PANES_GATE_JS,
arg = expanded,
timeout = ACTION_TIMEOUT_MS,
)
wait_for_highlighting_settled(page, ACTION_TIMEOUT_MS)
def one_repetition(page, cdp) -> dict[str, dict]:
"""The five scripted actions, once, in the order a user meets them."""
rep: dict[str, dict] = {}
# The previous repetition re-opened and deleted, so wait for re-highlighting before measuring.
build_fixture(page)
rep["keystroke"] = run_action(page, cdp, "keystroke", KEYSTROKE_JS, KEYSTROKES)
rep["scroll"] = run_action(
page,
cdp,
"scroll",
SCROLL_JS,
[SCROLL_STEPS, SCROLL_STEP_PX, SETTLE_TIMEOUT_MS],
)
rep["jump"] = run_action(page, cdp, "jump", JUMP_JS, SETTLE_TIMEOUT_MS)
# behavior "instant": the viewport is scroll-smooth and an animation would leak into the menu counters.
page.evaluate(
"""() => { const m = window.__heavyThread.lastAssistantMessage();
if (m) m.scrollIntoView({ block: "center", behavior: "instant" }); }"""
)
page.wait_for_function(
"""() => {
const top = window.__heavyThread.viewportMetrics().scrollTop;
const settled = window.__hvTop === top;
window.__hvTop = top;
return settled;
}""",
timeout = ACTION_TIMEOUT_MS,
)
page.locator('[data-role="assistant"]').last.hover(timeout = ACTION_TIMEOUT_MS)
rep["menu"] = run_action(page, cdp, "menu", MENU_JS, SETTLE_TIMEOUT_MS)
page.locator('[data-role="assistant"]').last.hover(timeout = ACTION_TIMEOUT_MS)
rep["delete"] = run_action(page, cdp, "delete", DELETE_JS, SETTLE_TIMEOUT_MS)
# The delete is permanent in the runtime, so restore before the next repetition, untimed.
restored = page.evaluate("() => window.__heavyThread.restore()")
page.wait_for_function(
"(n) => window.__heavyThread.messageCount() >= n",
arg = restored,
timeout = ACTION_TIMEOUT_MS,
)
build_fixture(page)
rep["reopen"] = run_action(
page,
cdp,
"reopen",
REOPEN_JS,
[SETTLE_TIMEOUT_MS, SETTLE_TIMEOUT_MS, HIGHLIGHT_GRACE_MS, HIGHLIGHT_PROBE_MS],
)
return rep
HEADLINE = {
"keystroke": ("median_sample_ms", True),
"scroll": ("gestureMs", False),
"jump": ("paintedMs", True),
"menu": ("open_close_ms", True),
"delete": ("ms", True),
"reopen": ("ms", False),
}
def summarise(reps: list[dict[str, dict]]) -> dict[str, dict]:
"""Median across repetitions, per action, per metric. Medians because the first repetition is
systematically the slowest and a mean would carry it into every cell."""
out: dict[str, dict] = {}
for action in ACTIONS:
rows = [r[action] for r in reps if action in r]
if not rows or not all(r.get("ran") for r in rows):
out[action] = {"ran": False}
continue
merged: dict = {"ran": True, "repetitions": len(rows)}
numeric_keys = set()
for row in rows:
for key, value in row.items():
# Keep None values so `menu["openMs"] is None` reports a never-opened menu instead of KeyError.
if value is None or (
isinstance(value, (int, float)) and not isinstance(value, bool)
):
numeric_keys.add(key)
for key in sorted(numeric_keys):
merged[key] = median([r.get(key) for r in rows])
for key in ("domText", "runtimeText", "bodyPointerEvents", "bodyPointerEventsAfterClose"):
if key in rows[-1]:
merged[key] = rows[-1][key]
merged["per_repetition"] = [r.get(HEADLINE[action][0]) for r in rows]
out[action] = merged
return out
def measure_cell(context, engine: str, size: int) -> dict:
"""Seed a fresh page to `size` characters of content and run the action set REPEATS times."""
page = context.new_page()
result: dict = {"chars_requested": size, "engine": engine}
# startswith, not `"/api/" in url`: vite serves source modules under paths like /src/.../api/.
api_prefix = f"{BASE}/api/"
stray_requests: list[str] = []
console_warnings: list[str] = []
# Errors kept separate from warnings: the warning allowance is only for Gecko scroll-anchoring notices.
console_errors: list[str] = []
page.on(
"request",
lambda r: stray_requests.append(r.url) if r.url.startswith(api_prefix) else None,
)
page.on(
"console",
lambda m: (
console_errors.append(m.text[:200])
if m.type == "error"
else console_warnings.append(m.text[:200])
if m.type == "warning"
else None
),
)
page.on("pageerror", lambda e: console_errors.append(f"pageerror: {e}"[:200]))
cdp = None
try:
page.goto(f"{BASE}/smoke-heavy-thread.html", wait_until = "domcontentloaded")
page.wait_for_function("() => Boolean(window.__heavyThread)", timeout = 60_000)
if engine == "chromium":
cdp = context.new_cdp_session(page)
cdp.send("Performance.enable")
plan = page.evaluate("(n) => window.__heavyThread.seed(n)", size)
result["plan"] = plan
# counts() walks every element, so polling it per frame makes seeding superlinear.
page.wait_for_function(
"(n) => window.__heavyThread.messageCount() >= n",
arg = plan["messages"],
timeout = SEED_TIMEOUT_MS,
)
# Radix unmounts collapsed content, so open the tool cards before the census.
result["tool_triggers_expanded"] = page.evaluate("() => window.__heavyThread.expandTools()")
page.wait_for_function(
EXPANDED_PANES_GATE_JS,
arg = max(1, result["tool_triggers_expanded"]),
timeout = SEED_TIMEOUT_MS,
)
# A <pre> exists before Shiki highlights it, so wait for the token count to stop moving.
wait_for_highlighting_settled(page, SEED_TIMEOUT_MS)
result["counts"] = page.evaluate("window.__heavyThread.counts()")
result["viewport"] = page.evaluate("window.__heavyThread.viewportMetrics()")
result["seed_api_requests"] = len(stray_requests)
result["seed_console_warnings"] = len(console_warnings)
result["first_seed_warning"] = console_warnings[0] if console_warnings else "-"
result["seed_console_errors"] = len(console_errors)
result["first_seed_error"] = console_errors[0] if console_errors else "-"
stray_requests.clear()
console_warnings.clear()
console_errors.clear()
result["cpu_throttle_rate"] = 1.0
if cdp is not None and CPU_THROTTLE_RATE != 1.0:
cdp.send("Emulation.setCPUThrottlingRate", {"rate": CPU_THROTTLE_RATE})
result["cpu_throttle_rate"] = CPU_THROTTLE_RATE
result["long_task_supported"] = page.evaluate("window.__longTaskSupported")
result["paint_floor_ms"] = round(page.evaluate(PAINT_FLOOR_JS, 9), 2)
reps = []
for index in range(REPEATS):
info(f" {engine} {size} chars: repetition {index + 1}/{REPEATS}")
reps.append(one_repetition(page, cdp))
result["repetitions"] = REPEATS
result["actions"] = summarise(reps)
result["raw_repetitions"] = reps
if cdp is not None and CPU_THROTTLE_RATE != 1.0:
cdp.send("Emulation.setCPUThrottlingRate", {"rate": 1})
result["raf_callbacks"] = page.evaluate("window.__rafCount")
result["stray_api_requests"] = len(stray_requests)
result["stray_api_urls"] = sorted(set(stray_requests))[:8]
result["stubbed_api_requests"] = len(page.evaluate("window.__stubbedApi || []"))
result["console_warnings"] = len(console_warnings)
result["first_console_warning"] = console_warnings[0] if console_warnings else "-"
result["console_errors"] = len(console_errors)
result["first_console_error"] = console_errors[0] if console_errors else "-"
finally:
page.close()
return result
def run() -> dict:
results: dict = {
"label": LABEL,
"base": BASE,
"sizes": SIZES,
"engines": ENGINES,
"repetitions": REPEATS,
"cpu_throttle_rate_requested": CPU_THROTTLE_RATE,
"by_engine": {},
}
with sync_playwright() as p:
for engine in ENGINES:
info(f"engine {engine}")
launcher = getattr(p, engine)
kwargs = {"headless": os.environ.get("SMOKE_HEADLESS", "1") == "1"}
# Chromium-only flags make Firefox/WebKit fail to launch.
if engine == "chromium":
kwargs["args"] = chromium_launch_args()
browser = launcher.launch(**kwargs)
results["by_engine"][engine] = {"version": browser.version, "by_size": {}}
context = browser.new_context(viewport = {"width": 1440, "height": 900})
context.add_init_script(RECORDER_INIT)
# Anchored at the origin so it cannot swallow vite module URLs under src/features/**/api/.
context.route(
re.compile(rf"^{re.escape(BASE)}/api/"),
lambda route: route.fulfill(
status = 200,
content_type = "application/json",
body = "{}",
),
)
for size in SIZES:
info(f"measuring {engine} at {size} chars")
# A crashed cell is recorded and the run continues; harness_failures still reports it.
try:
cell = measure_cell(context, engine, size)
except Exception as exc: # noqa: BLE001 - the message is the whole point
info(f"CRASHED {engine} at {size} chars: {type(exc).__name__}: {exc}")
cell = {
"chars_requested": size,
"engine": engine,
"crashed": f"{type(exc).__name__}: {exc}"[:400],
}
results["by_engine"][engine]["by_size"][str(size)] = cell
context.close()
browser.close()
return results
# CHROMIUM-ONLY rows name it so `-` never reads as zero; all metrics must be listed here.
def _action(action: str, key: str):
return lambda r: r["actions"][action][key]
def _floor_from(action: str, key: str):
"""A `floored` that is measured per row rather than declared once for every action."""
return lambda r: (r.get("actions", {}).get(action) or {}).get(key) or 0
TABLE_ROWS = (
("chars requested", lambda r: r["chars_requested"]),
("chars rendered", lambda r: r["plan"]["chars"]),
("messages seeded", lambda r: r["plan"]["messages"]),
("content cycles", lambda r: r["plan"]["cycles"]),
("chars per cycle", lambda r: r["plan"]["cycleChars"]),
("content kinds", lambda r: r["plan"]["kinds"]),
("tool cards expanded", lambda r: r["tool_triggers_expanded"]),
("repetitions", lambda r: r["repetitions"]),
("cpu throttle rate", lambda r: r["cpu_throttle_rate"]),
("paint floor ms", lambda r: r["paint_floor_ms"]),
("reopen paint waits", lambda r: (r.get("actions", {}).get("reopen") or {}).get("paintWaits")),
("longtask api supported", lambda r: r["long_task_supported"]),
("seed api requests", lambda r: r["seed_api_requests"]),
("seed console warnings", lambda r: r["seed_console_warnings"]),
("first seed warning", lambda r: r["first_seed_warning"]),
("action api requests", lambda r: r["stray_api_requests"]),
("stubbed api requests", lambda r: r.get("stubbed_api_requests", 0)),
("action console warnings", lambda r: r["console_warnings"]),
("seed console errors", lambda r: r.get("seed_console_errors", 0)),
("action console errors", lambda r: r.get("console_errors", 0)),
("first action error", lambda r: r.get("first_console_error", "-")),
("first action warning", lambda r: r["first_console_warning"]),
("messages rendered", lambda r: r["counts"]["messages"]),
("dom nodes", lambda r: r["counts"]["domNodes"]),
("code blocks", lambda r: r["counts"]["codeBlocks"]),
("code chars", lambda r: r["counts"].get("codeChars", 0)),
("highlighted tokens", lambda r: r["counts"]["highlightedTokens"]),
("fence blocks", lambda r: r["counts"].get("fenceBlocks", 0)),
("deferred fences", lambda r: r["counts"].get("deferredFences", 0)),
(
"mounted but unhighlighted fences",
lambda r: r["counts"].get("unhighlightedMountedFences", 0),
),
("tool parts", lambda r: r["counts"]["toolParts"]),
("collapsible tool outputs", lambda r: r["counts"]["collapsibleOutputs"]),
("code execution panes", lambda r: r["counts"]["codeExecutionPanes"]),
("artifact cards", lambda r: r["counts"]["artifactCards"]),
("images", lambda r: r["counts"]["images"]),
("action bars", lambda r: r["counts"]["actionBars"]),
("tooltip triggers", lambda r: r["counts"]["tooltipTriggers"]),
("viewport scrollHeight", lambda r: r["viewport"]["scrollHeight"]),
("viewport clientHeight", lambda r: r["viewport"]["clientHeight"]),
("rAF callbacks", lambda r: r["raf_callbacks"]),
)
for _name in ACTIONS:
TABLE_ROWS = TABLE_ROWS + (
(f"{_name} ran", _action(_name, "ran")),
(f"{_name} wall ms", _action(_name, "wall_ms")),
(f"{_name} longest stall ms", _action(_name, "longest_stall_ms")),
(f"{_name} worst frame ms", _action(_name, "worst_frame_ms")),
(f"{_name} median frame ms", _action(_name, "median_frame_ms")),
(f"{_name} frames over 33ms", _action(_name, "frames_over_33")),
(f"{_name} frames", _action(_name, "frames")),
(f"{_name} stalls over 33ms", _action(_name, "stalls_over_33")),
(f"{_name} stall ticks", _action(_name, "stall_ticks")),
(f"{_name} layouts (chromium only)", _action(_name, "layout_count")),
(f"{_name} layout ms (chromium only)", _action(_name, "layout_ms")),
(f"{_name} recalcs (chromium only)", _action(_name, "recalc_style_count")),
(f"{_name} recalc ms (chromium only)", _action(_name, "recalc_style_ms")),
(f"{_name} task ms (chromium only)", _action(_name, "task_ms")),
(f"{_name} longtasks (chromium only)", _action(_name, "long_tasks")),
(f"{_name} longtask ms (chromium only)", _action(_name, "long_task_ms")),
(f"{_name} worst longtask ms (chromium only)", _action(_name, "worst_long_task_ms")),
)
TABLE_ROWS = TABLE_ROWS + (
("keystroke median ms", _action("keystroke", "median_sample_ms")),
("keystroke worst ms", _action("keystroke", "worst_sample_ms")),
(
"keystroke per repetition",
lambda r: "/".join(str(v) for v in r["actions"]["keystroke"]["per_repetition"]),
),
(
"scroll per repetition",
lambda r: "/".join(str(v) for v in r["actions"]["scroll"]["per_repetition"]),
),
(
"menu per repetition",
lambda r: "/".join(str(v) for v in r["actions"]["menu"]["per_repetition"]),
),
(
"delete per repetition",
lambda r: "/".join(str(v) for v in r["actions"]["delete"]["per_repetition"]),
),
(
"reopen per repetition",
lambda r: "/".join(str(v) for v in r["actions"]["reopen"]["per_repetition"]),
),
("keystroke dom text", _action("keystroke", "domText")),
("keystroke runtime text", _action("keystroke", "runtimeText")),
("scroll gesture ms", _action("scroll", "gestureMs")),
("scroll settle ms", _action("scroll", "settleMs")),
("scroll px", _action("scroll", "scrolledPx")),
("jump painted ms", _action("jump", "paintedMs")),
("jump settle ms", _action("jump", "settleMs")),
("jump px", _action("jump", "travelledPx")),
("jump landed at", _action("jump", "landedAt")),
("menu open ms", _action("menu", "openMs")),
("menu close ms", _action("menu", "closeMs")),
("menu open+close ms", _action("menu", "open_close_ms")),
("menu items while open", _action("menu", "itemsWhileOpen")),
("menu body pe while open", _action("menu", "bodyPointerEvents")),
("menu body pe after close", _action("menu", "bodyPointerEventsAfterClose")),
("menu triggers hovered", _action("menu", "triggersWhileHovered")),
("delete ms", _action("delete", "ms")),
("delete messages before", _action("delete", "before")),
("delete messages after", _action("delete", "after")),
("reopen ms", _action("reopen", "ms")),
("reopen unmount ms", _action("reopen", "closedMs")),
("reopen settle ms", _action("reopen", "settleMs")),
("reopen messages before", _action("reopen", "before")),
("reopen messages after", _action("reopen", "after")),
)
def print_table(results: dict) -> None:
"""Every recorded metric, printed, one column per (engine, size)."""
columns = [(engine, str(size)) for engine in results["engines"] for size in results["sizes"]]
rows = []
for name, pick in TABLE_ROWS:
cells = []
for engine, size in columns:
try:
value = pick(results["by_engine"][engine]["by_size"][size])
cells.append("-" if value is None else str(value))
except (KeyError, TypeError):
cells.append("-")
rows.append((name, cells))
label_width = max(len(name) for name, _ in rows) + 2
headers = [f"{engine[:4]}/{int(size) // 1000}K" for engine, size in columns]
cell_width = max([len(c) for _, cells in rows for c in cells] + [len(h) for h in headers]) + 2
header = "".ljust(label_width) + "".join(h.rjust(cell_width) for h in headers)
info(header)
info("-" * len(header))
for name, cells in rows:
info(name.ljust(label_width) + "".join(cell.rjust(cell_width) for cell in cells))
# Exactly one double-rAF in `reopen ms` is observation: the check right after openThread() always sees
# the unmounted tree. Further waits are real progressive-mount latency, so the floor stays constant.
REOPEN_OBSERVATION_FLOOR = 1
# Reopen's window holds two terminal observation waits, one per loop.
WALL_FLOOR_OVERRIDES = {"reopen": 1 + REOPEN_OBSERVATION_FLOOR}
def _wall_floor(action: str):
"""The floor for `<action> wall ms`: the measured window count, unless the action overrides it
because part of that count is application work rather than instrument idle."""
override = WALL_FLOOR_OVERRIDES.get(action)
return _floor_from(action, "paint_waits") if override is None else override
# The third field is how many double-rAF waits the metric spans; each carries a ~33ms floor.
GROWTH_AXES = tuple(
[(f"{a} longest stall ms", _action(a, "longest_stall_ms"), 0) for a in ACTIONS]
+ [(f"{a} worst frame ms", _action(a, "worst_frame_ms"), 0) for a in ACTIONS]
+ [(f"{a} frames over 33ms", _action(a, "frames_over_33"), 0) for a in ACTIONS]
# Wall floors are read from the row since each window crosses a different number of waits.
+ [(f"{a} wall ms", _action(a, "wall_ms"), _wall_floor(a)) for a in ACTIONS]
+ [
# Axes measured from `__hv.startedAt` take the measured paint_waits; ones from a later mark keep a
# declared count. Scroll's waits come from a loop, so they must be counted at runtime.
("keystroke median ms", _action("keystroke", "median_sample_ms"), 1),
("scroll gesture ms", _action("scroll", "gestureMs"), _floor_from("scroll", "paint_waits")),
("scroll settle ms", _action("scroll", "settleMs"), _floor_from("scroll", "paint_waits")),
# NOT paint_waits: paintedMs starts after begin() and spans one wait.
("jump painted ms", _action("jump", "paintedMs"), 1),
("jump settle ms", _action("jump", "settleMs"), _floor_from("jump", "paint_waits")),
# Also NOT paint_waits: MENU_JS awaits no paint; its two floors come from settle() on entry.
("menu open+close ms", _action("menu", "open_close_ms"), 2),
("delete ms", _action("delete", "ms"), 1),
# Observation floor only; see REOPEN_OBSERVATION_FLOOR.
("reopen ms", _action("reopen", "ms"), REOPEN_OBSERVATION_FLOOR),
]
)
# At or below this ratio the axis did not respond to twelve times the content.
DISCRIMINATION_RATIO = float(os.environ.get("SMOKE_DISCRIMINATION_RATIO", "1.5"))
# Counters starting at zero have no ratio; one or two dropped frames is idle-machine noise.
ZERO_BASED_MIN_RISE = int(os.environ.get("SMOKE_ZERO_BASED_MIN_RISE", "5"))
# Counts are stated, not inferred: only `frames over 33ms` is a count; all other axes are ms.
COUNTER_AXES = frozenset(f"{a} frames over 33ms" for a in ACTIONS)
# Covers Gecko's two scroll-anchoring notices; per-message warnings would be far above it.
CONSOLE_WARNING_ALLOWANCE = int(os.environ.get("SMOKE_CONSOLE_WARNING_ALLOWANCE", "4"))
def resolve_floor(floored, row: dict) -> float:
"""Resolves a callable floor to a number, since a function in the JSON report breaks json.dumps."""
# NOT int(): a median over even repetitions can be 1.5 waits.
value = floored(row) if callable(floored) else floored
return value if isinstance(value, (int, float)) else 0
def growth(cells: dict, pick, floored, sizes: list[int]) -> tuple[float | None, float | None]:
"""floored counts the double-rAF waits in each metric; each adds a ~33ms vsync floor to subtract."""
try:
rows = (cells[str(sizes[0])], cells[str(sizes[-1])])
values = []
for row in rows:
value = pick(row)
if value is None:
return None, None
count = resolve_floor(floored, row)
if count:
value -= count * row["paint_floor_ms"]
values.append(round(value, 2))
return values[0], values[1]
except (KeyError, TypeError):
return None, None
def report_growth(results: dict) -> dict[str, dict[str, dict]]:
"""Per engine, per axis: the value at the smallest and largest size and their ratio."""
report: dict[str, dict[str, dict]] = {}
for engine in results["engines"]:
cells = results["by_engine"][engine]["by_size"]
per_axis: dict[str, dict] = {}
for name, pick, floored in GROWTH_AXES:
small, large = growth(cells, pick, floored, results["sizes"])
floor_counts = [
resolve_floor(floored, cells.get(str(size), {}))
for size in (results["sizes"][0], results["sizes"][-1])
]
if small is None or large is None:
per_axis[name] = {
"small": None,
"large": None,
"ratio": None,
"discriminated": False,
"reason": "not recorded",
}
continue
if small <= 0:
# A count that rises from 0 discriminates; a floored timing at or below zero has no room to move.
if floored:
per_axis[name] = {
"small": small,
"large": large,
"ratio": None,
"discriminated": False,
"reason": "at or under the paint floor at the smallest size",
"floored": floor_counts,
}
continue
# One incidental dropped frame must not mark a counter axis as discriminating.
if name not in COUNTER_AXES:
per_axis[name] = {
"small": small,
"large": large,
"ratio": None,
"discriminated": False,
"reason": (
"zero at the smallest size and this axis is a timing, not a count, "
"so there is no rise to measure"
),
"floored": floor_counts,
}
continue
rose = large >= ZERO_BASED_MIN_RISE
if rose:
reason = f"rose from zero to {large}"
elif large > small:
reason = (
f"rose from zero only to {large}, under the {ZERO_BASED_MIN_RISE} this "
"counter needs to be distinguishable from noise"
)
else:
reason = "zero at both ends"
per_axis[name] = {
"small": small,
"large": large,
"ratio": None,
"discriminated": rose,
"reason": reason,
"floored": floor_counts,
}
continue
ratio = round(large / small, 2)
# Noise floor for counters regardless of baseline: 1 -> 2 is a ratio of 2.0 about one frame.
noisy_counter = name in COUNTER_AXES and large < ZERO_BASED_MIN_RISE
per_axis[name] = {
"small": small,
"large": large,
"ratio": ratio,
"discriminated": ratio > DISCRIMINATION_RATIO and not noisy_counter,
"reason": (
f"only {large} events at the largest size, under the "
f"{ZERO_BASED_MIN_RISE} this counter needs to be distinguishable from noise"
if noisy_counter
else "-"
),
"floored": floor_counts,
}
report[engine] = per_axis
return report
def print_growth(results: dict, report: dict) -> None:
for engine, per_axis in report.items():
info("")
info(
f"growth on {engine} ({results['sizes'][0]} -> {results['sizes'][-1]} chars, "
f"median of {results['repetitions']} repetitions)"
)
for name, row in per_axis.items():
if row["ratio"] is None:
mark = "DISCRIMINATES" if row["discriminated"] else "flat"
small = "-" if row["small"] is None else row["small"]
large = "-" if row["large"] is None else row["large"]
info(
f" {name:<34} {small:>10} -> {large:>10} - " f"{mark} ({row['reason']})"
)
continue
mark = "DISCRIMINATES" if row["discriminated"] else "flat"
floor_note = " (paint floor removed)" if row.get("floored") else ""
info(
f" {name:<34} {row['small']:>10} -> {row['large']:>10} "
f"{row['ratio']:>6.2f}x {mark}{floor_note}"
)
# Declared double-rAF floor vs reported waits; a lower bound, as extra waits are real commits.
FLOOR_COUNTERS = {"reopen ms": ("reopen", "paintWaits")}
def declared_floor(axis_name: str) -> int | None:
"""Matches the axis name exactly: a prefix would let reopen ms pick up reopen settle ms's floor."""
for name, _pick, floored in GROWTH_AXES:
if name == axis_name:
return floored
return None
def floor_declaration_problems(results: dict) -> list[str]:
"""Axes that subtract more double-rAF floors than the action actually waited out."""
problems: list[str] = []
for engine in results["engines"]:
for size in results["sizes"]:
row = results["by_engine"][engine]["by_size"].get(str(size), {})
if "crashed" in row:
continue
for axis_name, (action, counter) in FLOOR_COUNTERS.items():
measured = row.get("actions", {}).get(action) or {}
# Already reported by harness_failures; do not add a second failure for one cause.
if not measured.get("ran", True):
continue
observed = measured.get(counter)
if observed is None:
problems.append(
f"{engine} at {size} chars recorded no {counter} for {action}, so the "
f"paint floor subtracted from '{axis_name}' is unverified"
)
continue
# Resolve against this row so row-read floors are verified as what will be subtracted.
declared = resolve_floor(declared_floor(axis_name), row)
if observed >= declared:
continue
problems.append(
f"{engine} at {size} chars paid {observed} paint wait(s) in {action} but "
f"GROWTH_AXES subtracts {declared} from '{axis_name}'; a metric cannot be "
"credited with a floor it never waited out"
)
return problems
def harness_failures(results: dict, report: dict) -> list[str]:
"""Only ways this harness can measure nothing; no performance budgets here."""
failures: list[str] = list(floor_declaration_problems(results))
for engine in results["engines"]:
for size in results["sizes"]:
row = results["by_engine"][engine]["by_size"][str(size)]
where = f"{engine} at {size} chars"
if "crashed" in row:
failures.append(
f"{where} crashed before it produced a measurement: {row['crashed']}"
)
continue
counts = row["counts"]
plan = row["plan"]
if row["stray_api_requests"]:
urls = row.get("stray_api_urls") or []
named = ", ".join(urls) if urls else "(urls not recorded)"
failures.append(
f"{where} let {row['stray_api_requests']} /api/ requests reach the network "
f"during the measured actions; the timings include a round trip per "
f"request. Endpoints: {named}"
)
# Absolute warning cap, not a growth check: Firefox emits two scroll-anchoring notices only at larger
# sizes. Errors get no allowance: one inside a timed interaction invalidates its timing.
for phase, count, first in (
("seeding", row.get("seed_console_errors", 0), row.get("first_seed_error", "-")),
(
"the measured actions",
row.get("console_errors", 0),
row.get("first_console_error", "-"),
),
):
if count:
failures.append(
f"{where} logged {count} console error(s) or page error(s) during "
f"{phase}, the first being {first!r}; an application exception is not "
"engine chatter and the timings around it are not measurements"
)
if row["console_warnings"] > CONSOLE_WARNING_ALLOWANCE:
failures.append(
f"{where} logged {row['console_warnings']} console warnings during the "
f"measured actions, the first being {row['first_console_warning']!r}; that is "
f"more than the {CONSOLE_WARNING_ALLOWANCE} allowed for engine chatter, so "
"the timings include work this harness is charging to the app"
)
if plan["chars"] < size * 0.9:
failures.append(
f"{where} only built {plan['chars']} characters of the {size} asked for"
)
if counts["messages"] < plan["messages"]:
failures.append(
f"{where} rendered {counts['messages']} of {plan['messages']} messages; the "
"seed did not land"
)
# Renderer silently drops some data URLs (non-PNG/JPEG/GIF/WebP); check every kind.
for key, per_cycle in plan["expectedPerCycle"].items():
want = per_cycle * plan["cycles"]
if counts.get(key, 0) < want:
failures.append(
f"{where} rendered {counts.get(key, 0)} {key}, short of the {want} its "
f"{plan['cycles']} content cycles should produce; the fixture is not the "
"heavy thread this harness claims to measure"
)
if counts.get("highlightedTokens", 0) <= 0:
failures.append(f"{where} highlighted nothing; Shiki never ran")
# Deferral is expected; a mounted, non-deferred, unhighlighted fence in a settled thread is not.
stuck = counts.get("unhighlightedMountedFences", 0)
if stuck:
failures.append(
f"{where} left {stuck} of {counts.get('fenceBlocks', 0)} code fences mounted "
f"but unhighlighted after settling, neither deferred nor coloured; the "
"thread had not finished building itself when it was measured"
)
viewport = row["viewport"]
if viewport["scrollHeight"] <= viewport["clientHeight"]:
failures.append(
f"{where} does not overflow its viewport; the scroll measures nothing"
)
actions = row["actions"]
for name in ACTIONS:
if not actions[name].get("ran"):
failures.append(f"{where} could not run the {name} action at all")
# A null settle time means the settle loop timed out, not "unsupported".
for name in ("scroll", "jump", "reopen"):
settling = actions[name]
if settling.get("ran") and settling.get("settleMs") is None:
failures.append(
f"{where} ran the {name} action but it never reached a settled state "
f"within {SETTLE_TIMEOUT_MS}ms, so its settle time and the frame counts "
"beside it are the timeout rather than a measurement"
)
keystroke = actions["keystroke"]
if keystroke.get("ran"):
# Only the runtime's copy proves the keystroke reached React; domText is what the harness wrote.
if keystroke["runtimeText"] != keystroke["domText"]:
failures.append(
f"{where} typed {keystroke['domText']!r} into the DOM but the runtime "
f"holds {keystroke['runtimeText']!r}; the keystroke never reached the "
"composer state"
)
# Sitting on the paint floor is a finding, not a failure: the character paints on the next frame.
scroll = actions["scroll"]
if scroll.get("ran") and scroll["scrolledPx"] < SCROLL_STEPS * SCROLL_STEP_PX * 0.9:
failures.append(
f"{where} travelled only {scroll['scrolledPx']}px of the "
f"{SCROLL_STEPS * SCROLL_STEP_PX}px gesture, so its scroll column is not "
"comparable with the others"
)
jumped = actions["jump"]
if jumped.get("ran"):
# The jump is deliberately bottom to top at every size; it must have arrived.
if jumped["landedAt"] > 1:
failures.append(
f"{where} jumped to the top of the thread and landed at "
f"{jumped['landedAt']}px; the viewport did not move"
)
if jumped["travelledPx"] <= viewport["clientHeight"]:
failures.append(
f"{where} had only {jumped['travelledPx']}px to jump through, which is "
"less than one viewport; nothing had to be painted"
)
menu = actions["menu"]
if menu.get("ran"):
if menu["openMs"] is None:
failures.append(f"{where} never opened the message action menu")
elif menu["closeMs"] is None:
failures.append(f"{where} opened the action menu and it never closed")
elif menu["bodyPointerEventsAfterClose"] == "none":
failures.append(
f"{where} left the body on the modal layer after closing the menu"
)
elif not menu["itemsWhileOpen"]:
failures.append(f"{where} opened an action menu with no items in it")
if not menu["triggersWhileHovered"] and counts["actionBars"] <= 0:
failures.append(
f"{where} mounted no action bar at rest and none under the pointer either"
)
deleted = actions["delete"]
if deleted.get("ran"):
if deleted["ms"] is None:
failures.append(f"{where} never deleted a message")
elif deleted["after"] >= deleted["before"]:
failures.append(f"{where} clicked delete and the message count did not drop")
reopened = actions["reopen"]
if reopened.get("ran"):
if reopened["ms"] is None:
failures.append(f"{where} re-opened the thread and it never came back")
elif reopened["closedMs"] is None:
failures.append(
f"{where} never saw the thread unmount, so its re-open time is the cost "
"of a thread that never left"
)
# Modal and non-modal menus cost very different amounts, so sizes must not mix them.
layers = {
results["by_engine"][engine]["by_size"][str(size)]
.get("actions", {})
.get("menu", {})
.get("bodyPointerEvents")
for size in results["sizes"]
if "crashed" not in results["by_engine"][engine]["by_size"][str(size)]
}
if len(layers) > 1:
failures.append(
f"on {engine} the menu put the body on {sorted(str(x) for x in layers)} across "
"sizes; the columns are not measuring the same mechanism"
)
# Discrimination, not a budget: equal cost at all sizes means the page was never driven.
if len(results["sizes"]) >= 2:
for engine, per_axis in report.items():
if not any(row["discriminated"] for row in per_axis.values()):
failures.append(
f"on {engine} no measured axis rose by more than {DISCRIMINATION_RATIO}x "
f"from {results['sizes'][0]} to {results['sizes'][-1]} characters. Either the "
"page was never driven or every action is being measured somewhere it does "
"not run; the numbers above cannot size any change."
)
return failures
def main() -> int:
vite = None
if OWNS_SERVER:
info(f"starting vite dev server on port {PORT}")
vite = start_vite(PORT)
try:
wait_for_smoke_page(
f"{BASE}/smoke-heavy-thread.html",
"smoke-heavy-thread-main.tsx",
proc = vite,
info = info,
)
results = run()
finally:
if vite is not None:
stop_process(vite)
info("vite stopped")
report = report_growth(results)
results["growth"] = report
out = OUT / f"{LABEL}.json"
out.write_text(json.dumps(results, indent = 2), encoding = "utf-8")
print_table(results)
print_growth(results, report)
info(f"wrote {out}")
failures = harness_failures(results, report)
for problem in failures:
info(f"HARNESS-BROKEN {problem}")
if failures:
return 1
info("measurement only: no budgets are asserted here, so this exits 0 on any timing.")
return 0
if __name__ == "__main__":
raise SystemExit(main())