Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Trim comments in the studiobench harness and unsloth_cli (#10118)

Daniel Han25c10a30ddd5
diff --git a/tests/studio/studiobench/__main__.py b/tests/studio/studiobench/__main__.py
index 1c6baa3cbeb..c0935538f70 100644
--- a/tests/studio/studiobench/__main__.py
+++ b/tests/studio/studiobench/__main__.py
@@ -24,33 +24,22 @@ import time
from pathlib import Path
from typing import Optional
-# BUMPED BECAUSE THE INSTRUMENTS CHANGED, not because the tool gained features. This is one of the
-# fields the comparability key is computed over, and it is the ONLY field that can separate a run
-# taken before this change from one taken after it: the corpus is untouched, so the corpus hash,
-# the tier and the rungs all match across the change.
-#
-# They must not match. `reasoning_toggle` now settles on a quiet DOM, so
-# `highlight_spans_while_open` reads 74,250 where it used to read 44,075 on the same bundle, and
-# `open_ms` now terminates on a settled mount rather than on the `data-state` flip, which makes it
-# a different quantity rather than a better-measured one. A key that certified those two runs as
-# comparable would be the exact failure this key exists to prevent, committed by the guard itself.
-#
-# Any future change to what an instrument MEASURES has to bump this for the same reason.
+# BUMPED BECAUSE THE INSTRUMENTS CHANGED, not because the tool gained features: corpus, tier
+# and rungs all match, so this is the only key field that can separate runs across it.
+# Bump whenever what an instrument MEASURES changes.
TOOL_VERSION = "0.2.0"
TIERS = ("fast", "quick", "standard", "full")
TIER_RUNGS = {
- # One rung, and it is 100K on purpose. The 10K rung was measured across six PRs and could
- # not separate any of them from the null control: at that size the UI work disappears
- # underneath the scene's own scripted timings (copy_markdown read 204 ms on all fourteen
- # arms). 100K is the smallest rung that carries real load -- jank index 29 against 0.6,
- # worst frame 1,855 ms against 214 -- so it is the only rung worth an iteration loop.
+ # One rung, 100K on purpose: 10K separated none of six PRs from the null control, while 100K
+ # carries real load (jank index 29 against 0.6).
+ # At 10K `copy_markdown` read 204 ms on all fourteen arms; the worst frame is 1,855 ms against 214.
"fast": ["100K"],
"quick": ["1K", "10K"],
"standard": ["1K", "10K", "100K"],
"full": ["1K", "10K", "100K", "500K", "1M"],
}
-# Wall clock budgets, from the design. The deficit-scheduled pacer is what makes them honest: a
-# slow machine gets the same stream duration, so it misses SLOTS rather than overrunning the tier.
+# Wall clock budgets from the design. The deficit-scheduled pacer keeps them honest: a slow
+# machine gets the same stream duration, so it misses SLOTS rather than overrunning the tier.
TIER_BUDGET_S = {"fast": 5 * 60, "quick": 5 * 60, "standard": 20 * 60, "full": 60 * 60}
@@ -106,10 +95,8 @@ def doctor(args) -> int:
import subprocess
import playwright # noqa: F401
- # In a SUBPROCESS. Starting and stopping Playwright's node driver just to read the engine
- # paths leaves asyncio tearing down a cancelled task, and it prints a TargetClosedError
- # traceback to stderr after the check has already passed. A doctor that prints a
- # traceback next to `[ok]` is a doctor nobody believes.
+ # In a SUBPROCESS: starting Playwright's node driver just to read engine paths leaves asyncio
+ # printing a TargetClosedError traceback next to `[ok]`.
probe = (
"from playwright.sync_api import sync_playwright\n"
"with sync_playwright() as pw:\n"
@@ -129,15 +116,8 @@ def doctor(args) -> int:
if got.returncode != 0:
raise RuntimeError((got.stderr or "the engine probe failed").strip().splitlines()[-1])
text = got.stdout.strip()
- # THE PACKAGE IS NOT THE ENGINE. `pip install playwright` and `playwright install` are two
- # steps and the README says so, so the machine with the package and no downloaded binary is
- # the ordinary case rather than an exotic one. Say so in the line the reader sees, with the
- # command that fixes it, rather than reporting a bare `[ok]` they will act on.
- #
- # REPORTED, NOT FATAL, and deliberately so. This doctor has to keep running on a machine
- # with no engine at all, because that is exactly the machine an external tester points it
- # at first; a non-zero exit there would turn the one command that explains the problem
- # into another thing that fails without saying why.
+ # THE PACKAGE IS NOT THE ENGINE: `pip install playwright` and `playwright install` are two steps.
+ # Reported, not fatal -- a machine with no engine is the first one a tester points this at.
if not engines_installed(text):
return (
f"{text}; no engine is downloaded. Run `playwright install webkit` "
@@ -300,8 +280,7 @@ def planned_rungs(args) -> list:
"""
if not args.rungs:
return list(TIER_RUNGS[args.tier])
- # Imported here, not at module scope, on the same rule the rest of this file follows: `--help`
- # has to answer on a machine with nothing installed.
+ # Imported here, not at module scope: `--help` has to answer on a machine with nothing installed.
from .fixture.corpus import RUNGS
rungs = [label.strip().upper() for label in args.rungs.split(",")]
@@ -316,19 +295,17 @@ def planned_rungs(args) -> list:
return rungs
-#: What a cell costs BESIDES its film, measured from the steps `CellRunner._run_inner` runs around
-#: it: seeding the thread and reading it back, the navigation and thread wait, the 1.5 s idle
-#: calibration, the chars-per-token measurement, the drain and the two censuses. Comfortably over
-#: what the small rungs cost and about what 1M costs.
+#: What a cell costs BESIDES its film: seeding and read-back, navigation, the 1.5 s idle
+#: calibration, chars-per-token, the drain and the two censuses.
+# Measured from the steps `CellRunner._run_inner` runs around the film.
CELL_OVERHEAD_S = 60
#: One optional `--surfaces` sweep per arm, before the cells.
SURFACE_SWEEP_S = 120
#: The same generosity the tier budget already carries, applied to the planned work.
WATCHDOG_MARGIN = 3
-#: An absolute ceiling on the MEASUREMENT half of the deadline. The point of a watchdog is that a
-#: wedged run cannot outlive it, so scaling with the plan must not scale without limit: `--reps
-#: 1000` would otherwise arm a deadline measured in months and the watchdog would guarantee
-#: nothing at all. 24 hours is past any plan this tool supports and short of forever.
+#: An absolute ceiling on the MEASUREMENT half of the deadline: without it `--reps 1000` arms a
+#: deadline measured in months and the watchdog guarantees nothing.
+#: 24 hours is past any plan this tool supports and short of forever.
WATCHDOG_MAX_MEASUREMENT_S = 24 * 60 * 60
@@ -607,15 +584,10 @@ def run(args, ab_ref = None) -> int:
"""
from .runtime.types import OutDirLock, Paths
- # THE ARM NAMES, BEFORE A SINGLE PROCESS EXISTS AND BEFORE A SINGLE DIRECTORY. Nothing below
- # this line can be undone without the cleanup `finally`, and that block is a long way down; see
- # `_windowed_arms`. The labels are READ OFF THE SPECS rather than spelled a second time, so the
- # check and the sides cannot drift into disagreeing about what an arm is called -- and
- # `side_specs` is a pure list build, so asking it this early starts nothing.
- #
- # AHEAD OF THE OUTPUT DIRECTORY as well as ahead of the processes. A mistyped `--windowed-arm`
- # must cost nothing at all, and a directory tree with a lock file in it is not nothing. See
- # test_studiobench_windowed_arm_names.
+ # THE ARM NAMES, BEFORE ANY PROCESS OR DIRECTORY EXISTS -- nothing below here undoes without the
+ # far-away cleanup `finally`. Labels are read off the specs so the check and the sides agree.
+ # Ahead of the output directory too: a mistyped `--windowed-arm` must not cost a lock file.
+ # Pinned by test_studiobench_windowed_arm_names.
specs = side_specs(args, ab_ref)
arm_labels = [label for label, _ref, _attach, _port, _password in specs]
windowed = _windowed_arms(getattr(args, "windowed_arm", ""), arm_labels)
@@ -656,19 +628,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
corpus = Corpus.load()
_log(f" corpus_hash {corpus.corpus_hash}")
- # READ NOW, BEFORE ANYTHING IS STARTED OR MOVED. The probe's source is not needed until the
- # browser is launched, but reading it there means a path typo raises after Unsloth and the pacer
- # are up and before the cleanup `finally` that would stop them is entered, so a detached Unsloth
- # keeps running and holds its port. The cheapest correct fix is to fail while there is nothing
- # to clean up: a missing, unreadable or non-UTF-8 file is a mistake in the invocation, and the
- # right moment to say so is the first second of the run.
- #
- # AHEAD OF `prepare_payload` FOR THE SAME REASON THE ARCHIVE IS AHEAD OF THE RECORDER. Reusing
- # an `--out` without `--resume` archives the payload that is already there, so reading the
- # probe afterwards meant a path typo exited 2 having run no benchmark and having already taken
- # `payload.jsonl` off the path every reader looks at: `--report`, `--assert-liveness` and the
- # next `--resume` all open that name. A refusal that the first millisecond of the run could
- # have issued must not cost the previous run's payload its standard path.
+ # READ NOW, BEFORE ANYTHING IS STARTED OR MOVED: reading the probe at launch time raises after
+ # Unsloth is up but before the cleanup `finally`, leaving a detached server on its port.
+ # Ahead of `prepare_payload`, or a late refusal takes payload.jsonl off its standard path.
extra_init = os.environ.get("SBENCH_EXTRA_INIT_SCRIPT")
extra_init_source = ""
if extra_init:
@@ -681,42 +643,26 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
)
return 2
- # EVERY A/B ARGUMENT CHECK, AHEAD OF `prepare_payload`, FOR THE REASON STATED DIRECTLY ABOVE.
- # These three refuse the INVOCATION -- nothing they read comes from an Unsloth, a browser or a
- # payload, only `args` and `specs`, both of which exist before this function starts anything.
- # Left below the archive they cost the previous run its payload's standard path: a fresh
- # `--ab X --home H --out DIR` over a finished DIR archived `payload.jsonl` and then exited 2
- # having run no benchmark, so the next `--resume` found nothing to continue and silently
- # re-ran the whole ladder, while `--report` and `--assert-liveness` opened the standard name
- # and found no rows. `rollback_session_rows` states the rule: a refusal has to leave the
- # payload it refused exactly as it found it. That rule covers `invalidate_stale_reports`
- # below for the same reason it covers the archive: it REPLACES `summary.md` and `ab.md`, so
- # a refusal reaching it would take the previous run's reports down with its payload.
+ # EVERY A/B ARGUMENT CHECK, AHEAD OF `prepare_payload`: these refuse the INVOCATION and read only
+ # `args` and `specs`. Below the archive they cost the previous run its payload's path, so the
+ # next `--resume` found nothing. See `rollback_session_rows`.
if ab_ref:
if args.attach and not args.attach_b:
_log(" --ab with --attach needs --attach-b URL: the second build has to be somewhere.")
return 2
- # Here rather than beside the init script it guards, because by then two Unsloth instances have been
- # cloned, built and launched to run a validation that cannot say anything. See
- # `stream_cost_injection_problem`.
+ # Here rather than beside the init script it guards: by then two Unsloth instances have been
+ # built and launched to run a validation that cannot say anything.
injection_problem = stream_cost_injection_problem(
specs, getattr(args, "inject_stream_cost_ms", None)
)
if injection_problem:
_log(f" {injection_problem}")
return 2
- # ONE HOME CANNOT HOLD TWO BUILDS. `install_studio` checks the ref out into a repo
- # directory derived from the home and installs from it, so two arms sharing a home share
- # one checkout: the second install overwrites the first and BOTH arms then serve whichever
- # build was installed last. The A/B compares a build against itself and reports a clean,
- # confident "no difference" -- which is the same failure mode as a copy timing that is
- # really a sleep, and just as invisible in the payload.
- #
- # Measured, before this refused: two runs of the same pair, one at 716 ms and 718 ms for
- # base and treatment, the other at 2,583 ms and 2,614 ms. Nearly equal WITHIN each run and
- # 3.6x apart BETWEEN them, because each run was internally uniform and the two runs were
- # serving different builds. Read as a within-run comparison it says the change does
- # nothing; the difference was sitting between the runs the whole time.
+ # ONE HOME CANNOT HOLD TWO BUILDS: `install_studio` derives the checkout from the home, so two
+ # arms sharing a home share one checkout: the second install overwrites the first and both
+ # arms then serve whichever build was installed last, so the A/B compares a build with itself.
+ # Measured: two runs of the same pair, equal within each and 3.6x apart between them.
+ # The pair read 716 ms and 718 ms within one run.
if not args.attach and args.home:
_log(
" --home cannot be used with --ab: both arms would install into that one "
@@ -726,23 +672,19 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
)
return 2
- # BEFORE the first install, the first launch and the first recorded row. See `prepare_payload`:
- # a refusal that arrives after two clones and two builds has cost the caller an hour to say
- # something it could have said in a millisecond, and an archive that arrives after the Recorder
- # has opened the file has already appended this run's header to the payload it was moving.
+ # BEFORE the first install, launch and recorded row: a refusal after two builds costs an hour to
+ # say what a millisecond could, and a late archive has already appended this run's header.
archived = prepare_payload(
paths, requested_identity(args, ab_ref, corpus.corpus_hash), resume = bool(args.resume)
)
- # AND THE REPORTS THE ARCHIVE LEFT BEHIND. See `invalidate_stale_reports`: this is the one
- # point every reuse of an output directory passes through, whichever of the two ways it
- # invalidates what is already sitting there.
+ # AND THE REPORTS THE ARCHIVE LEFT BEHIND: the one point every reuse of an output directory
+ # passes through. See `invalidate_stale_reports`.
invalidate_stale_reports(paths.out, archived = archived, extra_init = extra_init)
- # Armed AFTER the sides are known, because what it has to cover depends on them: see
- # `watchdog_deadline_s`. Nothing between the `side_specs` call above and here can hang --
- # `_windowed_arms` compares two sets, `Corpus.load` reads a generated fixture, and
- # `prepare_payload` and `invalidate_stale_reports` each touch a handful of files in `--out`.
+ # Armed AFTER the sides are known, because what it covers depends on them. Nothing between the
+ # `side_specs` call and here can hang.
+ # The budget is `watchdog_deadline_s`.
watchdog = browser_mod.install_wall_clock_watchdog(
watchdog_deadline_s(
args.tier,
@@ -759,20 +701,11 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
installs = []
sides = []
- # EVERY SIDE THIS RUN LAUNCHED IS STOPPED WHEN THE SETUP AROUND IT FAILS. The sides are
- # acquired one after another, so the base is already SERVING while the treatment clones and
- # builds, and the cleanup that stops them both is the `finally` under the cells -- which a
- # failure up here never reaches. The Unsloth instances are the resources that outlive this process:
- # `launch_studio` detaches the server with `setsid -f`, so an abandoned one keeps its port.
- # It is not idle there. Unsloth's own launcher ABORTS rather than binding when it finds one of
- # its own servers on the requested port (`studio/backend/run.py`, `_resolve_port` with
- # `avoid_own_studio`), so the next attempt's server exits and `wait_for_healthz` takes its 200
- # from the STALE process: that run measures the build this one installed while `run_meta`
- # records the ref it was asked for.
- #
- # A RETURN IS A FAILURE HERE TOO. The health check below and the development-build gate leave
- # by returning, with both Unsloth instances up, which is the same abandoned server by a quieter route --
- # so the guard is a `finally` on the whole of setup rather than an `except` on the install.
+ # EVERY SIDE THIS RUN LAUNCHED IS STOPPED WHEN THE SETUP AROUND IT FAILS: the base is already
+ # SERVING while the treatment builds, and the cleanup `finally` under the cells is never reached
+ # from up here. A stale detached server makes the next run's `wait_for_healthz` take its 200 from
+ # the wrong build. A RETURN IS A FAILURE HERE TOO, so this is a `finally` over all of setup.
+ # The next server exits on `avoid_own_studio` rather than binding.
setup_complete = False
try:
for label, ref, attach, port, password in specs:
@@ -799,10 +732,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
}
)
- # THE BUILD, NOW THAT IT IS KNOWN. `prepare_payload` has already agreed the refs match,
- # and a ref is a pointer: see `commit_problems`. This is the first moment the commit
- # behind it exists, and it is still before the browser, the pacer and every cell, so a
- # resume onto a moved branch costs the install it has already paid and nothing more.
+ # THE BUILD, NOW THAT IT IS KNOWN: `prepare_payload` has agreed the refs match, and a ref is a
+ # pointer. Still before the browser and every cell, so a moved branch costs only the install.
+ # `checkout_ref` is the only place a ref becomes a commit.
if args.resume:
resolved = resolved_commits(sides)
commit_issues: list = []
@@ -829,8 +761,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
_log(f" FATAL: {side['base_url']}/healthz did not answer 200")
return 2
+ # THE GATE, before anything else is measured. Both sides: an A/B where one side is a development
+ # build puts the whole 3.2x inflation on one arm and reads as a colossal regression or win.
# ── THE GATE. Before anything else is measured. Both sides, because an A/B where one side
- # is a development build is worse than no A/B: the 3.2x inflation lands entirely on one arm
# and reads as a colossal regression or win. ────────────────────
verdict = None
for side in sides:
@@ -859,15 +792,10 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
from .runtime.ab import origin_scoped
init_scripts = []
- # ONE PROVIDER PER ORIGIN, because the provider lives in the BACKEND and localStorage is
- # per-origin. An ATTACHED null control is `--attach U --attach-b U`, which `is_null_control`
- # accepts on purpose: two sides, one Unsloth. Registering per SIDE there registers the pacer
- # twice against the same backend, and `lifecycle.register_provider` is idempotent by DISPLAY
- # NAME -- so the second registration DELETES the id the first side's seed script had already
- # captured, and that script keeps selecting the dead id on every navigation it wins (the
- # order init scripts run in is not defined, and `StudioAuth.rotate` re-adds them mid-run).
- # A base cell that boots with a deleted provider selected renders "No longer offered" and
- # throws `Connection not found` without ever asking for a completion.
+ # ONE PROVIDER PER ORIGIN: the provider lives in the backend and localStorage is per-origin, so
+ # an attached null control is two sides on one Unsloth. `register_provider` is idempotent by
+ # DISPLAY NAME, so registering per side deletes the id the first side's seed script captured.
+ # `is_null_control` is what recognises it.
registered: dict = {}
for index, side in enumerate(sides):
side_install = installs[index][0]
@@ -878,16 +806,15 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
)
_log(f" {side['label']}: authenticated as {side_auth.username}")
- # BOTH sides register the SAME pacer, so the bytes on the wire are identical by
- # construction rather than by two configurations that are meant to agree.
+ # BOTH sides register the SAME pacer, so the bytes on the wire are identical by construction
+ # rather than by two configurations meant to agree.
side_origin = side["base_url"].rstrip("/")
shared = registered.get(side_origin)
if shared is None:
side_provider = pacer_provider(pacer.base_url, [model_id])
- # Registered in the BACKEND, and the id it assigns is what the selection names. See
- # lifecycle.register_provider: a provider that exists only in localStorage renders
- # in the picker as "No longer offered" and send throws `Connection not found`
- # without ever asking for a completion.
+ # Registered in the BACKEND, and the id it assigns is what the selection names: a provider
+ # existing only in localStorage renders as "No longer offered" and send throws
+ # `Connection not found`.
register_provider(side["base_url"], side_auth, side_provider)
side_checkpoint = external_checkpoint_id(side_provider, model_id)
registered[side_origin] = (side_provider, side_checkpoint)
@@ -911,25 +838,22 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
auth_now,
[provider],
extra_local_storage = {
- # The SELECTION, without which nothing is ever generated. See
- # lifecycle.external_checkpoint_id.
+ # The SELECTION, without which nothing is ever generated.
+ # See lifecycle.external_checkpoint_id.
"unsloth_chat_last_external_checkpoint": cp,
"unsloth_chat_connections_enabled": "true",
},
),
)
- # Origin-gated even in the single-target case, so the one-build and two-build paths are
- # the same code and the gate cannot rot while unused.
+ # Origin-gated even in the single-target case, so the one-build and two-build paths are the same
+ # code and the gate cannot rot while unused.
side["seed_script"] = _side_seed
init_scripts.append(_side_seed(side_auth))
if getattr(args, "inject_stream_cost_ms", None) and side is not sides[0]:
- # VALIDATION, not a measurement mode. Burns a known amount of main-thread time per
- # SSE chunk on the TREATMENT side only, so an A/B whose two arms are otherwise the
- # same build has a known answer. It is origin-gated like the seed above, because a
- # context init script fires on every document and burning on both sides would
- # inject the cost into the control as well and read back a recovery of zero.
+ # VALIDATION, not a measurement mode: burns known main-thread time per SSE chunk on the TREATMENT
+ # side only, so an otherwise-identical A/B has a known answer. Origin-gated, or recovery reads 0.
from .instruments.selfcheck import stream_cost_injection_init_script
init_scripts.append(
origin_scoped(
@@ -944,38 +868,16 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
auth = sides[0]["auth"]
- # ONE `add_init_script` CALL PER SCENE SCRIPT, which is what this always did.
- #
- # These were briefly concatenated into a single script, on the reasoning that Playwright
- # says "the order of evaluation of multiple scripts installed via
- # browserContext.addInitScript() and page.addInitScript() is not defined", and surfaces.js
- # reads what dom.js and parity.js put on `window.__sb`. The reasoning is sound and the
- # change was a REGRESSION, caught by CI: joining them means the browser parses and
- # evaluates the three as one unit, so a throw or a parse error in any one of them stops
- # the other two running as well. Separate scripts are separate failure domains, and on the
- # CI fixture that difference cost `message_menu` its More button and turned a green job
- # red. An ordering guarantee is not worth trading fault isolation for when the ordering
- # has been correct in practice for the life of the file.
- #
- # surfaces.js is loaded unconditionally, even without --surfaces. It defines selectors and
- # never runs on its own. Making the page's JS depend on a CLI flag would mean the flag
- # changes what is on the page during the FILM as well, and the film's numbers must not
- # depend on whether a later phase was asked for.
+ # ONE `add_init_script` CALL PER SCENE SCRIPT. Concatenating them was a regression caught by CI:
+ # one parse error stops all three. Separate scripts are separate failure domains. surfaces.js
+ # loads even without --surfaces, so the film cannot depend on whether a later phase was asked for.
init_scripts.append(resources.read_text("scene/dom.js"))
init_scripts.append(resources.read_text("scene/parity.js"))
init_scripts.append(resources.read_text("scene/surfaces.js"))
- # AN EXTERNAL PROBE OR ABLATION ARM, its own script like the three above. One environment
- # variable rather than a CLI flag per experiment, and WITH THE VARIABLE UNSET NOTHING IS
- # APPENDED: the run is byte-identical to a run of a tree that does not have this hook. That
- # property is the point. A potency probe perturbs the page it observes, so the probe run and
- # the scored run have to be different runs of one harness, and the only safe way to arrange
- # that is for the probe to be absent by default.
- #
- # BECAUSE THE ORDER IS UNDEFINED, A PROBE MUST BE SELF-CONTAINED. It cannot assume the
- # scene scripts have run, so it cannot read `window.__sb` at install time. That is a real
- # constraint and it is written down in CONTRIBUTING-perf.md rather than papered over with
- # a guarantee this file is not in a position to give.
+ # AN EXTERNAL PROBE OR ABLATION ARM, in its own script. One env var rather than a flag per
+ # experiment, and WITH IT UNSET NOTHING IS APPENDED, so the probe run and the scored run are
+ # different runs. Script order is undefined, so a probe cannot read `window.__sb` at install.
if extra_init:
init_scripts.extend(_probe_init_scripts(extra_init, extra_init_source))
_log(
@@ -993,11 +895,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
bundle = browser_mod.launch(
args.engine, headless = not args.headed, init_scripts = init_scripts, log = _log
)
- # THE RETURN PATH for a probe installed by the hook above. Unsloth ships `connect-src
- # 'self'`, so a beacon to a collector on another port is blocked by CSP before it leaves
- # the page, and the payload schema has no row for a one-off probe. The console is what is
- # left. Lines are filtered on a caller-supplied prefix so they can be recovered from the
- # run log by exact match, and so a probe cannot drown the log in the app's own traffic.
+ # THE RETURN PATH for a probe installed above: Unsloth ships `connect-src 'self'` so a beacon is
+ # blocked by CSP, and the schema has no row for a one-off probe. Filtered on a caller prefix.
console_prefix = os.environ.get("SBENCH_PAGE_CONSOLE")
if console_prefix:
bundle.page.on(
@@ -1005,11 +904,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
lambda m: _log(f" [page] {m.text}") if m.text.startswith(console_prefix) else None,
)
if extra_init:
- # A probe that throws on load is the same silence as a probe that was never installed,
- # and the console filter above cannot show it because a failing probe never gets as far
- # as printing its own prefix. Attached only when a probe was asked for, so an ordinary
- # run is unchanged. `console.error` from the isolation wrapper arrives here as a
- # console message rather than a page error, so both channels are listened to.
+ # A probe that throws on load is the same silence as one never installed, and the console filter
+ # cannot show it. Both channels are listened to: a wrapper's `console.error` is not a page error.
bundle.page.on("pageerror", lambda err: _log(f" [page error] {err}"))
bundle.page.on(
"console",
@@ -1023,23 +919,16 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
time.sleep(1.0)
procs = new_roots(os.getpid(), procs_before)
- # AN INIT SCRIPT IS A SNAPSHOT AND THE RUN OUTLIVES IT. The scripts above carry the access
- # token this run logged in with, they re-run on EVERY navigation, and there is one
- # navigation per cell -- so after `StudioAuth` re-mints a token, the next `page.goto` would
- # write the DEAD one back over whatever the SPA had rotated to for itself, which is the
- # failure this fixes wearing a different hat. Playwright has no way to replace an init
- # script and does not define the order they run in, so the seed script decides for itself
- # whether to write: it compares its token's `exp` against the one in storage and defers to
- # the later one. See `seed_init_script`. At most one extra script per hour of run.
+ # AN INIT SCRIPT IS A SNAPSHOT AND THE RUN OUTLIVES IT: the scripts carry this run's access token
+ # and re-run on every navigation, so after a re-mint the next `goto` would write the dead one
+ # back. Playwright cannot reorder them, so the seed script defers to the later `exp`.
for side in sides:
side["auth"].on_rotate = lambda auth_now, side = side: bundle.context.add_init_script(
side["seed_script"](auth_now)
)
- # WHAT WAS IN THE FILE BEFORE THIS SESSION COULD WRITE A BYTE. `make_context` opens the
- # `Recorder` on the next line, and the only check left after that point -- the ladder ratio,
- # which is not known until the corpus has been measured -- has to be able to put the payload
- # back the way it found it. See `rollback_session_rows`.
+ # WHAT WAS IN THE FILE BEFORE THIS SESSION COULD WRITE A BYTE: `make_context` opens the Recorder
+ # next, and the ladder-ratio check has to be able to put the payload back.
mark = payload_mark(paths.payload_jsonl)
ctx, session = make_context(
bundle,
@@ -1049,14 +938,13 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
paths,
_log,
procs,
- # ADOPTED, NOT TAKEN AGAIN. `run()` has held this directory since before the payload
- # was archived; the `Recorder` writes its session id into the marker it already holds.
+ # ADOPTED, NOT TAKEN AGAIN: `run()` has held this directory since before the payload was
+ # archived, so the `Recorder` writes its session id into the marker it already holds.
out_lock = out_lock,
)
rec = ctx.recorder
- # The ladder this run PROMISED, resolved before it is announced. Recorded rather than
- # left to be re-derived from the tier later: `--report` reads it back to decide which rungs
- # a payload owes, and a `--rungs` override the payload did not carry made that answer wrong.
+ # The ladder this run PROMISED, recorded rather than re-derived: `--report` reads it back to
+ # decide which rungs a payload owes, and a `--rungs` override it did not carry made that wrong.
rungs = planned_rungs(args)
meta_row = {
"row_type": "run_meta",
@@ -1064,22 +952,16 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
"tool_version": TOOL_VERSION,
"corpus_hash": corpus.corpus_hash,
"studio_ref": args.branch if owns_studio else f"attached:{base_url}",
- # WHICH COMMIT THAT REF NAMED, so a later `--resume` can tell a continuation from
- # a branch that moved. Empty for an attached Unsloth, whose build is not visible
- # from here. See `commit_problems`.
+ # WHICH COMMIT THAT REF NAMED, so a later `--resume` can tell a continuation from a branch that
+ # moved. Empty for an attached Unsloth. See `commit_problems`.
"studio_commit": sides[0].get("commit") or "",
"bundle": verdict.as_dict(),
"platform": {
"system": platform.system(),
"machine": platform.machine(),
- # WHICH PHYSICAL MACHINE, which `system` and `machine` do not answer between
- # them. `platform.machine()` is the ARCHITECTURE -- "the machine type, e.g.
- # 'AMD64'" -- so two ordinary Linux x86_64 boxes report `Linux` and `x86_64`
- # alike, and every number this tool produces is machine-local: `report/render.py`
- # says so in its own words, "machine-local; does not travel between machines".
- # `platform.node()` is the computer's network name, which is the only thing
- # recorded here that differs between two such hosts. Empty when it cannot be
- # determined, exactly as the other two are.
+ # WHICH PHYSICAL MACHINE, which `system` and `machine` do not answer: `platform.machine()` is the
+ # ARCHITECTURE, so two Linux x86_64 boxes report identically. `platform.node()` is the only
+ # recorded thing that differs; empty when undeterminable.
"node": platform.node(),
"python": sys.version.split()[0],
"engine": bundle.engine,
@@ -1092,60 +974,35 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
"tier_rungs": TIER_RUNGS[args.tier],
"reps": args.reps,
"instrument_level": args.instrument_level,
- # In the payload, not only in the log. Two runs with different fixtures are
- # not comparable, and a fixture difference that is not recorded is one a later
- # reader has no way to notice before quoting a ratio across it.
+ # In the payload, not only in the log: two runs with different fixtures are not comparable, and
+ # an unrecorded difference is one a later reader cannot notice before quoting a ratio across it.
"stream_tail_chars": args.stream_tail_chars,
"corpus_dollars": bool(args.corpus_dollars),
- # WHICH PROBE, IF ANY, WAS IN THE PAGE. Recorded next to the corpus hash and for
- # the same reason: a payload nobody can audit against the page it measured has to
- # be taken on trust. `null` is the normal case and the only scorable one.
+ # WHICH PROBE, IF ANY, WAS IN THE PAGE: a payload nobody can audit against the page it measured
+ # has to be taken on trust. `null` is the normal case and the only scorable one.
"probe_init_script": extra_init or None,
- # WHETHER THE PROBE RAN BEFORE THE FILM, for the same reason and with the same
- # consequence: it changes what the cell measures without moving the cell id, so
- # `--resume` needs it on the record to be able to refuse a toggle. See
- # `IDENTITY_AXES`.
+ # WHETHER THE PROBE RAN BEFORE THE FILM: it changes what the cell measures without moving the
+ # cell id, so `--resume` needs it on the record to refuse a toggle.
"click_probe": bool(getattr(args, "click_probe", False)),
- # AN ARM RUNNING THIS IS NOT A MEASUREMENT OF THE BUILD, and the payload has to
- # say so itself. A reader who finds a treatment arm 40% slower has no other way
- # to discover that the harness put the 40% there on purpose, and `--resume` reads
- # it back as an identity axis so a calibration cannot be continued as an ordinary
- # run or the other way round. See `IDENTITY_AXES`.
+ # AN ARM RUNNING THIS IS NOT A MEASUREMENT OF THE BUILD, and the payload has to say so: nothing
+ # else tells a reader the harness put the 40% there. `--resume` reads it as an identity axis.
"inject_stream_cost_ms": getattr(args, "inject_stream_cost_ms", None),
- # WHICH BROWSER ACTUALLY RENDERED THIS, which the engine name does not settle.
- #
- # `--headed` flips `headless` at launch, and since Playwright 1.57 the two modes
- # default to DIFFERENT BINARIES -- headed resolves to `chrome`, headless to
- # `chrome-headless-shell`. This workflow pins `playwright>=1.45,<2`, so that split is
- # in range. It is not a mode flag with a mode-shaped effect: one report measures
- # headless-shell at two to three times the processing time of chrome-stable, headless
- # falls back to software rendering for GPU-accelerated work, and the compositor still
- # ticks at 60 Hz in headless unless BeginFrameControl is driven by hand.
- #
- # This tool measures frames, jank and time-to-settle. A payload recorded headed and one
- # recorded headless are two different renderers, and the payload has to say which it
- # was before anything compares them.
+ # WHICH BROWSER ACTUALLY RENDERED THIS, which the engine name does not settle: since Playwright
+ # 1.57 headed resolves to `chrome` and headless to `chrome-headless-shell`, which is measured at
+ # two to three times chrome-stable and falls back to software rendering. This tool measures
+ # frames and jank, so the payload must name the renderer.
+ # This workflow pins `playwright>=1.45,<2`, so the split is inside the pinned range.
"headed": bool(getattr(args, "headed", False)),
}
rec.emit(meta_row)
from .scoring import payload_rules
- # THE COMPARABILITY KEY, emitted as its own row so a prose comparison can be checked later.
- #
- # `floor_table.load` already refuses to POOL payloads across tiers and corpora, and that
- # guard works. What it cannot reach is a comparison made in PROSE between two separately
- # published runs, which is how a wrong number nearly propagated here twice.
- #
- # Computed over the run_meta row that was actually emitted, not over a second dict built
- # to look like it. A hand-copied duplicate is one more value whose provenance is not
- # stable, which is the whole failure this module exists to stop: the key has to describe
- # the payload, not a parallel description of it that can drift.
- #
- # Keyed on the COMPUTED corpus hash rather than on the harness commit, because a commit is
- # a claim about provenance and the hash is the thing itself: one tree in this campaign ran
- # a corpus that matched neither the branch's pre-fix nor its post-fix hash, silently and
- # self-consistently, and only printing the hash revealed it.
+ # THE COMPARABILITY KEY, its own row so a prose comparison can be checked later. `floor_table`
+ # refuses to pool across tiers and corpora but cannot reach a comparison made in PROSE, which
+ # nearly propagated a wrong number twice. Keyed on the COMPUTED corpus hash: one tree ran a
+ # corpus matching neither the pre-fix nor the post-fix hash, and only printing it revealed that.
+ # `floor_table.load` is the refusal.
rec.emit(
{
"row_type": "comparability",
@@ -1163,12 +1020,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
rec.gate("production_build", verdict.production, verdict.as_dict())
if args.tier == "fast":
- # Said in the log AND recorded in the payload. A fast-tier reading is a DIRECTION, not
- # a number: it runs one rung, a 47 s film and however few repetitions the caller asked
- # for, so its detection floor is wider than the standard tier's and it has no null
- # control of its own unless one is run alongside. The gate exists so the analysis layer
- # can refuse to pool a fast payload with a standard one -- a fast reading quoted against
- # a standard floor is the single most likely way this tier gets somebody a wrong answer.
+ # Said in the log AND recorded. A fast-tier reading is a DIRECTION, not a number: one rung, a 47 s
+ # film, a wider detection floor and no null control, so the analysis layer must refuse to pool it.
_log("")
_log(" FAST TIER: for iteration while you are changing something, not for reporting.")
_log(
@@ -1190,12 +1043,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
else "standard measurement protocol",
},
)
- # THE SAME SHAPE OF GATE, for the same reason. A probe forces layout on a schedule of its
- # own, so its payload is perturbed, and "the docs say probe payloads are never scored" is a
- # convention rather than a gate. A convention is what this subsystem keeps being wrong
- # about: the run still records ordinary cells and still renders an A/B table, so a probe
- # invocation on the standard tier produces something that reads exactly like a result. The
- # gate is what lets `floor_table` refuse it instead of trusting the caller to remember.
+ # THE SAME SHAPE OF GATE: a probe forces layout on its own schedule, so its payload is perturbed,
+ # and "probe payloads are never scored" is a convention rather than a gate. The run still renders
+ # an A/B table, so a probe invocation reads like a result unless `floor_table` can refuse it.
rec.gate(
"probe_free",
not extra_init,
@@ -1215,16 +1065,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
_log("")
image_path = ensure_probe_image(paths)
- # WHICH SIDE, IF ANY, IS ALLOWED THE WINDOWED READINESS GATE.
- #
- # Per side and never global, so an A/B of the shipped build against a virtualised one runs
- # the base arm on the strict full-mount gate it has always had. A flag that relaxed both
- # sides would let a base arm that failed to finish mounting sail through and be compared
- # against a treatment that did the same, which is the exact "flattering garbage" the gate
- # exists to refuse -- on both arms at once, so the ratio would still look fine.
- #
- # `windowed` was parsed and validated at the top of this function, before the installs, the
- # pacer and the browser existed. Only the application of it is left here.
+ # WHICH SIDE, IF ANY, IS ALLOWED THE WINDOWED READINESS GATE. Per side and never global: relaxing
+ # both would compare two half-mounted arms with the ratio still looking fine. Parsed earlier;
+ # only its application is left here.
for side in sides:
side["readiness_mode"] = MODE_WINDOWED if side["label"] in windowed else MODE_FULL
if side["readiness_mode"] == MODE_WINDOWED:
@@ -1274,9 +1117,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
if args.surfaces:
_sweep_surfaces(sides, ctx, paths)
- # The ladder is sized by the ratio MEASURED on this corpus, not by the provisional 4.0.
- # Measured here, before any cell is built, because a rung named in tokens is planned in
- # characters and the ratio is what makes those the same claim.
+ # The ladder is sized by the ratio MEASURED on this corpus, not the provisional 4.0, and
+ # measured before any cell is built because a rung named in tokens is planned in characters.
cells = build_cells(
rungs,
corpus,
@@ -1292,9 +1134,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
corpus_dollars = args.corpus_dollars,
)
if args.stream_tail_chars or args.corpus_dollars:
- # Loud, because both change the fixture. A payload produced under either of them is
- # not comparable with one produced without, and the pair that says so is printed here
- # and written into the run manifest above.
+ # Loud, because both change the fixture: a payload produced under either is not comparable with
+ # one produced without.
_log(
f" FIXTURE CHANGED: stream tail {args.stream_tail_chars or 'default'}, "
f"dollars {'on' if args.corpus_dollars else 'off'}. Compare only against a run "
@@ -1304,10 +1145,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
ladder_ratio = cells[0][0].meta["ladder_chars_per_token"]
rec.gate("ladder_ratio_measured", not ladder_ratio["provisional"], ladder_ratio)
- # THE RATIO IS PART OF WHAT THE PAYLOAD MEASURED, and this is the first moment it is
- # known -- the same shape as the commit check above, and for the same reason: the
- # rungs already in the file were built at the ratio they record, the rungs still owed
- # would be built at the one measured now, and one report prints both as one ladder.
+ # THE RATIO IS PART OF WHAT THE PAYLOAD MEASURED, and this is the first moment it is known: rungs
+ # already in the file were built at the ratio they record, rungs still owed at the one measured
+ # now, and one report prints both as one ladder.
if args.resume:
ratio_issues: list = []
for recorded in recorded_identities(paths.payload_jsonl):
@@ -1315,12 +1155,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
if problem not in ratio_issues:
ratio_issues.append(problem)
if ratio_issues:
- # AND NOTHING THIS SESSION WROTE SURVIVES THE REFUSAL. Unlike the commit check
- # above, this one runs after the `Recorder` has opened the payload, so refusing
- # is not enough on its own: `run_meta` is already in the file and
- # `recorded_ladder` folds every one of them, so a refused
- # `--resume --rungs 1K,10K` would leave 10K promised and unrecorded in a
- # payload that was complete. See `rollback_session_rows`.
+ # AND NOTHING THIS SESSION WROTE SURVIVES THE REFUSAL: this runs after the Recorder opened the
+ # payload, so a refused `--resume --rungs 1K,10K` would leave 10K promised and unrecorded.
rec.close()
rollback_session_rows(paths.payload_jsonl, mark)
raise SystemExit(
@@ -1352,9 +1188,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
]
work = interleave(cells, targets)
if not order_is_balanced(work):
- # Said out loud rather than silently absorbed: with an odd number of reps one side
- # always runs first, so anything drifting monotonically through the session lands on
- # the other one instead of cancelling.
+ # Said out loud rather than absorbed: with an odd number of reps one side always runs first, so
+ # anything drifting monotonically through the session lands on the other rather than cancelling.
_log(
" WARNING: the run order is not balanced (use an even --reps). Linear drift "
"within the session is charged to whichever side runs second."
@@ -1364,11 +1199,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
"row_type": "ab_plan",
"base_ref": sides[0]["ref"],
"treatment_ref": sides[1]["ref"],
- # WHICH SERVER THE TREATMENT WAS, when the caller attached one. `run_meta`
- # already records the base that way in `studio_ref`; without the same for the
- # treatment a resume can only compare the `--ab` label, which says nothing about
- # the build that answered on the port. See `requested_identity`. Empty when this
- # run installed the treatment itself: then the ref above is the identity.
+ # WHICH SERVER THE TREATMENT WAS, when the caller attached one: without it a resume compares only
+ # the `--ab` label, which says nothing about the build that answered. Empty when we installed it.
"treatment_url": "" if sides[1]["owns"] else sides[1]["base_url"],
# The treatment's half of `studio_commit`. Same reason, same emptiness rule.
"treatment_commit": sides[1].get("commit") or "",
@@ -1380,10 +1212,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
work = [(None, cell, plan) for cell, plan in cells]
if done:
- # AT PAIR GRANULARITY. An A/B pair whose two arms are not both recorded is re-run
- # whole, so the resumed session never measures one arm on its own -- see
- # `ab.skippable_cells`. For a run without --ab every pair holds one cell and this is the
- # set it already was.
+ # AT PAIR GRANULARITY: an A/B pair whose two arms are not both recorded is re-run whole, so a
+ # resumed session never measures one arm on its own.
from .runtime.ab import skippable_cells
done = skippable_cells(work, done)
_log(f" resuming: {len(done)} cells already in {paths.payload_jsonl.name}")
@@ -1417,9 +1247,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l
rec.close()
if ab_ref:
- # The cells THIS session was asked to measure. A resumed A/B skips whole pairs or none
- # (`ab.skippable_cells`), so this is either every planned cell or none of them, and it is
- # what `_render_ab` checks the payload against before it is allowed to print a verdict.
+ # The cells THIS session was asked to measure: a resumed A/B skips whole pairs or none, so this
+ # is every planned cell or none, and it is what `_render_ab` checks the payload against.
+ # The skip set comes from `ab.skippable_cells`.
_render_ab(
paths,
sides,
@@ -1462,8 +1292,8 @@ def _sweep_surfaces(sides: list, ctx, paths) -> None:
recorder = ctx.recorder,
)
except Exception as exc: # noqa: BLE001
- # Recorded as a failed gate rather than swallowed. A sweep that raised and a sweep
- # that found nothing look identical in a payload that only carries the rows.
+ # Recorded as a failed gate rather than swallowed: a sweep that raised and a sweep that found
+ # nothing look identical in a payload that only carries rows.
_log(f" the surface sweep failed: {type(exc).__name__}: {exc}")
ctx.recorder.gate(
f"surface_sweep:{label}", False, {"error": f"{type(exc).__name__}: {exc}"}
@@ -1476,9 +1306,8 @@ def _sweep_surfaces(sides: list, ctx, paths) -> None:
out = paths.out / f"surfaces_{label}.md"
out.write_text(text, encoding = "utf-8")
_log(f"surface coverage manifest written to {out}")
- # PASSES only when every non-conditional surface was reached AND the digests were scoped.
- # An unscoped sweep reports one page-wide digest per surface, which agrees everywhere for
- # reasons that have nothing to do with the surfaces.
+ # PASSES only when every non-conditional surface was reached AND the digests were scoped: an
+ # unscoped sweep reports one page-wide digest per surface, which agrees everywhere.
passed = manifest["not_reached_hard"] == 0 and manifest["digests_scoped"]
ctx.recorder.gate(f"surface_sweep:{label}", passed, manifest)
@@ -1595,10 +1424,8 @@ def _render_ab(
out = paths.out / "ab.md"
- # NO TABLE AT ALL for a probe run, rather than a table with a warning printed above it. The
- # warning scrolls off; `ab.md` sits in the output directory and gets pasted into a pull
- # request. This is the same refusal `--report` and `floor_table` make, on the same evidence,
- # and it is the entry point that actually runs at the end of every session.
+ # NO TABLE AT ALL for a probe run, rather than a table with a warning above it: the warning
+ # scrolls off while `ab.md` gets pasted into a pull request.
from .scoring.from_payload import probe_scripts
probes = probe_scripts(records)
@@ -1610,11 +1437,8 @@ def _render_ab(
)
_log("")
_log(reason)
- # OVERWRITTEN, not left alone and not deleted. `--resume` reuses the output directory, so
- # an `ab.md` from an earlier clean run of the same directory would survive this refusal
- # and sit at the standard artifact path, where it reads as this run's result. Deleting it
- # would leave whoever opens the path with nothing to explain the absence, so the file is
- # replaced by the refusal itself.
+ # OVERWRITTEN, not left alone and not deleted: `--resume` reuses the directory, so an earlier
+ # clean `ab.md` would read as this run's result, while deleting it explains nothing.
stale = paths.out / "ab.md"
if stale.exists():
stale.write_text(f"# No A/B table\n\n{reason}\n", encoding = "utf-8")
@@ -1622,27 +1446,16 @@ def _render_ab(
_log("")
return
- # A FULLY RESUMED A/B MEASURED NOTHING OF ITS OWN. `--resume` against a finished output skips
- # every cell and is a success, but the ratio is scoped to THIS session, so re-rendering here
- # replaced a real table with NO READING and exited 0 while doing it. The run that measured
- # keeps its report; a run with no prior table still gets one, since there is nothing to lose.
- #
- # AFTER THE PROBE REFUSAL, NOT BEFORE IT. The payload the resume keeps is the payload this
- # check is deciding on behalf of, and a probed one is not scorable no matter which session
- # recorded it. Run first, this returned early for the one sequence that needs the refusal
- # most: a fresh PROBE run into a directory that already holds a clean `ab.md`, killed after
- # its last cell was fsynced and before it rendered -- `archive_payload` moves `payload.jsonl`
- # and nothing else, so the clean table stays at the standard artifact path -- and then
- # resumed, which finds every cell complete, records none of its own and left that table
- # standing over an unscorable probe payload.
+ # A FULLY RESUMED A/B MEASURED NOTHING OF ITS OWN: the ratio is scoped to THIS session, so
+ # re-rendering replaced a real table with NO READING and exited 0. AFTER the probe refusal, or
+ # this returns early for exactly the sequence that needs the refusal most.
if not any(r.get("row_type") == "cell" and r.get("session_id") == session_id for r in records):
if out.exists():
_log(f"\nno cell ran in this session; keeping the A/B table already at {out}")
return
- # Detected, not declared. `--ab main` IS a null control whether or not the caller says so, and
- # a null control that renders as an ordinary A/B invites somebody to quote "7.7% faster" from a
- # build compared with itself. See `is_null_control`.
+ # Detected, not declared: `--ab main` IS a null control whether the caller says so or not, and
+ # one rendered as an ordinary A/B invites somebody to quote a build compared with itself.
is_null = is_null_control(sides)
label = _ab_label(sides, is_null)
@@ -1674,8 +1487,8 @@ def _render_ab(
out.write_text(text, encoding = "utf-8")
_log(f"A/B table written to {out}")
if missing:
- # A noise floor derived from a partial null control is a number about the cells that did
- # not die, quoted afterwards at every real A/B on this machine.
+ # A noise floor derived from a partial null control is a number about the cells that did not
+ # die, quoted afterwards at every real A/B on this machine.
_log("the plan did not complete, so no noise floor is derived from it")
return
if is_null:
@@ -1695,103 +1508,51 @@ def _render_ab(
)
-#: THE PAYLOAD IDENTITY. The axes that decide whether a cell already in a payload measures the
-#: same thing a new invocation is asking for.
-#:
-#: `cell_id` is `r{rung}.{arm}.rep{rep}` and nothing else, so every one of these can change under a
-#: cell id that stays identical: the tier picks the FILM (the standard film runs 243 s, the quick
-#: one 77.5 s and the fast one 47 s, with different budgets), the cadence picks the rate the reply
-#: streams at, the instrument level decides how much of the number is the instrument, the corpus
-#: hash is the fixture, the engine is what rendered every frame, and the two refs are the builds
-#: under test.
-#:
-#: `rungs` and `reps` are deliberately NOT here. Resuming with more repetitions or another rung is
-#: a legitimate continuation -- it ADDS cells rather than reinterpreting the ones already recorded.
+#: THE PAYLOAD IDENTITY: the axes deciding whether a recorded cell measures what a new invocation
+#: asks for. `cell_id` is only `r{rung}.{arm}.rep{rep}`, so tier, cadence, instrument level,
+#: corpus, engine and both refs can change under one id. `rungs`/`reps` are NOT here: they ADD
+#: cells rather than reinterpreting recorded ones.
IDENTITY_AXES = (
"tier",
"cadence",
"engine",
- # THE INSTRUMENTS THEMSELVES. `TOOL_VERSION` is bumped when what an instrument MEASURES
- # changes and for no other reason -- this commit bumped it to 0.2.0 because
- # `reasoning_toggle.open_ms` now terminates on a settled mount rather than on the `data-state`
- # flip, which makes it a different quantity under the same name. None of that moves a cell id,
- # so `--resume` into a half-finished 0.1.0 payload from an upgraded tree skipped the completed
- # cells and appended the rest measured by 0.2.0: one ladder built from two instruments, which
- # is the same failure the tier and the corpus are on this list for. `merged_run_meta` names the
- # mixture afterwards, but only `--compare` and a floor-gated `floor_table.render` call it --
- # plain `--report` takes the FIRST header and pools the two quantities without a word. Refused
- # here instead, before anything is installed and before anything is measured.
+ # THE INSTRUMENTS THEMSELVES: `TOOL_VERSION` moves only when what an instrument MEASURES changes,
+ # and none of that moves a cell id, so `--resume` appended 0.2.0 measurements under 0.1.0 ids.
+ # Plain `--report` takes the FIRST header and pools the two quantities without a word.
+ # 0.2.0 was the bump: `reasoning_toggle.open_ms` now ends on a settled mount, not the
+ # `data-state` flip.
"tool_version",
"instrument_level",
"corpus_hash",
"studio_ref",
"treatment_ref",
"treatment_url",
- # Added by this branch with `--stream-tail-chars` / `--corpus-dollars`: both change the reply
- # the cell streams without moving its id, so they belong to the identity for the same reason
- # the tier does. Recorded already, so no schema change and a payload from before them reads
- # back as the defaults (see `HISTORICAL_DEFAULTS`).
+ # `--stream-tail-chars` / `--corpus-dollars`: both change the reply the cell streams without
+ # moving its id. A payload from before them reads back as the defaults.
"stream_tail_chars",
"corpus_dollars",
- # `--click-probe`, whose own help text says it "makes the cell's timings incomparable with a
- # cell that did not run it". That is the definition of an identity axis: the probe runs a full
- # `page.click`, a real mouse click, a dispatch, a focus and a hover over the thread before the
- # film starts, and the cell then records a `composer_click_ms` measured on a composer those
- # paths have already been through plus a `click_attribution` block a cell without the flag
- # does not have at all. None of it moves the cell id, so a resume that toggles the flag skips
- # the completed cells and appends the rest under the same ids -- the one ladder built from two
- # films this check exists to refuse.
+ # `--click-probe`, whose help text says it makes timings incomparable: it drives a real click,
+ # focus and hover before the film and adds a `click_attribution` block, none of it in the id.
"click_probe",
- # `--inject-stream-cost-ms`, on the same rule and for a larger perturbation than either of
- # those two. It burns a known amount of main-thread time per SSE chunk on the TREATMENT arm,
- # so a treatment cell recorded with it is a different reading from one recorded without it,
- # under a cell id that cannot tell -- the id carries the rung, the arm and the repetition and
- # stops there. Off it, `--resume` had two ways to lie about a calibration: against a FINISHED
- # uninjected payload every pair is skippable, the run exits 0 having measured nothing, and the
- # recovery gate is answered from cells that were never injected; against a HALF-FINISHED
- # injected one the resume drops the flag and the ladder ends up part injected and part not.
- # Both land as a recovery fraction near zero, which reads as "the metric is blind" and is the
- # single verdict this flag exists to make trustworthy.
+ # `--inject-stream-cost-ms`, a larger perturbation on the TREATMENT arm only. Off this list, a
+ # resume could answer the recovery gate from cells never injected, which reads as a blind metric.
"inject_stream_cost_ms",
- # `SBENCH_EXTRA_INIT_SCRIPT`, the external probe. Unlike the axes above it cannot end in a
- # wrong NUMBER -- `refuse_if_probed` reads every `run_meta` in the file and every scoring entry
- # point calls it, so one probed session makes the whole payload unscorable and there is no flag
- # to override that. It is here for what the refusal costs instead. The variable is an
- # ENVIRONMENT variable rather than a flag, so it survives in a shell after the experiment that
- # set it is over, and `--resume` into a half-finished clean payload then installs both sides,
- # runs the rungs that are still owed with the probe in the page, and appends a probed
- # `run_meta` to the file. The payload is append-only and the refusal is whole-file, so the
- # cells that were recorded cleanly before it are unscorable from then on and nothing this tool
- # offers takes it back. Refusing here costs a millisecond and happens before the first install.
+ # `SBENCH_EXTRA_INIT_SCRIPT`, the external probe. It cannot end in a wrong number, so it is here
+ # for what the refusal COSTS: an env var survives in a shell, and a resume would append a probed
+ # `run_meta` that makes every previously clean cell unscorable, irreversibly.
"probe_init_script",
- # `--headed`. On the same rule as `engine` directly above the others, and for the same reason
- # the engine is there at all: it decides which browser binary renders the film. Since
- # Playwright 1.57 headed and headless resolve to different executables (`chrome` against
- # `chrome-headless-shell`), headless falls back to software rendering for GPU-accelerated
- # work, and the compositor's own pacing differs. A resume that toggles it would skip the
- # completed cells and append the rest under the same ids, building one ladder from two
- # renderers -- exactly what this check exists to refuse.
+ # `--headed`, for the same reason `engine` is here: headed and headless resolve to different
+ # executables with different rendering, so toggling it builds one ladder from two renderers.
"headed",
)
-#: The axes that describe the SECOND side, which only exist when a run has one. See
-#: `identity_problems`: an A/B judged against a run that is not one may not differ on them.
+#: The axes that describe the SECOND side, which only exist when a run has one. An A/B judged
+#: against a run that is not one may not differ on them.
TREATMENT_AXES = ("treatment_ref", "treatment_url")
-#: The axes whose ABSENCE from a payload is itself a reading, and what it reads as.
-#:
-#: `identity_problems` otherwise skips an axis the payload never declared, because an axis a row
-#: never declared cannot be a difference. That is right for the axes `run_meta` has always carried:
-#: a payload missing one of those is a payload this check has nothing to say about.
-#:
-#: It is WRONG for the ones below. They arrived with the flag or the variable that sets them, so a
-#: payload written before them did not decline to record a value -- there was no way to ask for
-#: anything but the default, and it ran under `stream_tail_chars = None`, `corpus_dollars = False`,
-#: `click_probe = False`, `inject_stream_cost_ms = None` and `probe_init_script = None` by
-#: construction. Skipping them therefore accepted `--resume --stream-tail-chars 24000` against such
-#: a payload, skipped its completed cells, and recorded the rest under a different streamed fixture
-#: beneath the same cell ids: one ladder built from two films, which is what this check exists to
-#: refuse. Absence proves the value here, so it is read as the value.
+#: The axes whose ABSENCE from a payload is itself a reading. `identity_problems` otherwise skips
+#: an undeclared axis, which is wrong for these: they arrived with the flag that sets them, so an
+#: older payload ran under the default by construction and one ladder was built from two films.
HISTORICAL_DEFAULTS = {
"stream_tail_chars": None,
"corpus_dollars": False,
@@ -1801,33 +1562,25 @@ HISTORICAL_DEFAULTS = {
"headed": False,
}
-#: THE RATIO THE LADDER WAS SIZED BY, carried on every cell's `meta` by `session.build_cells`.
-#: Not in `IDENTITY_AXES`: those are checked by `prepare_payload` before anything is installed,
-#: and this one is not known until `build_cells` has measured the corpus. Checked instead by
-#: `ladder_ratio_problems`, on the same footing as `COMMIT_AXES` -- late enough to have the answer,
-#: early enough that nothing has been measured under it.
+#: THE RATIO THE LADDER WAS SIZED BY, carried on every cell's `meta`. Not in `IDENTITY_AXES`,
+#: which are checked before anything is installed, because this is not known until `build_cells`
+#: has measured the corpus; `ladder_ratio_problems` checks it instead.
+# Carried by `session.build_cells`.
LADDER_RATIO_AXIS = "ladder_chars_per_token"
-#: How close two ratios must be to be the same ratio. `measure_chars_per_token` rounds its answer
-#: to three decimals and `PROVISIONAL_CHARS_PER_TOKEN` is exact, so every value that reaches a
-#: payload sits on a 0.001 grid: half a grid step separates "the same measurement, re-read through
-#: JSON" from "a different one". Wide enough that 5.0 never refuses 5.0 for a float's sake, narrow
-#: enough that the case this exists for -- a provisional 4.0 against tiktoken's 3.336 on this
-#: corpus, a fifth of the character load of every rung -- can never be inside it.
+#: How close two ratios must be to be the same ratio. Payload values sit on a 0.001 grid, so half
+#: a step separates a re-read from a different measurement: 5.0 never refuses 5.0, and a
+#: provisional 4.0 can never be inside tiktoken's 3.336.
+# `measure_chars_per_token` rounds its answer onto that grid.
LADDER_RATIO_TOLERANCE = 5e-4
-#: The arm a run without `--ab` records every cell under. `Cell.arm` in `runtime.types`; an A/B
-#: overwrites it with the target's label, `base` or `treatment`. Named here because it is what
-#: tells `recorded_identities` which of the two a session already in the payload was.
+#: The arm a run without `--ab` records every cell under; an A/B overwrites it with base or
+#: treatment. Named here because it tells `recorded_identities` which a recorded session was.
+# `Cell.arm` in `runtime.types`.
SINGLE_ARM = "A0"
-#: THE BUILD, as opposed to the name it was asked for by. A ref is a POINTER: `main`, a topic
-#: branch and a movable tag all resolve afresh on every install, and `checkout_ref` is the only
-#: thing that ever knows which commit one named. So these are not in `IDENTITY_AXES` and are not
-#: checked by `prepare_payload`: that refusal is deliberately made BEFORE anything is installed,
-#: and a commit cannot be known before the fetch that resolves it. They are checked instead by
-#: `commit_problems`, once the sides are up and before the first cell -- late enough to have the
-#: answer, early enough that nothing has been measured under it. See `run`.
+#: THE BUILD, as opposed to the name it was asked for by. A ref is a POINTER that resolves afresh
+#: on every install, so `commit_problems` checks these once the sides are up, not earlier.
COMMIT_AXES = ("studio_commit", "treatment_commit")
@@ -1844,41 +1597,33 @@ def requested_identity(args, ab_ref, corpus_hash: str) -> dict:
return {
"tier": args.tier,
"cadence": args.cadence,
- # THE INSTRUMENTS THIS TREE CARRIES, as a module constant rather than anything the caller
- # can ask for. A payload recorded by another version was measured by other instruments.
+ # THE INSTRUMENTS THIS TREE CARRIES, a module constant rather than anything the caller can ask
+ # for: a payload recorded by another version was measured by other instruments.
"tool_version": TOOL_VERSION,
- # THE ENGINE THAT WILL RENDER, not the flag that asked for it. `--engine` defaults to
- # nothing and `browser.launch` resolves the platform's desktop webview family, so the
- # requested value has to be resolved the same way `launch` resolves it -- otherwise the
- # commonest engine change of all, one run naming `--engine chromium` and the resume
- # naming none, reads as no difference at all. Resolvable here, before anything is
- # installed, because `default_engine` reads `platform.system()` and nothing else.
+ # THE ENGINE THAT WILL RENDER, not the flag that asked for it: `--engine` defaults to nothing and
+ # `browser.launch` resolves the platform's webview family, so the request must resolve the same
+ # way or the commonest engine change of all reads as no difference.
+ # Resolvable before anything is installed: `default_engine` reads `platform.system()` and
+ # nothing else.
"engine": getattr(args, "engine", "") or default_engine()[0],
"instrument_level": args.instrument_level,
"corpus_hash": corpus_hash,
"studio_ref": base_ref,
"treatment_ref": ab_ref or "",
- # THE ATTACHED TREATMENT IS THE SERVER, NOT THE LABEL. `studio_ref` folds an attached base
- # down to its URL because that is the only thing about it this harness can see; the
- # treatment carried nothing but the name typed after `--ab`, and that name is free. So
- # `--attach A --attach-b B --ab fix --resume`, re-run later against `--attach-b C`, passed
- # the identity check, skipped every completed treatment cell and reported B's measurements
- # as C's result. Empty for a treatment this run installs itself, whose ref IS its identity.
+ # THE ATTACHED TREATMENT IS THE SERVER, NOT THE LABEL: carrying only the free-form name let
+ # `--attach-b B` pass the identity check against a later `--attach-b C` and report B's numbers
+ # as C's. Empty for a treatment this run installs itself.
+ # The field is `studio_ref`; the resolved build is `requested_identity`.
"treatment_url": attach_b if (ab_ref and attach_b) else "",
"stream_tail_chars": args.stream_tail_chars,
"corpus_dollars": bool(args.corpus_dollars),
"click_probe": bool(getattr(args, "click_probe", False)),
"inject_stream_cost_ms": getattr(args, "inject_stream_cost_ms", None),
- # Spelled the same way `run_meta` records it, through `bool`, so the recorded False and
- # the requested value are the same type. Adding the axis without this line made every
- # resume refuse itself -- the payload declared `False` and the request declared `None` --
- # which is the failure mode an identity axis has when it is only half wired up.
+ # Spelled the same way `run_meta` records it, through `bool`, so the recorded False and the
+ # requested value are the same type; without this every resume refused itself.
"headed": bool(getattr(args, "headed", False)),
- # FROM THE ENVIRONMENT, because that is where this one is asked for: the probe hook is a
- # variable rather than a flag on purpose, so `args` never sees it and there is nothing to
- # read it from. Spelled exactly as `run_meta` records it -- the path as it was given -- for
- # the same reason `studio_ref` is, so the requested value and the recorded one compare
- # without a second convention to keep in step.
+ # FROM THE ENVIRONMENT, because the probe hook is a variable rather than a flag, so `args` never
+ # sees it. Spelled exactly as `run_meta` records it, so requested and recorded compare directly.
"probe_init_script": os.environ.get("SBENCH_EXTRA_INIT_SCRIPT") or None,
}
@@ -1914,17 +1659,13 @@ def recorded_identities(payload_path) -> list:
by_session[session] = {}
order.append(session)
if row_type == "cell":
- # THE MODE A SESSION MEASURED IN, declared by the cells it actually wrote rather
- # than by a header it wrote before it measured anything. A run that died between
- # `run_meta` and `ab_plan` recorded no cells and so declares no mode, and resuming
- # it is not a transition; a session holding cells declares the mode those cells
- # were recorded under, whichever row types the version that wrote them emitted.
+ # THE MODE A SESSION MEASURED IN, declared by the cells it wrote rather than a header written
+ # before it measured anything: a run that died before `ab_plan` declares no mode.
arm = str((row.get("cell") or {}).get("arm") or row.get("arm") or "")
if arm:
by_session[session].setdefault("mode", "single" if arm == SINGLE_ARM else "ab")
- # THE RATIO THE SESSION'S RUNGS WERE SIZED BY, on the same rule as the mode: read
- # off a cell the session actually wrote, so a session that recorded none declares
- # none. See `ladder_ratio_problems`.
+ # THE RATIO THE SESSION'S RUNGS WERE SIZED BY, on the same rule as the mode: read off a cell the
+ # session actually wrote. See `ladder_ratio_problems`.
sized = ((row.get("cell") or {}).get("meta") or {}).get("ladder_chars_per_token")
if isinstance(sized, dict) and sized.get("chars_per_token") is not None:
by_session[session].setdefault(
@@ -1934,13 +1675,9 @@ def recorded_identities(payload_path) -> list:
for axis in IDENTITY_AXES + COMMIT_AXES:
if axis in row:
by_session[session][axis] = row[axis]
- # THE ENGINE IS NESTED, under `run_meta.platform`, so the loop above cannot see it.
- # Read from there rather than from a new top-level field for the same reason as
- # everything else here: a payload written before this check existed still declares
- # what it rendered with, and is judged on it. `run_meta` is emitted AFTER
- # `browser.launch` returns, so what is recorded is the RESOLVED engine and a session
- # that never got a browser up declares none -- and an axis a row never declared
- # cannot be a difference, so that session still resumes.
+ # THE ENGINE IS NESTED under `run_meta.platform`, so the loop above cannot see it. `run_meta` is
+ # emitted after `browser.launch`, so the RESOLVED engine is recorded and a session that never got
+ # a browser up declares none -- and an undeclared axis cannot be a difference.
if row_type == "run_meta":
engine = str((row.get("platform") or {}).get("engine") or "")
if engine:
@@ -1954,16 +1691,10 @@ def recorded_identities(payload_path) -> list:
def identity_problems(recorded: dict, requested: dict) -> list:
"""Every axis on which a recorded session and this invocation disagree."""
problems = []
- # AN A/B AND A RUN THAT IS NOT ONE MAY NOT SHARE A PAYLOAD. The arm in the cell id ("A0"
- # against "base"/"treatment") keeps the new cells from being SKIPPED, which is why this used
- # to be waved through -- but it does not keep them out of each other's REPORT. `score_payload`
- # hands every cell to `measures_from_records`, which keys by rung and keeps the first reading,
- # and `_completion_by_rung` keeps a failure over a success at the same rung. So a single-build
- # run that died at 10K, resumed as `--ab fix` and re-measured successfully on both arms, still
- # printed `INCOMPLETE: the base died at 10K`, scored the rung 0 and reported ONSET RUNG: none,
- # while the identical resume WITHOUT `--ab` -- same crash, same repair, same cell id, so the
- # dead attempt is superseded -- scored it 82.2. The mode is part of what a payload measures,
- # so a run that changes it resumes nothing: it starts a payload of its own.
+ # AN A/B AND A RUN THAT IS NOT ONE MAY NOT SHARE A PAYLOAD. The arm in the cell id keeps the new
+ # cells from being SKIPPED but not out of each other's REPORT: `measures_from_records` keeps the
+ # first reading per rung and `_completion_by_rung` keeps a failure over a success, so a resume as
+ # `--ab fix` scored a rung 0 where the same resume without `--ab` scored 82.2.
requested_mode = "ab" if requested.get("treatment_ref") else "single"
recorded_mode = recorded.get("mode")
if recorded_mode is not None and recorded_mode != requested_mode:
@@ -1972,22 +1703,19 @@ def identity_problems(recorded: dict, requested: dict) -> list:
f"mode: the payload was recorded by a run measuring {names[recorded_mode]}, "
f"this run measures {names[requested_mode]}"
)
- # Is there a second side on BOTH sides of this comparison? When there is not, the treatment
- # axes describe a side one of them does not have, and the mode difference above is the whole
- # of what they disagree on. Decided from the ref, so that an A/B whose treatment this run
- # installed -- which records no treatment URL -- is still judged against an attached one on
- # the axis where the two of them do differ.
+ # Is there a second side on BOTH sides of this comparison? When there is not, the treatment axes
+ # describe a side one of them lacks. Decided from the ref, so an installed treatment still
+ # compares against an attached one on the axis where they do differ.
both_ab = bool(requested.get("treatment_ref")) and bool(recorded.get("treatment_ref"))
for axis in IDENTITY_AXES:
declared = axis in recorded
if declared:
got = recorded[axis]
elif axis in HISTORICAL_DEFAULTS:
- # Not declared, but not silent either: this axis postdates the payload, so the payload
- # ran under the default. See `HISTORICAL_DEFAULTS`.
+ # Not declared, but not silent either: this axis postdates the payload, so the payload ran under
+ # the default. See `HISTORICAL_DEFAULTS`.
got = HISTORICAL_DEFAULTS[axis]
else:
- # An axis this payload never declared. See `recorded_identities`.
continue
if axis in TREATMENT_AXES and not both_ab:
continue
@@ -2319,8 +2047,8 @@ def _resume_set(paths) -> set:
except ValueError:
continue
for row in latest_attempt_rows(records):
- # Only a COMPLETED cell is skipped. A cell that died is re-run, because its failure
- # may have been the machine and not the build.
+ # Only a COMPLETED cell is skipped: a cell that died is re-run, because its failure may have
+ # been the machine and not the build.
if row.get("row_type") == "cell" and row.get("completed"):
done.add(row.get("cell_id"))
return done
@@ -2338,8 +2066,8 @@ def _summarise(rows: list, paths) -> None:
for r in rows:
actions = r.get("actions") or []
ran = sum(1 for a in actions if a.get("ran"))
- # The PEAK, not the end state: the film's last two actions reopen the thread and delete
- # a message, so an end-of-film census describes a thread that is no longer there.
+ # The PEAK, not the end state: the film's last two actions reopen the thread and delete a
+ # message, so an end-of-film census describes a thread that is no longer there.
census = r.get("census_peak") or r.get("census_after") or {}
_log(
f"{r['cell_id']:<16} {r.get('assistant_chars_in_dom') or 0:>10,} "
@@ -2352,11 +2080,9 @@ def _summarise(rows: list, paths) -> None:
if not r.get("completed"):
f = r.get("failure") or {}
_log(f" FAILED: {f.get('kind')}: {str(f.get('message'))[:100]}")
- # SAID UNDER THE TABLE THAT PRINTS THEM. `elems` and `spans` come from `census_peak`, whose
- # action is chosen by a max() over per-action censuses that race their own teardown, so it is
- # not the same moment on two arms. Differencing these two columns between arms is what
- # produced a withdrawn "43% more standing DOM" claim. The columns stay because the high-water
- # mark is a useful diagnostic; the warning stays with them so it cannot be read without it.
+ # SAID UNDER THE TABLE THAT PRINTS THEM: `elems` and `spans` come from `census_peak`, whose action
+ # is chosen by a max() over censuses racing their own teardown, so it is not the same moment on
+ # two arms. Differencing them produced a withdrawn "43% more standing DOM" claim.
_log(
"\n elems/spans are census_peak: a diagnostic high-water mark, taken at whichever "
"action mounted most.\n Do NOT difference them between arms. For a cross-arm census use "
@@ -2427,9 +2153,8 @@ def report_only(args) -> int:
_log(f"no payload at {path}")
return 2
- # `--rungs` first, then the payload's own ladder, then the tier. An explicit `--tier` still
- # wins over the recording, so a reader can deliberately score a payload against another tier's
- # ladder; what is gone is the DEFAULT tier silently shortening somebody else's run.
+ # `--rungs` first, then the payload's own ladder, then the tier. An explicit `--tier` still wins;
+ # what is gone is the DEFAULT tier silently shortening somebody else's run.
recorded = [] if getattr(args, "tier_explicit", True) else recorded_ladder(path)
if args.rungs:
declared = _rung_tokens(args.rungs.split(","))
@@ -2442,20 +2167,17 @@ def report_only(args) -> int:
try:
text, ladder, _payload = build_report(path, declared)
except SystemExit as exc:
- # A REFUSAL, not a crash, and it has to reach the artefact rather than only the terminal.
- # `SystemExit` is not an `Exception`, so without this clause it would leave the process
- # before the write below. `--resume` reuses the output directory, so a `summary.md` from
- # an earlier clean report of the same directory would survive the refusal and sit next to
- # a now-probed payload, reading as its result. Same reasoning as the stale `ab.md` in
- # `_render_ab`: overwritten rather than deleted, so opening the path gives the reason.
+ # A REFUSAL, not a crash, and it has to reach the artefact rather than only the terminal:
+ # `SystemExit` is not an `Exception`, so without this clause a clean `summary.md` would sit next
+ # to a now-probed payload. Overwritten rather than deleted, as with the stale `ab.md`.
_log(str(exc))
if out.exists():
out.write_text(f"# No summary\n\n{exc}\n", encoding = "utf-8")
_log(f" a previous {out} was replaced by this refusal")
return 2
except Exception as exc: # noqa: BLE001
- # A payload that cannot be scored is reported as such rather than half-rendered: a
- # partial report is exactly the artefact that gets quoted without its caveats.
+ # A payload that cannot be scored is reported as such rather than half-rendered: a partial
+ # report is exactly the artefact that gets quoted without its caveats.
_log(f"could not build a report from {path}: {type(exc).__name__}: {exc}")
return 1
@@ -2482,21 +2204,10 @@ def compare_payloads(args) -> int:
if not path.exists():
print(f"no such payload: {path}")
return 2
- # EVERY header, not the first. `--resume` appends a second `run_meta` describing the run
- # that continued the file, and it may legitimately have extended the ladder: `rungs` is
- # deliberately outside `IDENTITY_AXES` because adding a rung ADDS cells rather than
- # reinterpreting the recorded ones. Keying on the first header therefore hashes a ladder
- # the file has since outgrown, and declares a resumed payload comparable with the one-rung
- # run it started as. `floor_table.tiers_of` and `corpora_of` were both fixed for exactly
- # this; `--compare` was the reader left behind.
- #
- # READ THE WAY EVERY OTHER READER IN THIS TOOL READS. The payload is append-only and every
- # row is flushed as it is written, so a run killed during its last append leaves valid rows
- # followed by a torn one -- the failure this format is chosen to survive, and the reason
- # `report.read_records` counts a malformed line instead of raising. `recorded_identities`
- # and `_resume_set` skip one too. Parsing it here with a bare `json.loads` made `--compare`
- # the only reader that answered an interrupted payload with a JSONDecodeError traceback
- # rather than with the check it was asked for.
+ # EVERY header, not the first: `--resume` appends a second `run_meta` and may legitimately have
+ # extended the ladder, so keying on the first hashes a ladder the file has outgrown. Read the way
+ # every other reader does -- a run killed mid-append leaves a torn last line that
+ # `report.read_records` counts rather than raising on.
records, discarded = read_records(path)
if discarded:
print(
@@ -2508,8 +2219,8 @@ def compare_payloads(args) -> int:
print(f"{path} carries no run_meta row, so it cannot be checked at all")
return 2
if conflicts:
- # A file whose own headers disagree is not one measurement, so no single key can stand
- # for it and the honest answer is a refusal rather than a token.
+ # A file whose own headers disagree is not one measurement, so no single key can stand for it
+ # and the honest answer is a refusal.
print(f"{path} disagrees with ITSELF across its own run_meta rows:")
for line in conflicts:
print(f" {line}")
@@ -2588,24 +2299,16 @@ def assert_liveness(args) -> int:
except ValueError:
problems.append("a payload line is not valid JSON")
- # A CELL THAT WAS RE-RUN IS JUDGED ON THE RUN THAT FINISHED IT, which is the same rule the
- # report and the A/B already apply and the same helper that applies it. `--resume` appends to
- # the payload it continues and re-runs the cells that died under the SAME deterministic
- # `cell_id`, so the dead attempt's `completed: false` and its NOT RUN actions are still in the
- # file after a resumed run has succeeded. Read raw, this loop found them forever: the resumed
- # run exited 0 and `--report` scored the retry, while `--assert-liveness` on the same payload
- # failed permanently on a cell that had already been re-run. A gate that cannot be satisfied
- # by fixing the run is a gate people learn to pass with `--allow-not-run`.
+ # A CELL THAT WAS RE-RUN IS JUDGED ON THE RUN THAT FINISHED IT, as the report and the A/B do:
+ # the dead attempt's `completed: false` stays in the file, so read raw, `--assert-liveness`
+ # failed permanently on a cell that had already been re-run.
from .scoring.from_payload import ATTEMPT_ROW_TYPES, latest_attempt_rows
- # AN ATTEMPT THAT NEVER REACHED ITS CELL ROW IS A FAILURE, NOT AN ABSENCE. `latest_attempt_rows`
- # names the latest attempt from ANY attempt-keyed row, because the Recorder flushes and fsyncs
- # every action and window row while `CellRunner.run` writes the terminal `cell` row in a
- # `finally` a SIGKILL, an OOM kill or a lost machine never reaches. So the killed attempt
- # supersedes the completed one and then contributes no cell row of its own -- and this loop,
- # reading cell rows only, dropped the cell out of the payload entirely. A resume killed inside a
- # cell whose first attempt had already recorded `completed: false` with NOT RUN actions turned
- # `1` into `0`: the gate stopped reporting a failure that is still written in the file.
+ # AN ATTEMPT THAT NEVER REACHED ITS CELL ROW IS A FAILURE, NOT AN ABSENCE: the Recorder fsyncs
+ # every action row while the terminal `cell` row is written in a `finally` a SIGKILL never
+ # reaches, so reading cell rows only turned a recorded failure into a 0.
+ # `CellRunner.run` writes it.
+ # `latest_attempt_rows` is what has to notice.
attempted, recorded = set(), set()
kept = latest_attempt_rows(rows)
for row in kept:
@@ -2628,32 +2331,23 @@ def assert_liveness(args) -> int:
for action in row.get("actions") or []:
name = action.get("action") or action.get("name") or "?"
if action.get("slot_missed"):
- # A MISSED SLOT IS NOT AN ACTION THE PLATFORM CANNOT PERFORM, so it is classified
- # before the allowance rather than inside it. `scene/schedule.py` records an
- # overrun as `ran = False, slot_missed = True`, so under an allowance-first order a
- # listed name never reached this branch at all and its slot miss left the run
- # silently: an excuse written for "the fixture cannot mount this" swallowed a fact
- # about the machine. Ordering it first is what keeps the two buckets below honest.
- # It is a machine-speed fact whether or not the action also reports ran=False, so
- # it lands in `missed` and is judged against `--allow-slot-misses`, never here.
+ # A MISSED SLOT IS NOT AN ACTION THE PLATFORM CANNOT PERFORM, so it is classified before the
+ # allowance: an excuse written for "the fixture cannot mount this" was swallowing a fact about
+ # the machine. It lands in `missed` and is judged against `--allow-slot-misses`.
+ # `scene/schedule.py` records an overrun as `ran=False, slot_missed=True`.
missed.append(f"{where}: {name} missed its slot ({action.get('reason') or '?'})")
elif not action.get("ran"):
- # THE ALLOWANCE IS EXACTLY WHAT ITS NAME SAYS. `--allow-not-run` excuses an action
- # the fixture cannot mount at all; it is not a blanket exemption from the gate.
- # `image_upload` is listed in studiobench-ci.yml only because the fixture cannot
- # mount it, so if it ever does mount, an upload that produces no attachment must
- # be a failure rather than an excuse inherited from a different reason.
+ # THE ALLOWANCE IS EXACTLY WHAT ITS NAME SAYS: `--allow-not-run` excuses an action the fixture
+ # cannot mount at all. If `image_upload` ever does mount, an upload with no attachment fails.
if name in allowed:
continue
problems.append(f"{where}: {name} NOT RUN ({action.get('reason') or 'no reason'})")
elif action.get("expect_ok") is False:
- # RAN IS NOT DID WHAT IT CLAIMED. `scoring/from_payload.py` and
- # `report/payload.py` both already refuse a timing whose own assertion failed;
- # this loop did not, so `ran = True` with `expect_ok = False` fell past both
- # branches above and the gate exited 0. A selector regression that fails EVERY
- # cell's assertion left the workflow green with the surface non-functional --
- # the same "absent reported as no effect" this gate exists for, one branch over.
- # It is a scene problem, not machine speed, so no allowance and no slack reach it.
+ # RAN IS NOT DID WHAT IT CLAIMED: the scoring and report layers both refuse a timing whose
+ # assertion failed, but this loop did not, so a selector regression failing every cell's
+ # assertion left the workflow green. A scene problem, so no allowance reaches it.
+ # The two layers are `scoring/from_payload.py` and `report/payload.py`.
+ # The shape that fell through both branches was `ran=True` with `expect_ok=False`.
problems.append(
f"{where}: {name} ran but its own assertion failed "
f"({action.get('reason') or 'no reason'})"
@@ -2674,8 +2368,8 @@ def assert_liveness(args) -> int:
+ (f", {len(allowed)} action(s) allowed not to run" if allowed else "")
)
if missed and not over:
- # Said out loud rather than passed over: a run with missed slots has holes in its table,
- # and the only thing this exit code claims is that the harness was not the cause.
+ # Said out loud rather than passed over: a run with missed slots has holes in its table, and the
+ # only thing this exit code claims is that the harness was not the cause.
_log(
" the missed slots above are machine speed, not a harness fault, but every one of "
"them is a hole in this run's table. Do not quote a number from this payload."
@@ -2691,9 +2385,8 @@ def parse_args(argv: list):
ap.add_argument(
"--tier",
choices = TIERS,
- # No default here, and `quick` is applied below instead. `--report` needs to tell "the
- # caller asked for this ladder" apart from "the caller said nothing", because a payload
- # records the ladder its run promised and that answer beats a CLI default.
+ # No default here, and `quick` is applied below: `--report` must tell "the caller asked for this
+ # ladder" from "the caller said nothing", because a payload's recorded ladder beats a CLI default.
default = None,
help = (
"fast ~5min (100K only, the iteration loop), quick ~5min (1K,10K, a wiring check), "
@@ -2763,10 +2456,8 @@ def parse_args(argv: list):
"--windowed-arm",
metavar = "ARMS",
dest = "windowed_arm",
- # An ENV FALLBACK as well as the flag. `scripts/pr_perf_sweep.py` builds studiobench's
- # command line itself and is shared with other people's in-flight sweeps, so adding an
- # argument there mid-run is a hazard; this lets the sweep driver select the gate through
- # the environment without anybody editing the driver.
+ # An ENV FALLBACK as well as the flag: `scripts/pr_perf_sweep.py` builds this command line
+ # itself and is shared with other in-flight sweeps, so adding an argument mid-run is a hazard.
default = os.environ.get("SBENCH_WINDOWED_ARM", ""),
help = "comma-separated arm labels (base, treatment) that mount a WINDOW of the thread "
"rather than all of it, and are therefore gated on the windowed readiness signal "
@@ -2855,9 +2546,8 @@ def parse_args(argv: list):
ap.add_argument("--branch", default = "main", help = "Unsloth ref to install when not attaching")
ap.add_argument("--home", help = "UNSLOTH_STUDIO_HOME for an install")
ap.add_argument("--port", type = int, default = 5399)
- # `unsloth`, not `admin`. Unsloth's first run prints "DEFAULT ADMIN ACCOUNT CREATED / username:
- # unsloth", and the wrong one answers 401 with a message about resetting the PASSWORD, which
- # sends you looking in the wrong place.
+ # `unsloth`, not `admin`: Unsloth's first run prints "DEFAULT ADMIN ACCOUNT CREATED / username:
+ # unsloth", and the wrong one answers 401 with a message about resetting the PASSWORD.
ap.add_argument("--username", default = "unsloth")
ap.add_argument("--password", default = "")
ap.add_argument(
@@ -2905,8 +2595,7 @@ def parse_args(argv: list):
"investigation by about 3.2x",
)
args = ap.parse_args(argv)
- # Whether the ladder was ASKED FOR or merely defaulted. Only `--report` cares; everything else
- # sees the tier it always saw.
+ # Whether the ladder was ASKED FOR or merely defaulted. Only `--report` cares.
args.tier_explicit = args.tier is not None
if args.tier is None:
args.tier = "quick"
diff --git a/tests/studio/studiobench/analysis/__init__.py b/tests/studio/studiobench/analysis/__init__.py
index e3780ab8f08..caa8fdd7992 100644
--- a/tests/studio/studiobench/analysis/__init__.py
+++ b/tests/studio/studiobench/analysis/__init__.py
@@ -46,16 +46,13 @@ class CellFailure(RuntimeError):
self.detail = detail
-# ── the no-bare-zero convention (INTERFACES.md, "Rules that apply to every dict") ──
-#
-# `0` means "measured, and it was zero". It never means "did not run". Getting
-# this wrong is the single cheapest way to publish a false negative: a frame
-# budget of 0.00 ms reads as a fast app and is indistinguishable from an
-# instrument that never attached. Every numeric key these layers emit therefore
-# goes through one of the two helpers below, so the distinction is structural
-# rather than remembered.
+# The no-bare-zero convention (INTERFACES.md, "Rules that apply to every dict"). `0` means
+# "measured, and it was zero" and never "did not run": a frame budget of 0.00 ms reads as a fast
+# app and is indistinguishable from an instrument that never attached. Every numeric key these
+# layers emit goes through one of the two helpers below, so the distinction is structural.
+# ── the no-bare-zero convention (INTERFACES.md, "Rules that apply to every dict") ──
def measured(key: str, value: Any) -> dict:
"""A value that WAS measured, even if it came out zero."""
return {key: value, f"{key}_attempted": True}
diff --git a/tests/studio/studiobench/analysis/behaviour.py b/tests/studio/studiobench/analysis/behaviour.py
index da0bd223e76..5f3596dc7a6 100644
--- a/tests/studio/studiobench/analysis/behaviour.py
+++ b/tests/studio/studiobench/analysis/behaviour.py
@@ -54,42 +54,29 @@ from typing import Any, Optional
from .parity import MATCH, NOT_APPLICABLE, NOT_COMPARABLE, NOT_EXERCISED
#: Verdict for a behavioural invariant that moved. Distinct from the digest's DIFFER so a report
-#: can never present the two as the same kind of evidence.
+#: cannot present the two as the same kind of evidence.
BROKEN = "broken"
-#: How far a quantity that should be IDENTICAL may drift. 2%, the same figure the seeded-versus-
-#: streamed equivalence check uses, and for the same reason: two runs of one build never produce
-#: bit-identical character counts once a stream is involved.
+#: How far a quantity that should be IDENTICAL may drift. 2%, as in the seeded-versus-streamed
+#: equivalence check: two runs of one build never produce bit-identical character counts once a
+#: stream is involved.
EXACT_TOLERANCE = 0.02
-#: How far the scroll extent may drift. 10%, much looser, because a windowed list computes its
-#: total height from estimated row heights and corrects them as real rows are measured. Loose
-#: enough to admit a correct virtualizer, nowhere near loose enough to admit one that dropped
-#: rows: the failure this catches is an extent that is a FRACTION of the real one, not one that is
-#: 6% out.
+#: How far the scroll extent may drift. 10%, looser, because a windowed list computes its height
+#: from estimated row heights and corrects them; the failure this catches is an extent that is a
+#: FRACTION of the real one, not one 6% out.
EXTENT_TOLERANCE = 0.10
-#: How much of the thread the clipboard must carry, and how much more than the thread it may carry.
-#:
-#: TWO-SIDED, AND MEASURED AGAINST THE THREAD, because there are two ways to get a copy wrong and
-#: the check used to see only one of them.
-#:
-#: The lower bound catches TRUNCATION, which is the defect this invariant was written for: a
-#: windowed mount cannot select what it has not mounted, so a naive copy carries only the visible
-#: fraction. Measured at 0.61 of the thread on a real 100K arm.
-#:
-#: The upper bound catches SUBSTITUTION, which is what a fix for the first defect turns into if
-#: nobody is watching. Serialising from the message store is the right repair, but the obvious
-#: serialiser is the "save this reply" one, which emits reasoning, tool-call arguments and tool
-#: results -- none of which a user can select, because the panes holding them are collapsed and a
-#: collapsed Radix Collapsible is not in the DOM at all. Measured at 2.16 of the thread on the same
-#: arm: the truncation was fixed and the content was then wrong in the other direction.
-#:
-#: The gap between them is the honest allowance for two different serialisations of the same
-#: content. The base arm's clipboard is the DOM's RENDERED TEXT; a store-based copy is markdown
-#: SOURCE, so fences, emphasis and LaTeX delimiters exist in one and not the other. Measured at
-#: ~0.9% on a scale fixture. Ten percent is generous against that and still refuses 2.16 by a
-#: factor of twenty.
+#:How much of the thread the clipboard must carry, and how much more than the thread it may carry.
+#:TWO-SIDED, AND MEASURED AGAINST THE THREAD, because there are two ways to get a copy wrong.
+#: The lower bound catches TRUNCATION: a windowed mount cannot select what it has not mounted,
+#: so a naive copy carries only the visible fraction (0.61 of the thread on a real 100K arm).
+#: The upper bound catches SUBSTITUTION, what a fix for the first turns into unwatched:
+#: serialising from the message store is right, but the obvious serialiser emits reasoning,
+#: tool-call arguments and tool results, none of which a user can select (2.16 on the same arm).
+#: The gap between them allows for two serialisations of the same content: the base arm's
+#: clipboard is the DOM's RENDERED TEXT, a store-based copy is markdown SOURCE (~0.9% on a scale
+#: fixture). Ten percent is generous against that and still refuses 2.16 by a factor of twenty.
MIN_CLIPBOARD_COVERAGE = 0.95
MAX_CLIPBOARD_COVERAGE = 1.10
@@ -175,8 +162,8 @@ def clipboard_coverage(base_row: dict, treat_row: dict) -> list[dict]:
):
mounted, total = _expect(row, "messages_mounted"), _expect(row, "messages_total")
if not _expect(row, "clipboard_readable"):
- # NOT A PASS. An unreadable clipboard is a surface that went unmeasured, and this is
- # the one invariant where "we could not tell" must never look like "it was fine".
+ # NOT A PASS. An unreadable clipboard is a surface that went unmeasured, and this is the one
+ # invariant where 'we could not tell' must never look like 'it was fine'.
out.append(
_check(
f"clipboard_readable:{label}",
@@ -196,19 +183,14 @@ def clipboard_coverage(base_row: dict, treat_row: dict) -> list[dict]:
required = True,
)
)
- # THE CHECK THAT DETECTS THE DATA LOSS, scored against THE THREAD rather than against the
- # other arm -- which is what the docstring above has always said and what the code did not do.
- #
- # Comparing the two clipboards directly, at a 2% tolerance, asks whether two different
- # serialisations of the same conversation are the same LENGTH. They are not and cannot be: the
- # base arm's clipboard is the DOM's rendered text and a store-based copy is markdown source.
- # A correct fix therefore fails that comparison, and the only way to make it pass is to widen
- # the tolerance until it stops testing anything.
- #
- # The reference is the thread's own visible text, measured by the arm that has all of it in
- # the DOM: on a fully mounted arm `Selection.toString()` over the whole thread IS the thread.
- # If neither arm mounts everything there is no reference and the pair is not comparable, which
- # is reported rather than assumed either way.
+ # THE CHECK THAT DETECTS THE DATA LOSS, scored against THE THREAD rather than the other arm.
+ # Comparing the two clipboards directly at 2% asks whether two different serialisations of the
+ # same conversation are the same LENGTH. They cannot be, so a correct fix fails and the only
+ # way to pass is to widen the tolerance until it tests nothing.
+ # The reference is the thread's own visible text, measured by the arm that has all of it: on a
+ # fully mounted arm `Selection.toString()` over the thread IS the thread. If neither arm mounts
+ # everything there is no reference and the pair is not comparable, which is reported rather
+ # than assumed.
reference = _expect(base_row, "selected_chars")
base_full = _expect(base_row, "mounted_fraction")
if not isinstance(reference, (int, float)) or reference <= 0 or base_full != 1:
@@ -242,8 +224,7 @@ def clipboard_coverage(base_row: dict, treat_row: dict) -> list[dict]:
required = True,
)
)
- # Reported, never gated: on a windowed arm the selection is SUPPOSED to be short, and gating
- # on it would fail the fix.
+ # Reported, never gated: on a windowed arm the selection is SUPPOSED to be short.
out.append(
_check(
"selection_shrank_as_expected",
@@ -295,33 +276,22 @@ def thread_survives_reopen(base_row: dict, treat_row: dict) -> list[dict]:
completed = _reopen_completed(row)
detail = f"the thread had {before} messages and came back with {after}"
# MATCHING COUNTS ARE ONLY AN INVARIANT IF THE THREAD ACTUALLY CAME BACK.
- #
- # `messages_after` is `threadTotal()`, which is `aria-setsize` -- the store's DECLARATION of
- # how long the conversation is, published by the very first reopened row. When the rebuild
- # times out, scene/actions.py records `ran = True`, `expect_ok = False`, a null `reopen_ms`
- # and the outstanding conditions, and leaves the two counts equal because both are that same
- # declaration. Scored as equality it read as a held invariant, the route check passed
- # because the sidebar click had worked, and the pair came out MATCH with the rebuild having
- # timed out at three of eighteen messages mounted. "A declaration is not a rebuild" is the
- # defect the action itself was fixed for; this is the same defect one layer up.
- #
+ # `messages_after` is `aria-setsize`, the store's DECLARATION of the conversation length,
+ # published by the first reopened row. On a timed-out rebuild scene/actions.py records
+ # `ran=True, expect_ok=False` and leaves the two counts equal because both are that
+ # declaration, so equality read as a held invariant with three of eighteen messages mounted.
+ # A timed-out rebuild also leaves `reopen_ms` null.
+ # Read through `threadTotal()`.
# NOT COMPARABLE RATHER THAN BROKEN, AND ONLY FOR THE EQUAL CASE:
- #
- # counts DISAGREE BROKEN, whatever the gate said. The thread came back shorter than it
- # left, which is the data loss this invariant exists for, and a
- # readiness failure corroborates it rather than excusing it. Routing
- # this to NOT COMPARABLE would have downgraded the one finding the
- # whole action is written to catch.
- # counts AGREE, no
- # finished rebuild NOT COMPARABLE. The comparison was not made: both numbers are the
- # same declaration read off a thread that never finished building. And
- # the timeout that produced it is bounded by the harness's OWN
- # remaining budget (a 10 s floor, a 60 s ceiling, against the 180 s the
- # cell's opening gate had), so calling it BROKEN would file a budget
- # exhaustion on a shared machine as a user-visible defect of the arm.
- #
- # The arm's failure is not lost by this: `ran = True, expect_ok = False` already excludes
- # the cell from scoring through `report/payload.py`, and the reason travels with the row.
+ # counts DISAGREE: BROKEN whatever the gate said. The thread came back shorter than it left,
+ # which is the data loss this exists for; routing it to NOT COMPARABLE would downgrade the one
+ # finding the action is written to catch.
+ # counts AGREE with no finished rebuild: NOT COMPARABLE. Both numbers are the same declaration
+ # off a thread that never finished building, and the timeout is bounded by the harness's own
+ # remaining budget, so BROKEN would file a budget exhaustion as a defect of the arm.
+ # The arm's failure is not lost: `ran=True, expect_ok=False` already excludes the cell from
+ # scoring, and the reason travels with the row.
+ # The exclusion happens in `report/payload.py`.
if before is None or after is None:
ok: Optional[bool] = None
required = False
@@ -343,8 +313,8 @@ def thread_survives_reopen(base_row: dict, treat_row: dict) -> list[dict]:
f"(outstanding {failed or 'unrecorded'})"
)
out.append(_check(f"reopen_keeps_every_message:{label}", ok, detail, required = required))
- # The route matters as much as the count. A row measured after a full page navigation is a
- # row about a page load; see `_click_or_navigate` in scene/actions.py.
+ # The route matters as much as the count: a row measured after a full page navigation is a row
+ # about a page load. See `_click_or_navigate`.
via = _expect(row, "reopened_via")
out.append(
_check(
@@ -429,21 +399,15 @@ def _comparable_extents(base_row: dict, treat_row: dict) -> tuple[Any, Any, str]
def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:
"""scroll_after: the gesture covers the ground it commanded and no more, on both arms."""
out = []
- # THE PAIR'S REFERENCE EXTENT, NOT THE ARM'S OWN, and for the same reason `_drift` divides by
- # the larger of the two: an estimate correction is the arm closing the gap to the REAL extent,
- # so the gap is what bounds it, and the arm that is wrong is the one whose own extent is the
- # worse yardstick. Taken per arm the two checks read one tolerance off two denominators, and
- # disagreed inside it: extents of 10,000 and 9,050 pass `scroll_extent` at 9.5% drift while
- # the 950 px correction that closes that very gap came out BROKEN against a ceiling granting
- # 10% of 9,050. A false red is not free here -- it removes the cell from `readings_by_arm`,
- # takes its healthy partner with it through the arm intersection, and
- # `unmeasured_planned_cells` can then VOID the plan -- so the two now enforce the same
- # allowance on the same quantity.
- #
- # It only ever loosens: `max` is never below the arm's own extent. An arm carrying NO extent
- # still gets no ceiling rather than borrowing its partner's, because that would newly bound an
- # arm this check has always left unbounded above, which is the one direction that could invent
- # a red rather than retire one.
+ # THE PAIR'S REFERENCE EXTENT, NOT THE ARM'S OWN, for the same reason `_drift` divides by the
+ # larger of the two: an estimate correction closes the gap to the REAL extent, so the gap
+ # bounds it. Per arm, extents of 10,000 and 9,050 pass `scroll_extent` at 9.5% drift while the
+ # 950 px correction closing that gap came out BROKEN against 10% of 9,050.
+ # A false red is not free: it removes the cell from `readings_by_arm`, takes its healthy partner
+ # with it through the arm intersection, and `unmeasured_planned_cells` can then VOID the plan.
+ # It only ever loosens: `max` is never below the arm's own extent. An arm with NO extent still
+ # gets no ceiling rather than borrowing its partner's, which would newly bound an arm always
+ # left unbounded above.
reference = max(
(
abs(extent)
@@ -457,30 +421,22 @@ def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:
commanded = _expect(row, "commanded_px")
travelled = _expect(row, "travelled_px")
# THE ALLOWANCE IS A FRACTION OF THE EXTENT, so it is taken on the extent. `bottom` is
- # `scrollHeight - clientHeight`, and reading `EXTENT_TOLERANCE` off it is the same defect
- # `_extent_of` exists to fix one check below: on a 10,000 px extent behind an 800 px
- # viewport it grants 920 px where the tolerance says 1,000, so a 941 px correction -- 9.4%
- # of the extent, inside the declared 10% -- came out BROKEN.
+ # `scrollHeight - clientHeight`: on a 10,000 px extent behind an 800 px viewport it grants 920
+ # px where the tolerance says 1,000, so a 941 px correction inside the declared 10% came out
+ # BROKEN.
extent, _reconstructed = _extent_of(row)
# BOUNDED ABOVE AS WELL AS BELOW, and the ceiling is DERIVED rather than chosen.
- #
- # The lower bound is what this invariant was written for: Unsloth's intent-aware autoscroll
- # snapping a programmatic move back to the bottom leaves the gesture having covered
- # nothing. But the predicate was `fraction >= 0.9` and nothing above it, so an arm whose
- # viewport moved TWICE as far as commanded passed exactly as 1.0 did and the pair returned
- # MATCH. `travelled` sums `|scrollTop_after - target_before|`, so every pixel above
- # `commanded` is the viewport being moved by something other than the gesture -- the same
- # anchor instability this action exists to detect, in the other direction.
- #
- # WHY THIS CEILING AND NOT A NUMBER PICKED TO LOOK SYMMETRIC WITH 0.9. Overshoot has one
- # legitimate source: a windowed list correcting estimated row heights, which moves the
- # offset by the error in the estimate. Those errors total the error in the arm's extent,
- # and this file already declares how far the extent may be wrong (`EXTENT_TOLERANCE`). So
- # the gesture may exceed its command by that fraction of the arm's own extent, both terms
- # read off the row. No second constant, and it scales with the rung.
- #
- # It degrades to no ceiling rather than to a guess: an arm carrying no extent gets the
- # lower bound alone, said in the detail, because a ceiling of zero fails every correct arm.
+ # The lower bound is what this was written for: intent-aware autoscroll snapping a programmatic
+ # move back to the bottom leaves the gesture having covered nothing. But `fraction >= 0.9` with
+ # no upper bound passed an arm whose viewport moved TWICE as far as commanded; `travelled` sums
+ # |scrollTop_after - target_before|, so every pixel above `commanded` is the same anchor
+ # instability in the other direction.
+ # WHY THIS CEILING. Overshoot has one legitimate source, a windowed list correcting estimated
+ # row heights, and those errors total the error in the arm's extent, already declared as
+ # `EXTENT_TOLERANCE`. So the gesture may exceed its command by that fraction of the arm's own
+ # extent, both terms read off the row: no second constant, and it scales with the rung.
+ # It degrades to no ceiling rather than a guess: an arm carrying no extent gets the lower bound
+ # alone, said in the detail, because a ceiling of zero fails every correct arm.
ceiling: Optional[float] = None
if (
isinstance(commanded, (int, float))
@@ -506,14 +462,12 @@ def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:
)
out.append(_check(f"scroll_travelled:{label}", ok, detail))
# THE EXTENT, NOT `bottom`, AND AT THE EXTENT'S OWN ALLOWANCE. This compared `bottom` through
- # `_same_number` and so through `EXACT_TOLERANCE`, 2%, while `scroll_extent` grants the same
- # physical quantity 10% a few checks earlier and says why a correct virtualizer needs it. An
- # arm inside the declared allowance was reported behaviourally BROKEN, and a false red is not
- # free: it removes the cell from `readings_by_arm`, takes its healthy partner with it through
- # the arm intersection, and `unmeasured_planned_cells` can then VOID the plan.
- #
- # `_same_number` is deliberately NOT widened. Its other three keys -- `selected_chars`,
- # `visible_chars`, `clipboard_chars` -- are not extents and are correctly strict at 2%.
+ # `_same_number` at 2% while `scroll_extent` grants the same physical quantity 10%, so an arm
+ # inside the declared allowance was reported BROKEN, and a false red removes the cell, its
+ # partner, and can VOID the plan.
+ # The 2% is `EXACT_TOLERANCE`.
+ # `_same_number` is deliberately NOT widened: its other three keys are not extents and are correctly strict at 2%.
+ # They are `selected_chars`, `visible_chars` and `clipboard_chars`.
b_ext, t_ext, what = _comparable_extents(base_row, treat_row)
drift = _drift(b_ext, t_ext)
out.append(
@@ -527,9 +481,9 @@ def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:
return out
-#: action -> the invariants that apply to it. An action absent from this table has no behavioural
-#: invariant declared and is reported as UNCHECKED rather than as passing, on the same principle
-#: that keeps NOT_COMPARABLE out of the pass column in the digest report.
+#: action -> the invariants that apply to it. An action absent from this table has no declared
+#: invariant and is reported as UNCHECKED rather than as passing, on the principle that keeps
+#: NOT_COMPARABLE out of the pass column.
INVARIANTS = {
"select_all_copy": clipboard_coverage,
"select_text": lambda b, t: [
@@ -581,9 +535,8 @@ def compare_behaviour(base_row: Optional[dict], treat_row: Optional[dict]) -> di
got = rule(base_row, treat_row)
checks.extend(got if isinstance(got, list) else [got])
- # A REQUIRED CHECK THAT COULD NOT BE READ VOIDS THE PAIR. See `_check`: without this, an
- # action whose entire subject went unmeasured scores MATCH on the strength of the checks that
- # happened to survive.
+ # A REQUIRED CHECK THAT COULD NOT BE READ VOIDS THE PAIR: without this, an action whose entire
+ # subject went unmeasured scores MATCH on the checks that survived.
unread = [c for c in checks if c.get("required") and c["ok"] is None]
if unread:
return {
@@ -600,13 +553,10 @@ def compare_behaviour(base_row: Optional[dict], treat_row: Optional[dict]) -> di
"checks": checks,
}
# A PASS REQUIRES AN ACTION-SPECIFIC INVARIANT TO HAVE HELD.
- #
- # The scroll extent is checked on every action and it is a property of the THREAD, not of the
- # action: it holds or fails identically across all eighteen. Counting an action as passing
- # because the thread's scrollbar was the right length would report `model_change`,
- # `image_upload` and `settings` as verified on a windowed arm when nothing whatsoever about
- # them was examined. That is the same mistake `compare_rows` exists to prevent on the digest
- # side, where an action that never ran used to contribute a match.
+ # The scroll extent is checked on every action but is a property of the THREAD, so it holds or
+ # fails identically across all eighteen. Counting an action as passing on it would report
+ # `model_change`, `image_upload` and `settings` as verified when nothing about them was
+ # examined.
specific = [c for c in checks if c["ok"] is not None and c["invariant"] != "scroll_extent"]
if not specific:
return {
diff --git a/tests/studio/studiobench/analysis/bridge_build.py b/tests/studio/studiobench/analysis/bridge_build.py
index f14fed7d030..f82697b4aa8 100644
--- a/tests/studio/studiobench/analysis/bridge_build.py
+++ b/tests/studio/studiobench/analysis/bridge_build.py
@@ -29,9 +29,8 @@ from typing import Any, Callable, Sequence
from . import CellFailure
from .symbols import FAILED, Bridge, build_bridge
-# A `RungRunner` puts the page into the state for one rung and returns when the
-# work for that rung is finished. It must be deterministic: the same rung must
-# do the same amount of work every time, or the count vectors are noise.
+# A `RungRunner` puts the page into the state for one rung and returns when that rung's work is
+# finished. It must be deterministic, or the count vectors are noise.
RungRunner = Callable[[Any, str], None]
DEFAULT_BRIDGE_RUNGS: tuple[str, ...] = ("bridge_s", "bridge_m", "bridge_l")
@@ -177,38 +176,23 @@ PROFILER_RECORDER_JS = """
"""
-# ═══════════════════════════════════════════════════════════════════════════
# Build provenance: is the thing under measurement the thing we think it is?
-# ═══════════════════════════════════════════════════════════════════════════
-#
-# The worst output this layer can produce is a real number from the wrong
-# bundle under the right label. A dev server inflates React's own work several
-# fold and would MANUFACTURE the symptom we are hunting; a stale shipping dist
-# left in the output directory would silently measure code without the
-# profiling renderer. Both look like a clean run.
-#
-# Three independent checks, because each catches something the others cannot:
-#
-# 1. `/@vite/client` must not answer 200. That file exists only when a Vite
-# dev server is serving, and it is a fact about the SERVER.
-# 2. React must report `bundleType: 0` in the same renderer entry as
-# `rendererPackageName: "react-dom"`. That is a fact about the RENDERER
-# THAT ACTUALLY LOADED, which is the only thing that matters and the only
-# one of the three that a build-config mistake cannot fake.
-# 3. `__STUDIOBENCH_ATTRIBUTION_BUILD__` must be true. That is a fact about
-# WHICH dist is mounted, and it is what catches a stale shipping build
-# sitting in the directory passed to `--frontend`.
-#
-# DO NOT ADD A `jsxDEV` GREP. It false-positives: `hast-util-to-jsx-runtime`,
-# which Streamdown pulls in, references the dev JSX runtime in shipping code, so
-# a production bundle contains the string and the check condemns a correct
-# build.
-#
-# `bundleType` is read through the DevTools global hook. React only injects into
-# that hook if it already exists when the renderer initialises, so the stub must
-# be installed with `add_init_script` BEFORE the first `goto`, exactly like the
-# JWT seeding in `studio_test_kit.auth`. Installed after, the hook is empty and
-# the check reads as "React is missing" on a perfectly good page.
+# The worst output this layer can produce is a real number from the wrong bundle under the right
+# label: a dev server inflates React's own work several fold and would MANUFACTURE the symptom,
+# and a stale shipping dist would silently measure code without the profiling renderer. Both look
+# like a clean run.
+# Three independent checks: `/@vite/client` must not answer 200 (a fact about the SERVER); React
+# must report `bundleType: 0` in the same renderer entry as `rendererPackageName: "react-dom"` (a
+# fact about the RENDERER THAT ACTUALLY LOADED, the only one a build-config mistake cannot fake);
+# and `__STUDIOBENCH_ATTRIBUTION_BUILD__` must be true (which dist is mounted, catching a stale
+# shipping build in the `--frontend` directory).
+# DO NOT ADD A `jsxDEV` GREP: `hast-util-to-jsx-runtime`, which Streamdown pulls in, references
+# the dev JSX runtime in shipping code, so a production bundle contains the string and the check
+# condemns a correct build.
+# `bundleType` is read through the DevTools global hook, and React only injects into it if it
+# already exists when the renderer initialises, so the stub must be installed with
+# `add_init_script` BEFORE the first `goto`. Installed after, the hook is empty and the check
+# reads as 'React is missing' on a good page.
DEVTOOLS_HOOK_STUB_JS = """
(() => {
@@ -241,14 +225,10 @@ DEVTOOLS_HOOK_STUB_JS = """
})();
"""
-# React's `bundleType`: 0 is a production build, 1 is development. Confirmed on
-# a real pair: the dev server reported 1 and the profiling build reported 0.
-#
-# NOTE THAT 0 DOES NOT MEAN "profiling renderer is loaded". React's profiling
-# build is a production build, so it also reports 0, and `bundleType` alone
-# cannot tell the two apart. That is why `assert_profiling_build_loaded` is a
-# separate check on `onRender` actually firing, and why neither one substitutes
-# for the other.
+# React's `bundleType`: 0 is production, 1 is development. Confirmed on a real pair. NOTE THAT 0
+# DOES NOT MEAN 'profiling renderer is loaded' (React's profiling build is a production build and
+# also reports 0) which is why `assert_profiling_build_loaded` separately checks `onRender`
+# actually firing.
BUNDLE_TYPE_PRODUCTION = 0
BUNDLE_TYPE_DEVELOPMENT = 1
@@ -270,10 +250,9 @@ def assert_production_bundle(page: Any, *, base_url: str | None = None) -> dict[
"or DEVTOOLS_HOOK_STUB_JS was installed after the first navigation instead of "
"through add_init_script before it, in which case React had nothing to inject into.",
)
- # Match on the SAME entry, not across entries. A page can host more than one
- # renderer (react-dom plus react-reconciler in a canvas library, say), and
- # checking `bundleType` from one against `rendererPackageName` from another
- # is how a development react-dom passes behind a production sibling.
+ # Match on the SAME entry, not across entries: a page can host more than one renderer, and
+ # checking `bundleType` from one against `rendererPackageName` from another is how a development
+ # react-dom passes behind a production sibling.
dom = [r for r in renderers if str(r.get("rendererPackageName") or "") == "react-dom"]
if not dom:
raise CellFailure(
diff --git a/tests/studio/studiobench/analysis/classify.py b/tests/studio/studiobench/analysis/classify.py
index 06bd975540f..940c99d4ed4 100644
--- a/tests/studio/studiobench/analysis/classify.py
+++ b/tests/studio/studiobench/analysis/classify.py
@@ -57,11 +57,9 @@ RAF = "raf"
IDLE = "idle"
GC = "gc"
BROWSER_INTERNAL = "browser_internal"
-# Browser-to-renderer IPC that ran page script. In a harnessed run this is
-# dominated by the devtools pipe delivering `Runtime.evaluate`, which is to say
-# it is OUR OWN cost and it belongs in a column with its own name so that it can
-# be watched for growth with the treatment, not folded into a residual and not
-# hidden inside `browser_internal`.
+# Browser-to-renderer IPC that ran page script. In a harnessed run this is dominated by the
+# devtools pipe delivering `Runtime.evaluate`, which is OUR OWN cost and belongs in a column with
+# its own name so it can be watched for growth with the treatment.
AGENT_IPC = "agent_ipc"
UNCLASSIFIED = "unclassified"
@@ -78,12 +76,12 @@ ORIGINS = (
UNCLASSIFIED,
)
-# Origins that are the harness observing, not the app working. Reported, never
-# subtracted: a correction you cannot see is a correction you cannot check.
+# Origins that are the harness observing, not the app working. Reported, never subtracted: a
+# correction you cannot see is a correction you cannot check.
HARNESS_ORIGINS = (AGENT_IPC,)
-# Above this share of main-thread task time in the unclassified bucket, the cell
-# is not quotable. The point of the tool is that we can name things.
+# Above this share of main-thread task time in the unclassified bucket, the cell is not quotable.
+# The point of the tool is that we can name things.
UNCLASSIFIED_FAIL_PCT = 2.0
_JS_EVENTS = frozenset(
@@ -147,8 +145,8 @@ _INPUT_EVENT_TYPES = frozenset(
}
)
-# Network-ish streaming: an SSE frame arrives as a `message` event on an
-# EventSource, and a fetch-based stream as a resource event.
+# Network-ish streaming: an SSE frame arrives as a `message` event on an EventSource, and a
+# fetch-based stream as a resource event.
_NETWORK_EVENT_TYPES = frozenset({"message", "readystatechange", "load", "error", "open"})
_POSTED_FROM_RULES: tuple[tuple[str, str, str], ...] = (
@@ -164,8 +162,8 @@ _POSTED_FROM_RULES: tuple[tuple[str, str, str], ...] = (
("v8/src/tasks/", "", GC),
)
-# Tasks the browser posts to itself. Only ever applied to a task that ran NO
-# page script, so this can never swallow a JS bottleneck.
+# Tasks the browser posts to itself. Only ever applied to a task that ran NO page script, so this
+# can never swallow a JS bottleneck.
_BROWSER_INTERNAL_FILES = (
"web_frame_widget_impl.cc",
"main_thread_scheduler_impl.cc",
@@ -189,11 +187,9 @@ _BROWSER_INTERNAL_FILES = (
_IDLE_MARKERS = ("DoIdleWork", "RunIdleTask", "IdleTask", "ScheduleIdleTask")
# Blink `TaskType` enum names as they appear in
-# `args.renderer_main_thread_task_execution.task_type`. Matched by SUBSTRING on
-# the part after `TASK_TYPE_`, so that both
-# `TASK_TYPE_JAVASCRIPT_TIMER_DELAYED_HIGH_NESTING` and
-# `TASK_TYPE_JAVASCRIPT_TIMER_IMMEDIATE` land on `timer` without enumerating
-# every nesting variant.
+# `args.renderer_main_thread_task_execution.task_type`. Matched by SUBSTRING on the part after
+# `TASK_TYPE_`, so both `..._TIMER_DELAYED_HIGH_NESTING` and `..._TIMER_IMMEDIATE` land on
+# `timer`.
_TASK_TYPE_RULES: tuple[tuple[str, str], ...] = (
("JAVASCRIPT_TIMER", TIMER),
("POSTED_MESSAGE", MESSAGE_CHANNEL),
@@ -367,15 +363,10 @@ def classify_task(task: Task) -> ClassifiedTask:
queue_name = queue_name,
)
- # 0. Blink's own label, when the `scheduler` category recorded one. This
- # outranks everything below it because it is the scheduler stating what
- # it dispatched rather than us reconstructing it. Two exceptions are
- # handled first, both cases where the label is true but not what a reader
- # wants attributed:
- # - a V8 task queue that ran no JS is GC or compilation bookkeeping,
- # but one that DID run JS is a real JS task and must not be filed
- # under GC;
- # - a compositor-queue task with a nested input dispatch is input.
+ # 0. Blink's own label, when the `scheduler` category recorded one. It outranks everything below
+ # because it is the scheduler stating what it dispatched. Two exceptions are handled first: a V8
+ # task queue that ran no JS is GC or compilation bookkeeping while one that DID run JS is a real
+ # JS task, and a compositor-queue task with a nested input dispatch is input.
if task_type:
if types & _INPUT_EVENT_TYPES:
return done(
@@ -388,16 +379,14 @@ def classify_task(task: Task) -> ClassifiedTask:
continue
return done(origin, f"task_type:{task_type}")
- # 1. Input first: a keystroke that also fires a timer is still a keystroke,
- # and input latency is the metric a user actually feels.
+ # 1. Input first: a keystroke that also fires a timer is still a keystroke, and input latency is what a user feels.
if types & _INPUT_EVENT_TYPES:
return done(INPUT, f"EventDispatch:{sorted(types & _INPUT_EVENT_TYPES)[0]}")
if "main_thread_event_queue.cc" in src_file:
return done(INPUT, "posted_from:main_thread_event_queue")
- # 2. GC before everything else that could contain it, because a major GC
- # posted from the incremental marking job is not a timer task even when
- # it happens to run inside one.
+ # 2. GC before everything else that could contain it, because a major GC posted from the
+ # incremental marking job is not a timer task even when it runs inside one.
gc_names = {
n
for n in names
@@ -422,16 +411,16 @@ def classify_task(task: Task) -> ClassifiedTask:
if "dom_timer.cc" in src_file:
return done(TIMER, "posted_from:dom_timer")
- # 6. Network / SSE. Checked before message-channel because both arrive over
- # a mojo pipe and only the resource events tell them apart.
+ # 6. Network / SSE. Checked before message-channel because both arrive over a mojo pipe and only
+ # the resource events tell them apart.
if names & _RESOURCE_EVENTS:
return done(NETWORK, f"nested:{sorted(names & _RESOURCE_EVENTS)[0]}")
if types & _NETWORK_EVENT_TYPES and "connector.cc" in src_file:
return done(NETWORK, f"EventDispatch:{sorted(types & _NETWORK_EVENT_TYPES)[0]}")
- # 7. Message channel, that is, the React scheduler. Blink dispatches a
- # `MessagePort` message as a mojo connector task, so the discriminator is
- # that page script ran and nothing network-shaped happened.
+ # 7. Message channel, i.e. the React scheduler. Blink dispatches a `MessagePort` message as a
+ # mojo connector task, so the discriminator is that page script ran and nothing network-shaped
+ # happened.
if ran_js and (
"connector.cc" in src_file
or "message_port" in src_file
@@ -439,17 +428,16 @@ def classify_task(task: Task) -> ClassifiedTask:
):
return done(MESSAGE_CHANNEL, "mojo_pipe_running_page_script")
- # 8. The devtools / browser IPC channel running page script. This is what
- # `Runtime.evaluate` and `Page.*` look like from inside the renderer, so
- # it is measurement cost. Named, so it can be watched for correlation
- # with the treatment; never subtracted.
+ # 8. The devtools / browser IPC channel running page script: what `Runtime.evaluate` and `Page.*`
+ # look like from inside the renderer, so it is measurement cost. Named so it can be watched for
+ # correlation with the treatment; never subtracted.
if ran_js and any(
f in src_file for f in ("ipc_mojo_bootstrap.cc", "simple_watcher.cc", "mojo_bootstrap")
):
return done(AGENT_IPC, f"posted_from:{src_file.rsplit('/', 1)[-1]}")
- # 9. Browser bookkeeping, and ONLY when no page script ran inside. This
- # guard is what stops this class becoming the residual under a new name.
+ # 9. Browser bookkeeping, and ONLY when no page script ran inside. This guard is what stops the
+ # class becoming the residual under a new name.
if not ran_js and any(f in src_file for f in _BROWSER_INTERNAL_FILES):
return done(BROWSER_INTERNAL, f"posted_from:{src_file.rsplit('/', 1)[-1]}")
@@ -457,9 +445,8 @@ def classify_task(task: Task) -> ClassifiedTask:
if needle in src_file and (not func_needle or func_needle in src_func):
return done(origin, f"posted_from:{needle}")
- # 10. Last resort before giving up: the queue the task sat on. Weaker than
- # the task type, because a queue carries a mix, but still read from the
- # scheduler rather than guessed.
+ # 10. Last resort before giving up: the queue the task sat on. Weaker than the task type, because
+ # a queue carries a mix, but still read from the scheduler rather than guessed.
for needle, origin in _QUEUE_NAME_RULES:
if needle == queue_name:
if origin == GC and ran_js:
diff --git a/tests/studio/studiobench/analysis/cpuprofile.py b/tests/studio/studiobench/analysis/cpuprofile.py
index d7d1054fb25..0c64b6f9765 100644
--- a/tests/studio/studiobench/analysis/cpuprofile.py
+++ b/tests/studio/studiobench/analysis/cpuprofile.py
@@ -37,19 +37,16 @@ from typing import Any, Iterable, Sequence
from . import CellFailure
from .traceparse import Trace
-# V8 synthetic frames. They are real samples and their time is real, but they
-# are not call frames anyone can go and fix, so they are labelled and excluded
-# from frame ranking rather than silently dropped from totals.
+# V8 synthetic frames. Their time is real but they are not call frames anyone can fix, so they are
+# labelled and excluded from frame ranking rather than dropped from totals.
SYNTHETIC_FRAMES = frozenset({"(root)", "(program)", "(idle)", "(garbage collector)"})
-# How far the summed sample deltas may drift from the wall span of the chunks
-# before the cell is unusable.
+# How far the summed sample deltas may drift from the wall span of the chunks before the cell is unusable.
DELTA_WALL_TOLERANCE = 0.02
-# Below this many non-synthetic samples inside a window set, a leaf ranking is
-# not a ranking. At a ~150 us sampling interval a 40 us task contributes at most
-# one sample, so a set of short tasks can produce a confident-looking table
-# built on five data points. Reported as `underpowered`, never hidden.
+# Below this many non-synthetic samples inside a window set, a leaf ranking is not a ranking: at a
+# ~150 us interval a 40 us task contributes at most one sample, so short tasks can produce a
+# confident-looking table built on five data points. Reported as `underpowered`, never hidden.
MIN_JS_SAMPLES_FOR_RANKING = 100
@@ -104,17 +101,14 @@ class CpuProfile:
chunk_ts_first: int = 0
chunk_ts_last: int = 0
negative_deltas: int = 0
- # `Profile.args.data.startTime`, kept only so the clock skew against `ts`
- # can be reported. Never used as an anchor.
+ # `Profile.args.data.startTime`, kept only so the clock skew against `ts` can be reported. Never used as an anchor.
declared_start_time: int = 0
source: str = ""
- # `sampleTraceId` -> node id, accumulated from
- # `ProfileChunk.args.data.cpuProfile.trace_ids`. This is an EXACT stack, not
- # a sampled one: when the `disabled-by-default-devtools.timeline.stack`
- # category is on, Blink calls `v8::CpuProfiler::CollectSample` with a tagged
- # id at specific instrumentation points, so the stack recorded for that id
- # is the real stack at that instant rather than whatever the 100 us sampler
- # happened to catch.
+ # `sampleTraceId` -> node id, from `ProfileChunk.args.data.cpuProfile.trace_ids`. An EXACT stack,
+ # not a sampled one: with `disabled-by-default-devtools.timeline.stack` on, Blink calls
+ # `v8::CpuProfiler::CollectSample` with a tagged id at specific instrumentation points, so the
+ # stack is the real one at that instant.
+ # Not whatever the 100 us sampler happened to catch.
trace_ids: dict[str, int] = field(default_factory = dict)
@property
@@ -277,13 +271,11 @@ def parse_cpu_profiles(trace: Trace) -> dict[str, CpuProfile]:
profiles[key] = CpuProfile(
pid = pid,
tid = int(e.get("tid", 0)),
- # ANCHOR ON `ts`, NOT ON `args.data.startTime`. V8 writes
- # `startTime` straight from `base::TimeTicks` without converting it
- # into the trace's clock domain, and both V8's own source comment
- # and the DevTools frontend say to use the event timestamp instead.
- # The two agree to within a few microseconds on a healthy capture,
- # but the drift is unbounded in principle, and it would show up as
- # every sample being attributed to the neighbouring task.
+ # ANCHOR ON `ts`, NOT ON `args.data.startTime`. V8 writes `startTime` straight from
+ # `base::TimeTicks` without converting into the trace's clock domain, and both V8's source
+ # comment and the DevTools frontend say to use the event timestamp. They agree to within
+ # microseconds on a healthy capture, but the drift is unbounded in principle and would attribute
+ # every sample to the neighbouring task.
start_time = int(e.get("ts", data.get("startTime", 0))),
declared_start_time = int(data.get("startTime", 0)),
source = str(data.get("source", "")),
@@ -301,8 +293,8 @@ def parse_cpu_profiles(trace: Trace) -> dict[str, CpuProfile]:
key = f"{pid}:{e.get('id')}"
prof = profiles.get(key)
if prof is None:
- # A chunk with no matching Profile event cannot be anchored to a
- # start time, so its sample timestamps would be meaningless.
+ # A chunk with no matching Profile event cannot be anchored to a start time, so its sample
+ # timestamps would be meaningless.
continue
data = (e.get("args") or {}).get("data") or {}
cpu = data.get("cpuProfile") or {}
@@ -313,8 +305,8 @@ def parse_cpu_profiles(trace: Trace) -> dict[str, CpuProfile]:
parent = raw.get("parent")
if parent is not None:
prof.parents[nid] = int(parent)
- # Some producers give `children` instead of `parent`; honour both so
- # ancestry does not silently collapse to a forest of roots.
+ # Some producers give `children` instead of `parent`; honour both so ancestry does not silently
+ # collapse to a forest of roots.
for child in raw.get("children") or ():
prof.parents.setdefault(int(child), nid)
diff --git a/tests/studio/studiobench/analysis/oracles.py b/tests/studio/studiobench/analysis/oracles.py
index 1b0afa43335..67f9919f931 100644
--- a/tests/studio/studiobench/analysis/oracles.py
+++ b/tests/studio/studiobench/analysis/oracles.py
@@ -38,9 +38,8 @@ NEAR_MISS = "near_miss"
UNEXPLAINED = "unexplained_hot_frame"
NOT_MEASURED = "not_measured"
-# Integer ratios worth naming when a count is off by a clean factor. Each has a
-# specific mechanical meaning, so reporting the ratio hands over a hypothesis
-# rather than a mystery.
+# Integer ratios worth naming when a count is off by a clean factor. Each has a specific mechanical
+# meaning, so reporting the ratio hands over a hypothesis rather than a mystery.
_KNOWN_RATIOS: dict[Fraction, str] = {
Fraction(
2, 1
@@ -276,35 +275,25 @@ def predicted_next_rung(
return quantity_fn(next_structural_input).value
-# ═══════════════════════════════════════════════════════════════════════════
# M2 and M3: oracle shapes over PAGE-SIDE counters
-# ═══════════════════════════════════════════════════════════════════════════
-#
-# M1 is settled with a call count from precise coverage, because the mechanism
-# lives inside react-dom where we can count invocations. M2 and M3 do not work
-# that way: their cost is in how much text is rescanned and how much layout is
-# forced, and neither is an invocation count. Those must be counted IN THE PAGE.
-#
-# Layer 3 owns `instruments/layoutcost.js` and already counts `scrollHeight`
-# reads, `scrollTop` writes and MutationObserver callbacks. This section does
-# NOT duplicate that file. It fixes the KEY NAMES that file should emit and
-# supplies the oracles that consume them, so that the counting and the
-# interpretation are owned separately and neither has to guess at the other.
-#
-# ONE HONEST DIFFERENCE, stated plainly rather than buried. The M1 oracle is an
-# EXACT INTEGER MATCH and it either holds or it does not. The M2 oracle is a
-# REGIME TEST: SSE deltas do not arrive in equal sizes, so the quadratic
-# prediction is exact only for a uniform stream and must be compared as a ratio
-# with a refusal band. That is weaker evidence and it is labelled as such
-# everywhere it is emitted. Do not let a regime verdict sit in a table next to a
-# naming as though they were the same thing.
+# M1 is settled with a call count from precise coverage, because the mechanism lives inside
+# react-dom where invocations can be counted. M2 and M3 do not work that way: their cost is in how
+# much text is rescanned and how much layout is forced, and those must be counted IN THE PAGE.
+# Layer 3 owns `instruments/layoutcost.js` and already counts `scrollHeight` reads, `scrollTop`
+# writes and MutationObserver callbacks. This section does NOT duplicate it: it fixes the KEY
+# NAMES that file should emit and supplies the oracles that consume them, so counting and
+# interpretation are owned separately.
+# ONE HONEST DIFFERENCE: the M1 oracle is an EXACT INTEGER MATCH that either holds or does not,
+# while the M2 oracle is a REGIME TEST, since SSE deltas do not arrive in equal sizes, so the
+# quadratic prediction is exact only for a uniform stream and must be compared as a ratio with a
+# refusal band. That is weaker evidence and is labelled as such everywhere it is emitted.
REGIME_QUADRATIC = "cumulative_reparse_quadratic"
REGIME_LINEAR = "incremental_parse_linear"
REGIME_UNDECIDED = "undecided"
-# The counters Layer 3's page-side instrument should emit, per streamed reply.
-# Every one is an integer; none is a duration.
+# The counters Layer 3's page-side instrument should emit, per streamed reply. Every one is an
+# integer; none is a duration.
PAGE_COUNTER_CONTRACT: dict[str, dict[str, str]] = {
"m2_reparse": {
"parse_calls": "invocations of parseAssistantContent during the reply",
@@ -394,8 +383,7 @@ def reparse_regime(
"refusal band. This reply is too short to distinguish the regimes; use a longer rung."
)
return out
- # Closer in log space wins; the ratio is reported either way so a reader can
- # see how decisive the call was.
+ # Closer in log space wins; the ratio is reported either way so a reader can see how decisive the call was.
r_lin = chars_rescanned / lin
r_quad = chars_rescanned / quad
out["ratio_to_linear"] = round(r_lin, 3)
diff --git a/tests/studio/studiobench/analysis/parity.py b/tests/studio/studiobench/analysis/parity.py
index 75b4b100894..d53023aefbd 100644
--- a/tests/studio/studiobench/analysis/parity.py
+++ b/tests/studio/studiobench/analysis/parity.py
@@ -25,75 +25,33 @@ from __future__ import annotations
import collections
from typing import Any, Iterable, Optional
-# What a comparison concluded. Kept as plain strings so a payload row carries them verbatim.
-#: WHAT "MATCH" ACTUALLY CLAIMS, because the name promises more than the instrument delivers.
-#:
-#: Everything in this module is computed from `scene/parity.js`, whose structural digest walks the
-#: THREAD and nothing else. It is sidebar-blind and layout-blind by construction: a change confined
-#: to the sidebar, or one that alters computed geometry without altering thread structure, passes
-#: it while being entirely invisible to it. That is not a theory. A concurrent campaign measuring
-#: the sidebar drag found the shipped thread digest returned 0 of 34 differing pairs on a real,
-#: visible change -- and its null control returned 0 of 34 as well, so the instrument was not
-#: discriminating in either direction. Three purpose-built captures (sidebar-inclusive structure,
-#: sidebar inline style, custom-property reach) found the same change 34 of 34, with the null at
-#: zero in every category.
-#:
-#: So MATCH means NO THREAD-STRUCTURE CHANGE WAS DETECTED. It does not mean the UI is unchanged.
-#: Not covered, and not detectable here at all:
-#:
-#: the sidebar outside the digest root
-#: computed layout / geometry positions and sizes are never read
-#: CSS custom properties never read
-#: stylesheet changes only insofar as they alter `display`, `visibility` or
-#: `pointer-events` on the <=64 elements the bounded style probe
-#: reaches, which is reported separately and as an advisory
-#:
-#: Anything relying on a stronger reading than that needs its own capture.
+#: WHAT "MATCH" ACTUALLY CLAIMS: everything here is computed from `scene/parity.js`, whose
+#: structural digest walks the THREAD only, so it is sidebar-blind and layout-blind by
+#: construction. Measured: on a real visible sidebar-drag change the thread digest found 0 of 34
+#: differing pairs, while three purpose-built captures found it 34 of 34. So MATCH means NO
+#: THREAD-STRUCTURE CHANGE WAS DETECTED, not that the UI is unchanged.
MATCH = "match"
DIFFER = "differ"
NOT_COMPARABLE = "not_comparable"
NOT_EXERCISED = "not_exercised"
-#: THE FOURTH OUTCOME, and it is not a softer NOT_COMPARABLE.
-#:
-#: NOT_COMPARABLE means the reading failed or the two readings are of different things by
-#: accident. NOT_APPLICABLE means the question itself is wrong for this pair: the structural
-#: digest asks "is the same DOM on screen", and an arm whose entire purpose is to put less DOM on
-#: screen answers "no" by construction. Reporting that as a UI difference would be true and
-#: useless -- eighteen red rows, none of them a finding, and the real behavioural questions
-#: drowned underneath them.
-#:
-#: It is kept distinct from NOT_COMPARABLE so a reader can tell "we could not measure this" from
-#: "this measurement does not apply here", and so `derive_unstable` counts neither as evidence.
+#: THE FOURTH OUTCOME, and not a softer NOT_COMPARABLE: that one means the reading failed, while
+#: NOT_APPLICABLE means the question is wrong for this pair -- the digest asks "is the same DOM on
+#: screen" and an arm whose purpose is to put less DOM on screen answers "no" by construction.
+#: Kept distinct so `derive_unstable` counts neither as evidence.
NOT_APPLICABLE = "not_applicable"
-#: A KEY ON A NOT_COMPARABLE RESULT, not a fifth verdict.
-#:
-#: `compare` has one refusal that ALSO carries a complete positive reading: the scaffold, every
-#: overlay, the mounted message set and every message not being written at capture time all
-#: agreed, and the only leftover is the subtree of a reply that was mid-tail. It stays a refusal
-#: because a half-arrived digest names a point in a stream rather than a rendering, and it must
-#: never be a pass on the A/B result side: a treatment build whose live message renders wrongly
-#: sits in exactly that shape, and `structural_report` files it under `blind`.
-#:
-#: THE VERDICT DOES NOT MOVE; only `derive_unstable` reads this. The result asks "did this build
-#: change the UI", the null asks "does this action differ against ITSELF", and for the second
-#: question a pair where everything readable agreed is an observation of non-difference rather
-#: than a blank. Calling it blank is what left `keystroke` and `send_turn` permanently undecided
-#: at 100K on a runner where the in-flight tail landed 24 characters apart.
-#:
-#: AND IT CAN ONLY NARROW THE EXCUSE SET: the observation is a NON-difference, so `differed`
-#: cannot grow and nothing can be marked unstable on its strength. The worst case is calling an
-#: action stable that nothing measured as differing, which costs a loud false alarm on the
-#: result and never a silenced regression.
+#: A KEY ON A NOT_COMPARABLE RESULT, not a fifth verdict. `compare` has one refusal that also
+#: carries a complete positive reading: everything agreed except the subtree of a reply that was
+#: mid-tail. It stays a refusal, because a half-arrived digest names a point in a stream. THE
+#: VERDICT DOES NOT MOVE; only `derive_unstable` reads this, since for "does this action differ
+#: against ITSELF" a pair where everything readable agreed is an observation of non-difference.
+#: It can only NARROW the excuse set, so the worst case is a loud false alarm.
SETTLED_MATCH = "settled_match"
# Actions whose rendered result legitimately differs between two runs of the SAME build, so a
-# digest mismatch there carries no information about the pull request under test.
-#
-# A MECHANISM PER ENTRY, not a hunch, and the value is the mechanism rather than a comment beside
-# it so that a test can require one. An action silenced without a stated reason is a hole somebody
-# punched in the instrument and nobody can audit afterwards; the null control's empirically
-# derived set (`derive_unstable`) is the cross-check that each of these earns its place.
+# digest mismatch there says nothing about the pull request. A MECHANISM PER ENTRY, as the value
+# rather than a comment beside it so a test can require one: an action silenced without a stated
+# reason is an unauditable hole, and `derive_unstable` cross-checks that each earns its place.
UNSTABLE_ACTIONS: dict[str, str] = {
"stop_generation": "stops a live stream, so how many characters arrived before the stop is a race with the "
"network and differs run to run on one build. NOT reproduced by the 100K fast-tier null "
@@ -106,8 +64,8 @@ UNSTABLE_ACTIONS: dict[str, str] = {
"scroll_after": "same gesture against a settled thread; the resting offset still depends on the observer. "
"NOT reproduced by the 100K fast-tier null control (0 of 4); kept on the mechanism alone",
# The four below were NOT in the hand-written set and were found by the null control, each
- # scoring as a hard UI-change signal on a build compared with itself. The mechanism for each
- # was read out of the payload, not guessed.
+ # scoring as a hard UI-change signal on a build compared with itself; the mechanism for each was
+ # read out of the payload.
"keystroke": "types characters into the composer over wall clock, and the composer's text is in the "
"DOM, so how many keystrokes had landed by the capture deadline is a race. Measured 4 of "
"4 differing, by 8 signature characters, with assistant_chars identical",
@@ -124,24 +82,14 @@ UNSTABLE_ACTIONS: dict[str, str] = {
}
-# WHOSE ABILITY TO RUN IS A RACE, which is a different claim from whose DIGEST varies and needs
-# its own list. Every entry in UNSTABLE_ACTIONS above describes what makes the CAPTURE move --
-# how many characters had arrived, where the scroll came to rest, whether the send button had
-# re-enabled. Not one of them says the action sometimes cannot be performed at all. Using that
-# list to excuse one-arm-only EXECUTION exempted nine of the sixteen scheduled actions from the
-# one regression shape that leaves no digest to differ, on the strength of a measurement about
-# something else.
-#
-# `slot_missed` already covers the runner arriving late, so what is left here is the narrow case
-# of an action that legitimately cannot run because the stream it needs is not there. Read out of
-# each action's own `not_run` reasons in `scene/actions.py` rather than assumed:
-# `action -> (why, the not_run reasons that qualify)`. KEYED ON THE REASON, not just the action,
-# because each of these has non-racy failure paths too: send_turn also returns not_run for "no
-# composer on the page" (scene/actions.py:1016) and stop_generation for "the stop button is not
-# present" (scene/actions.py:501). A treatment build that REMOVES either control records exactly
-# the one-arm-only regression this instrument exists to catch, and keyed by name alone the
-# exemption swallowed it. The mechanism was always written down here; it just was not what the
-# code matched on.
+# WHOSE ABILITY TO RUN IS A RACE, a different claim from whose DIGEST varies. Every
+# UNSTABLE_ACTIONS entry describes what makes the CAPTURE move, so using that list to excuse
+# one-arm-only EXECUTION exempted nine of sixteen actions from the one regression shape that
+# leaves no digest to differ. `slot_missed` already covers the runner arriving late, so what is
+# left is an action that cannot run because the stream it needs is not there. KEYED ON THE
+# REASON, because each has non-racy paths too, and a treatment that REMOVES a control records
+# exactly the regression this exists to catch.
+# `send_turn` also returns not_run for "no present" (scene/actions.py:501).
RACY_EXECUTION: dict[str, tuple[str, tuple[str, ...]]] = {
"stop_generation": (
"needs a live stream to stop, and returns not_run when nothing was generating and a new "
@@ -173,11 +121,9 @@ def racy_execution(action: str, reason: str) -> bool:
return any(marker in (reason or "") for marker in entry[1])
-# The six that are NOT here, and why, since dropping an exemption needs as much justification as
-# granting one. `composer_fill`, `keystroke`, `copy_markdown`, `message_menu` and `select_text`
-# fail to run only when the control they need is absent or unresponsive -- "no composer on the
-# page", "no Copy button on the last assistant message" -- and that IS the build. `scroll_after`
-# has no `not_run` path at all, so a `ran: false` for it cannot be a race under any reading.
+# The six that are NOT here: `composer_fill`, `keystroke`, `copy_markdown`, `message_menu` and
+# `select_text` fail to run only when the control they need is absent or unresponsive, and that
+# IS the build; `scroll_after` has no `not_run` path at all.
def _messages(capture: dict) -> dict[int, dict]:
@@ -331,9 +277,9 @@ def comparability(base: Optional[dict], treat: Optional[dict]) -> Optional[str]:
f"{label} could not be captured: " f"{side.get('reason') or 'no reason recorded'}"
)
assert base is not None and treat is not None
- # `root_kind` is absent in captures taken before it was recorded. Missing on BOTH sides is an
- # old payload and is allowed through; missing on one side means the two runs were produced by
- # two different versions of the instrument, which is not a comparison of two builds.
+ # `root_kind` is absent in captures taken before it was recorded. Missing on BOTH sides is an old
+ # payload and is allowed through; missing on one side means two different versions of the
+ # instrument, which is not a comparison of two builds.
bk, tk = base.get("root_kind"), treat.get("root_kind")
if (bk is None) != (tk is None):
return (
@@ -399,9 +345,8 @@ def localise(
bm, tm = _messages(base), _messages(treat)
for i in sorted(set(bm) | set(tm)):
if i in skip and i in bm and i in tm:
- # In flight on BOTH sides of the comparison. A message present on one arm only is a
- # different statement -- one arm is not rendering a message the other is -- and is
- # reported below whatever its status.
+ # In flight on BOTH sides of the comparison: a message present on one arm only is a different
+ # statement and is reported below whatever its status.
continue
if i not in bm:
moved.append(f"msg{i}({tm[i].get('role', '?')}):only treatment")
@@ -415,9 +360,8 @@ def localise(
moved.extend(overlays_moved(base, treat))
if not moved:
- # The whole-thread digest moved but no message and no overlay did. That is the thread
- # scaffolding itself -- the viewport, the composer, the empty-state -- and saying so is
- # more useful than an empty list, which reads as "nothing differs".
+ # The whole-thread digest moved but no message and no overlay did, so the difference is the
+ # thread scaffolding itself; saying so beats an empty list that reads as "nothing differs".
bc, tc = _scaffold(base)[1], _scaffold(treat)[1]
moved.append(f"thread scaffolding outside any message ({bc}->{tc}c)")
return moved
@@ -437,13 +381,10 @@ def compare_styles(base: dict, treat: dict) -> tuple[str, str]:
f"the probe hit its element cap "
f"(base={bs.get('elements')}, treatment={ts.get('elements')})"
)
- # A POSITIVE CONTROL ON THE SCAN ITSELF. Two probes that matched no elements have equal
- # counts and equal digests -- both are the hash of an empty string -- so a probe that scanned
- # NOTHING reports MATCH, which is the strongest verdict this function can return and is
- # supported by no observation whatsoever. That is not hypothetical: the selector list is
- # written against Unsloth's markup, and a class rename anywhere in it silently empties the
- # scan. A DOM or CSSOM scan that can return zero has to be able to tell "I looked and they
- # agree" from "I did not look", and this one could not.
+ # A POSITIVE CONTROL ON THE SCAN ITSELF: two probes that matched no elements have equal counts
+ # and equal digests (both the hash of an empty string), so a probe that scanned NOTHING reports
+ # MATCH on no observation whatsoever. The selector list is written against Unsloth's markup, so a
+ # class rename anywhere in it silently empties the scan.
if not bs.get("elements") or not ts.get("elements"):
return NOT_COMPARABLE, (
f"the style probe matched no elements (base={bs.get('elements')}, "
@@ -451,28 +392,16 @@ def compare_styles(base: dict, treat: dict) -> tuple[str, str]:
"selector list is written against Unsloth's markup and does not survive a rename; "
"this is a probe that needs fixing, not two arms that agree"
)
- # THE RUN-STATE CONTROL IS IN THE SELECTOR LIST, so this reading straddles the same composer
- # swap the structural refusals below are about, and it has to be elided for the same reason.
- # `STYLE_SELECTORS` (scene/parity.js) names `button[aria-label="Send message"]` and
- # `button[aria-label="Stop generating"]`, and the signature carries the SELECTOR THAT MATCHED
- # (`parts.push(sel + "#" + n)`) -- so an arm still generating and a settled arm walk two
- # different entries for the one composer button. Measured on two byte-identical fixture threads
- # differing only in that control: Send against Stop holds all three properties at
- # `inline-block` / `visible` / `auto` on both arms and still moves the digest, and Send against
- # either queue control -- neither of which is in the selector list at all -- moves the element
- # COUNT from 7 to 6. Both come back DIFFER over identical CSS, and `sweep/ui_parity.report`
- # prints that as a style-regression advisory for precisely the ordinary cross-stream-state
- # timing the refusals exist to withhold.
- #
- # THE SAME CORROBORATION THE SCAFFOLD SUPPRESSION REQUIRES, and for the same reason: a
- # differing `composer_control` alone is what a treatment that DROPS or renames the control
- # produces, and that is a rendering regression this probe should still report. `streaming` and
- # `queued_idle` are read off the thread's run state rather than off the composer, so they are
- # evidence the composer cannot manufacture.
- #
- # NOT_COMPARABLE AND NOT MATCH. The probe reads ONE aggregate digest over up to `STYLE_CAP`
- # elements, so a genuine CSS difference elsewhere on the page is inside the same number and
- # cannot be separated from the swap here. This withholds the reading; it does not pass it.
+ # THE RUN-STATE CONTROL IS IN THE SELECTOR LIST, so this reading straddles the same composer swap
+ # the structural refusals below are about. `STYLE_SELECTORS` names both the Send and Stop buttons
+ # and the signature carries the SELECTOR THAT MATCHED, so a generating arm and a settled one walk
+ # two entries for one button: on two byte-identical threads Send against Stop moves the digest
+ # with all three properties identical. THE SAME CORROBORATION THE SCAFFOLD SUPPRESSION REQUIRES,
+ # because a differing `composer_control` alone is what a treatment that DROPS the control
+ # produces. NOT_COMPARABLE AND NOT MATCH: the probe reads ONE aggregate digest, so a genuine CSS
+ # difference elsewhere is inside the same number.
+ # Over up to `STYLE_CAP` elements.
+ # `streaming` and `queued_idle` are read off the run state, not the composer.
if (bs.get("elements") != ts.get("elements") or bs.get("digest") != ts.get("digest")) and (
generation_disagrees(base, treat) and _run_state_disagrees(base, treat)
):
@@ -543,7 +472,7 @@ def _messages_moved(
"""The message half of `_any_moved`, on its own."""
skip = skip or set()
bm, tm = _messages(base), _messages(treat)
- # A message PRESENT ON ONE ARM ONLY is a difference whether or not it was streaming. Being
+ # A message PRESENT ON ONE ARM ONLY is a difference whether or not it was streaming: being
# mid-reply excuses a digest, never an absence.
if set(bm) != set(tm):
return True
@@ -583,31 +512,20 @@ def settled_messages_moved(base: dict, treat: dict) -> list[str]:
if b.get("role") != t.get("role"):
out.append(f"msg{i}:role {b.get('role')}->{t.get('role')}")
continue
- # Flagged in flight by the arm that COULD place its stream. Its digest is a point in a
- # stream on that arm whatever role it carries, so it is withheld like any other.
+ # Flagged in flight by the arm that COULD place its stream; its digest is a point in a stream on
+ # that arm whatever role it carries, so it is withheld like any other.
if i in streaming:
continue
if b.get("role") == "user" and b.get("digest") != t.get("digest"):
out.append(f"msg{i}(user):{b.get('chars')}->{t.get('chars')}c")
- # WHY THERE IS NO ASSISTANT-ROW RULE HERE, since it looks like the obvious next one to add.
- #
- # An earlier assistant row on a fully mounted thread really is settled while the tail is being
- # written, and a counting argument even makes it provable without knowing WHICH row is in
- # flight: a thread writes one reply at a time, so two differing assistant rows cannot both be
- # it. Both readings are sound and both are still unusable, because of what BLINDS the probe in
- # the first place.
- #
- # The reachable blind pair is a build that renamed the `data-status` hook. That attribute is on
- # the assistant text part and the digest walks attributes, so the rename that blinds the probe
- # ALSO moves the digest of every assistant row, by itself. Reporting those rows therefore turns
- # every genuinely blind pair into a difference, which is the wall-clock false alarm this file
- # exists to remove, wearing the other hat. Measured: implementing the counting rule flipped
- # `test_a_settled_queued_idle_pair_is_scored_rather_than_refused`'s blind control from NOT
- # COMPARABLE to DIFFER.
- #
- # A user row is not exposed to that, which is exactly why it is the one shape that qualifies:
- # the hook is not on it. Separating "the hook attribute moved" from "the content moved" would
- # need a digest that excludes the attribute, which is not something this capture carries.
+ # WHY THERE IS NO ASSISTANT-ROW RULE HERE, though it looks like the obvious next one: an earlier
+ # assistant row on a fully mounted thread really is settled, and a counting argument even proves
+ # which cannot be in flight. Both readings are sound and unusable, because the reachable blind
+ # pair is a build that renamed the `data-status` hook -- that attribute is on the assistant text
+ # part and the digest walks attributes, so the rename that blinds the probe ALSO moves every
+ # assistant row's digest. A user row is not exposed to that.
+ # Measured: the counting rule flipped test_a_settled_queued_idle_pair_is_scored_rather_than_refused's
+ # blind control from NOT COMPARABLE to DIFFER.
return out
@@ -628,13 +546,10 @@ def _any_moved(
what surfaced it.
"""
skip = skip or set()
- # THE SCAFFOLD, not the whole-thread digest. The whole-thread digest serialises the streamed
- # message too, so with a reply in flight it differs on essentially every pair -- measured at
- # 175 of 175 adjacent 24-character steps of the shipped corpus -- and every check below it
- # would be unreachable behind it, exactly as the whole-thread digest once made the overlay walk
- # unreachable. The scaffold plus the per-message rows is the same reading taken apart, so
- # nothing is lost by decomposing it. On a payload with no scaffold this IS the whole-thread
- # digest and the behaviour is the old one.
+ # THE SCAFFOLD, not the whole-thread digest: the latter serialises the streamed message too, so
+ # with a reply in flight it differs on essentially every pair (175 of 175 adjacent 24-character
+ # steps) and every check below would be unreachable behind it. The scaffold plus the per-message
+ # rows is the same reading taken apart.
if _scaffold(base)[0] != _scaffold(treat)[0]:
return True
bo, to = _overlays(base), _overlays(treat)
@@ -663,9 +578,8 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:
"verdict": NOT_APPLICABLE,
"reason": not_applicable,
"moved": [],
- # The style probe goes with it. Its verdict is `elements` counts matching, and those
- # counts are element counts over `[data-role]` among other things, so it reports
- # DIFFER for exactly the same reason and with exactly as much meaning.
+ # The style probe goes with it: its verdict is `elements` counts matching, and those are element
+ # counts over `[data-role]` among other things, so it reports DIFFER for the same reason.
"style_verdict": NOT_APPLICABLE,
"style_reason": not_applicable,
}
@@ -682,46 +596,24 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:
style_verdict, style_reason = compare_styles(base, treat)
blind = streaming_probe(base, treat)
if blind is not None:
- # THE REFUSAL MUST NOT SWALLOW A READING THAT DOES NOT DEPEND ON THE STREAM. An overlay is
- # walked from `document`, outside the thread root, so a dialog that mounted when it should
- # not, or one whose contents were rewritten, is a finding whether or not the streamed
- # message could be identified. Without this it went out with the refusal, and
- # `structural_report` buckets a refusal as blind and never consults it for the exit code,
- # so the run went green on it.
- #
- # THE SCAFFOLD IS DELIBERATELY NOT CONSULTED HERE, and this is the part of the review item
- # that measurement does not support. `ThreadPrimitive.Root` wraps `ThreadComposerDock`
- # (thread.tsx), so the composer is inside `.aui-thread-root` and inside the scaffold -- and
- # the composer is exactly the surface that changes when a reply starts and stops. Measured
- # on two byte-identical threads differing only in the composer control: Stop against Send
- # moves the scaffold from 373 to 381 characters and changes its digest, with no message
- # content involved at all. On the very pair this branch is about -- one arm generating with
- # a quiet hook, the other finished -- the scaffold therefore differs BECAUSE one arm is
- # generating, and reporting that as a rendering difference would manufacture the wall-clock
- # false alarm this whole file exists to remove.
+ # THE REFUSAL MUST NOT SWALLOW A READING THAT DOES NOT DEPEND ON THE STREAM. An overlay is walked
+ # from `document`, outside the thread root, so a dialog that mounted when it should not is a
+ # finding either way; without this it went out with the refusal, which `structural_report` buckets
+ # as blind, so the run went green. THE SCAFFOLD IS DELIBERATELY NOT CONSULTED HERE:
+ # `ThreadPrimitive.Root` wraps `ThreadComposerDock`, so the composer is inside the scaffold and
+ # Stop against Send moves it on two byte-identical threads.
+ # `compare_visible` already refuses these.
+ # thread.tsx; the swap moves the scaffold from 373 to 381 characters.
independent = overlays_moved(base, treat)
- # AND THE SETTLED MESSAGE ROWS, which are the other half of what this refusal must not
- # swallow. A row both arms call the user's, and a row whose role itself changed, cannot be
- # the reply being written -- so their meaning does not depend on where the stream had got
- # to, and discarding them costs exactly the finding this instrument exists to make. Without
- # this a user message rendered differently by the treatment left here as NOT COMPARABLE
- # with an empty `moved`, and `structural_report` buckets a refusal as blind and never
- # consults it for the exit code, so the run went green on it. `compare_visible` already
- # applies this rule to its own rows and says it is the same rule as this one; it has to be.
+ # AND THE SETTLED MESSAGE ROWS, the other half of what this refusal must not swallow: a row both
+ # arms call the user's, and a row whose role itself changed, cannot be the reply being written,
+ # so their meaning does not depend on the stream. Without this a user message rendered
+ # differently by the treatment left as NOT COMPARABLE with an empty `moved`.
independent = independent + settled_messages_moved(base, treat)
- # AND THE SCAFFOLD, but only when the two arms agree about whether a reply was running.
- # The composer is inside the scaffold and is a function of exactly that, so the scaffold is
- # readable here when they agree and meaningless when they do not. That is the correct form
- # of the half of this refusal that measurement did not support.
- #
- # AND THE DISAGREEMENT HAS TO BE CORROBORATED OFF THE RUN STATE, exactly as the composer
- # suppression below requires. `generation_disagrees` reads `composer_control`, which IS the
- # composer, so on its own it excuses a composer difference with the composer -- and a
- # treatment that DROPS the Stop control, renames it, or selects the wrong one makes the
- # tokens differ for that reason and suppressed the whole scaffold with it, while `streaming`
- # and `queued_idle` were saying the arms were in the same run state all along. That is a
- # rendering regression withheld by the branch written to admit rendering regressions, and
- # since a refusal is filed under `blind` the run goes green on it.
+ # AND THE SCAFFOLD, but only when the two arms agree about whether a reply was running: the
+ # composer is inside the scaffold and is a function of exactly that. AND THE DISAGREEMENT HAS TO
+ # BE CORROBORATED OFF THE RUN STATE, since `generation_disagrees` reads `composer_control`, which
+ # IS the composer -- so on its own it excuses a composer difference with the composer.
if scaffold_moved(base, treat) and not (
generation_disagrees(base, treat) and _run_state_disagrees(base, treat)
):
@@ -748,86 +640,23 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:
"style_verdict": style_verdict,
"style_reason": style_reason,
}
- # ── THE STREAMED MESSAGE ────────────────────────────────────────────────────────────────────
- #
- # `streaming` holds the messages that were still being written on one arm or the other. They
- # are scored on NOTHING, and that is a refusal rather than a normalisation: the digest of a
- # half-written message names a point in a stream, the two arms are at two different points in
- # the same stream by construction, and no amount of text normalisation can turn one into the
- # other. See `in_flight`.
- #
- # THREE OUTCOMES, and the ordering between them is the load-bearing part.
- #
- # the settled document differs DIFFER. This is the case that used to be lost. An action
- # landing inside a stream was silenced wholesale by
- # UNSTABLE_ACTIONS, so a real regression anywhere else in
- # the thread -- another message, an overlay, the composer --
- # printed under "expected to vary" and the run exited 0.
- # Now the streamed message is elided and the rest is scored.
- # the settled document agrees and
- # the in-flight message agrees MATCH, unchanged. Two arms that landed on the same point
- # in the stream serialised identically, which is the claim
- # this mode makes, and demoting it would cost coverage for
- # nothing.
- # the settled document agrees and
- # the in-flight message differs NOT COMPARABLE. Not a pass. `CLAIM_STRUCTURAL` quantifies
- # over the whole thread, one message did not serialise
- # identically, and the reason it did not is a reading with
- # no defined moment. Calling that MATCH would be the
- # instrument certifying a surface it could not look at,
- # which is the failure `compare_styles` and
- # `compare_visible` each already refuse in their own way.
- #
- # WHAT THIS GIVES UP, plainly, and both of these are measured rather than reasoned about.
- #
- # 1. A genuine rendering regression INSIDE the message that happened to be streaming is no
- # longer distinguishable here and lands as NOT COMPARABLE. It was not distinguishable before
- # either -- every action that lands in a stream is on the declared unstable list -- so
- # nothing that used to be caught stops being caught, and the outcome moves from "expected to
- # vary, exit 0" to "not measured, not a pass".
- # 2. A REORDER that moves the streamed message past another message of the same role is
- # demoted from DIFFER to NOT COMPARABLE. The per-message rows are keyed by mounted index, so
- # swapping two messages puts a streaming row at one index on one arm and at the other index
- # on the other; both indices are then in flight on one side or the other, both are withheld,
- # and the scaffold markers that survive carry the same role in both orders. Measured on the
- # live-DOM battery: 11 injected rendering differences, 10 still reported DIFFER and this one
- # demoted. It is a demotion and not a hole -- NOT COMPARABLE is not a pass and the run does
- # not go green on it -- and the only shape that reaches it needs the streamed message not to
- # be the last one, which the app does not do today.
+ # THE STREAMED MESSAGE. `streaming` holds messages still being written on one arm or the other;
+ # they are scored on NOTHING, a refusal rather than a normalisation, because the two arms are at
+ # two different points in one stream by construction. THREE OUTCOMES, and the ordering is
+ # load-bearing: a settled document that differs is DIFFER; settled agreeing plus in-flight
+ # agreeing is MATCH; settled agreeing with the in-flight message differing is NOT COMPARABLE, not
+ # a pass, because `CLAIM_STRUCTURAL` quantifies over the whole thread. WHAT THIS GIVES UP: a
+ # regression inside the streaming message lands as NOT COMPARABLE, and a REORDER past another
+ # message of the same role is demoted from DIFFER (10 of 11 injected differences still DIFFER).
streaming = in_flight(base, treat)
+ # THE COMPOSER IS NOT A RENDERING DIFFERENCE. One arm finished and one still writing has its
+ # MESSAGES withheld correctly, but the dock is inside `.aui-thread-root`, so `digest_scaffold`
+ # carries Stop on one arm and Send on the other and `_any_moved` reported DIFFER while every
+ # settled row was byte-identical. THE NULL BATTERY CANNOT SEE THIS -- one build against itself
+ # has both arms generating and the bias cancels. WITHHELD RATHER THAN IGNORED: if a message or
+ # overlay also moved this never runs. AND THE RUN STATE HAS TO SAY SO INDEPENDENTLY, or the
+ # suppression argues in a circle.
# ── THE COMPOSER IS NOT A RENDERING DIFFERENCE ──────────────────────────────────────────────
- #
- # The pair this whole change is about is one arm that has finished its reply against one that
- # is still writing it. Its MESSAGES are withheld correctly. Its COMPOSER is not: the dock is
- # inside `.aui-thread-root`, so `digest_scaffold` carries Stop on the arm that is generating
- # and Send on the arm that is not, and `_any_moved` then reports the pair as DIFFER with the
- # single claim `thread scaffolding outside any message (373->381c)`. Measured on exactly that
- # pair, with every settled message row byte-identical across the two arms.
- #
- # THE NULL BATTERY CANNOT SEE THIS, which is why it survived a 15-of-15-to-0 null. The null is
- # one build against itself at six points in ONE stream, so BOTH arms are generating and both
- # render Stop; the bias is symmetric within the control and cancels exactly. A flat null proves
- # repeatability, never comparability.
- #
- # WITHHELD RATHER THAN IGNORED. If a message or an overlay also moved, that is reported as
- # usual and this never runs. It is only when the scaffold is the ONLY thing that moved, and the
- # arms disagree about generation, that the reading has no defined moment -- and then the honest
- # answer is a refusal, not a pass. Calling it MATCH would hide a genuine composer regression.
- #
- # AND THE RUN STATE HAS TO SAY SO INDEPENDENTLY, or the suppression argues in a circle.
- # `generation_disagrees` reads `composer_control`, the token naming which control the composer
- # rendered -- so "the arms were at different points in the turn" is being proved by the very
- # surface whose difference is in question. A treatment that DROPS the Send button, renames it,
- # or selects the wrong control reaches this branch with every message and overlay agreeing, and
- # a refusal here is a green run: `report` files NOT COMPARABLE under `blind` and takes its exit
- # code from `stable_bad or one_sided`. That is the regression the note above says must not be
- # hidden, hidden by the branch written to say so.
- #
- # `streaming` and `queued_idle` are read off the run state rather than off the composer, so
- # they are evidence the composer cannot manufacture. Every legitimate suppression still has
- # one: Stop against Send differs in `streaming`, and Queue against Send in the queued-idle
- # interval differs in `queued_idle`. What is left, the arms agreeing on both and still
- # rendering different controls, has no run-state explanation and is a rendering difference.
if (
generation_disagrees(base, treat)
and _run_state_disagrees(base, treat)
@@ -866,12 +695,10 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:
f"msg{i}({bm[i].get('role', '?')}):{bm[i].get('chars')}->{tm[i].get('chars')}c"
for i in unsettled[:4]
)
- # A ROW WHOSE ROLE CHANGED IS NOT SOMETHING THIS PAIR AGREED ON, and `_any_moved`
- # cannot see it: the role sits beside the digest, and an in-flight digest is withheld.
- # `settled_messages_moved` reports a role change even in flight -- how far a reply has
- # arrived says nothing about whose message it is -- and the two readings must not
- # disagree. The verdict is the refusal either way; this only decides whether the
- # refusal ALSO carries a positive reading, and a role disagreement carries none.
+ # A ROW WHOSE ROLE CHANGED IS NOT SOMETHING THIS PAIR AGREED ON, and `_any_moved` cannot see it
+ # because an in-flight digest is withheld while `settled_messages_moved` reports a role change
+ # even in flight. The verdict is the refusal either way; this only decides whether the refusal
+ # also carries a positive reading.
roles_agree = all(bm[i].get("role") == tm[i].get("role") for i in set(bm) & set(tm))
out = {
"verdict": NOT_COMPARABLE,
@@ -891,11 +718,9 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:
"style_reason": style_reason,
}
if roles_agree:
- # THE ONE REFUSAL THAT ALSO CARRIES A POSITIVE READING. See `SETTLED_MATCH` for
- # why this flag exists and what it is and is not allowed to move. It is set here
- # and nowhere else, because this is the only branch reached with `_any_moved`
- # already false: the scaffold agreed, every overlay agreed, the two arms mounted
- # the same set of messages, and every message that was NOT being written agreed.
+ # THE ONE REFUSAL THAT ALSO CARRIES A POSITIVE READING (see `SETTLED_MATCH`). Set here and
+ # nowhere else, because this is the only branch reached with `_any_moved` already false: the
+ # scaffold, every overlay, the mounted message set and every settled message all agreed.
out[SETTLED_MATCH] = True
return out
return {
@@ -950,18 +775,11 @@ def execution_verdict(base_row: Optional[dict], treat_row: Optional[dict]) -> Op
row = base_row if label == "base" else treat_row
assert isinstance(row, dict)
reason = row.get("reason") or "no reason recorded"
- # A MISSED SLOT IS NOT A BUILD DIFFERENCE, even when only one arm missed it, and this
- # distinction is the whole load-bearing part of `one_sided`. `ran=false` has two causes
- # that look identical in the count and mean opposite things. The runner arriving after
- # the slot closed says nothing about the build: the schedule is fixed, the budgets are
- # small, and a machine slow enough to miss a slot at 45700ms is slow enough to miss it
- # on the next repetition too, so corroboration does not separate them -- the misses are
- # correlated through the runner, not independent draws. Treating that as asymmetric
- # execution reds the job on machine speed and breaks the invariant the mutation study
- # established, that a missed slot cannot move this verdict. A precondition failure is
- # the opposite: `image_upload` records `slot_missed=false` with "no visible attachments
- # button", and a control that stops opening records exactly that. So the signal is
- # `slot_missed`, which the driver already writes, and not merely which arm went idle.
+ # A MISSED SLOT IS NOT A BUILD DIFFERENCE, even when only one arm missed it. `ran=false` has two
+ # causes that look identical: a runner arriving after the slot closed says nothing about the
+ # build, and because misses are correlated through the runner rather than independent draws,
+ # corroboration does not separate them. A precondition failure is the opposite, so the signal is
+ # `slot_missed` and not merely which arm went idle.
missed_slot = bool(row.get("slot_missed"))
detail = (
f"the action did not run on the {label} arm ({reason}), so any "
@@ -978,8 +796,8 @@ def execution_verdict(base_row: Optional[dict], treat_row: Optional[dict]) -> Op
"verdict": NOT_EXERCISED,
"moved": [],
"one_sided": live[0] if live else "",
- # The idle arm's OWN not_run string, unwrapped. `reason` above is prose built for a
- # reader; the exemption has to match on what the action actually recorded.
+ # The idle arm's OWN not_run string, unwrapped: `reason` above is prose built for a reader, while
+ # the exemption has to match what the action recorded.
"idle_reason": reason,
"reason": detail,
"style_verdict": NOT_EXERCISED,
@@ -1095,15 +913,14 @@ def derive_unstable(
verdict = result.get("verdict")
if verdict not in (MATCH, DIFFER):
if verdict == NOT_COMPARABLE and result.get(SETTLED_MATCH):
- # Everything this pair could read agreed: an observation of NON-difference, so
- # `differed` cannot grow and nothing can be called unstable on its strength.
- # Tallied separately so a reader can see how much of a decision rests on it.
+ # Everything this pair could read agreed: an observation of NON-difference, so `differed` cannot
+ # grow. Tallied separately so a reader can see how much of a decision rests on it.
seen[action] += 1
settled[action] += 1
continue
- # Not comparable and not exercised are both "no reading", and neither may be counted
- # as an observation of stability. An action derived as stable from four pairs that
- # never ran would be permanently trusted on the strength of nothing.
+ # Not comparable and not exercised are both "no reading", and neither may count as an observation
+ # of stability: an action derived as stable from four pairs that never ran would be permanently
+ # trusted on nothing.
blind[action] += 1
continue
seen[action] += 1
@@ -1116,23 +933,16 @@ def derive_unstable(
"observations": n,
"differed": d,
"not_comparable": blind[action],
- # How many of `observations` came from a settled-match refusal rather than a MATCH.
- # Reported rather than folded in: an action decided entirely this way was decided on
- # the settled thread only, which is the weaker reading.
+ # How many of `observations` came from a settled-match refusal rather than a MATCH. Reported
+ # rather than folded in: an action decided entirely this way was decided on the settled thread.
SETTLED_MATCH: settled[action],
- # Unstable only with enough observations to mean it. Below that the honest answer is
- # "not enough evidence", which is not the same as "stable" and is not reported as it.
- #
- # FROM THE COMPLETE COMPARISONS ONLY: a settled-match refusal can decide an action
- # and never help CLASSIFY one as unstable. One DIFFER beside one settled match is one
- # differing comparison, and letting the refusal supply the second would mint an
- # exemption from a reading that never saw the live subtree.
+ # Unstable only with enough observations to mean it; below that the honest answer is "not enough
+ # evidence". FROM THE COMPLETE COMPARISONS ONLY: a settled-match refusal can decide an action but
+ # never help CLASSIFY one as unstable, or it would mint an exemption from a partial reading.
"unstable": bool(d and (n - settled[action]) >= min_observations),
- # THE SAME COUNT `unstable` WAS DECIDED ON, once anything has differed. On the raw
- # total the mixed state -- one DIFFER beside one settled match -- read as "decided and
- # stable", so `cross_check` filed a declared action under `declared_stable_in_practice`
- # ("the null never saw it differ") on a run where it differed once. With nothing
- # differing there is no classification to make and the total is the right count.
+ # THE SAME COUNT `unstable` WAS DECIDED ON, once anything has differed. On the raw total, one
+ # DIFFER beside one settled match read as "decided and stable", so `cross_check` filed a declared
+ # action under `declared_stable_in_practice` on a run where it differed once.
"undetermined": (n - settled[action] if d else n) < min_observations,
}
return out
@@ -1180,36 +990,19 @@ def summarise(results: Iterable[dict[str, Any]]) -> dict[str, int]:
return dict(tally)
+# THE POLICY: changes must preserve UI and UX idempotency, with three exemptions -- a dramatic
+# performance win accepted on the record, a difference that exists only OFF SCREEN, and a
+# select-all that need not select all PROVIDED the copy stays complete. The third is what makes
+# deferral and virtualization cheap: the copy path may serialise from the message store, since
+# completeness is REQUIRED and visual selection fidelity is not. `compare()` cannot express the
+# second exemption, so it returns NOT_APPLICABLE and withholds a verdict. This supplies one.
+
+#: The claim each verdict makes, so a reader knows which of the three was checked. NOT
+#: "whole-document structural parity", which this instrument cannot support: `scene/parity.js`
+#: digests the thread root plus overlay selectors, never the sidebar, geometry or custom
+#: properties. Printing the stronger claim is how a sidebar-drag campaign came to be scored 0 of
+#: 34 differing pairs under a banner saying the DOM was identical.
# ── visible-region parity ───────────────────────────────────────────
-#
-# THE POLICY. All changes must preserve UI and UX idempotency, with three exemptions: a difference
-# may be accepted deliberately when performance improves dramatically; a difference that exists
-# only OFF SCREEN is fine by definition, because rendering only what is visible is an accepted
-# technique rather than a parity violation; and a select-all need not select all, PROVIDED the copy
-# it produces stays complete.
-#
-# The third is what makes deferral and virtualization cheap: the copy path stops depending on what
-# is mounted, so it may serialise the thread from the message store as markdown or plain text
-# instead of reproducing a DOM selection. Completeness of the copied content is REQUIRED -- silent
-# truncation is data loss -- and visual selection fidelity is NOT.
-#
-# `compare()` above cannot express the second exemption. It digests the thread whether or not any
-# of it is on screen, so deferred off-screen work fails it by construction: virtualization,
-# deferred fence highlighting, content-visibility, lazy images. Returning NOT_APPLICABLE for those
-# pairs withholds a verdict.
-# This supplies one.
-
-#: The claim each verdict makes, so a reader knows which of the three was actually checked. A bare
-#: "PARITY OK" has meant three very different things in this file's history and the difference
-#: between them is the difference between a strong result and a weak one.
-#: NOT "whole-document structural parity: every element in the DOM is identical on both arms",
-#: which is what this string used to say and which the instrument cannot support. `scene/parity.js`
-#: digests the THREAD ROOT plus a list of overlay selectors; it never walks the sidebar, never reads
-#: geometry and never reads CSS custom properties. Printing the stronger claim turns a limited
-#: thread-structure reading into an experimental conclusion about the whole UI, which is how a
-#: sidebar-drag campaign came to be scored 0 of 34 differing pairs by a digest whose own null
-#: control also returned 0 of 34: the instrument was not discriminating in either direction, and
-#: the banner said the DOM was identical.
CLAIM_STRUCTURAL = (
"thread-structure parity: the thread root and the declared overlay selectors serialise "
"identically on both arms, on screen and off. It does NOT cover the sidebar, computed layout "
@@ -1228,41 +1021,24 @@ CLAIM_BEHAVIOURAL = (
"copied"
)
-#: THE POLICY EVERY VERDICT IS JUDGED AGAINST, printed beside the claim.
-#:
-#: A bare "PARITY OK" reads as "the UI is unchanged" and none of the three modes can support that
-#: sentence. Each supports a narrower one, and which one is being made changes what a pass MEANS,
-#: so the policy is named next to the claim rather than left in a document somebody has to find.
-#:
-#: The exemptions are not loopholes. The first is a decision someone makes on the record with a
-#: number attached; the second is a definition, because rendering only what is visible is an
-#: accepted technique rather than a parity violation; the third is CONDITIONAL, and the condition
-#: is the part that is required -- the copy must be complete, and only the visual fidelity of the
-#: selection is given up. None of the three removes the need for a floor: a difference that is
-#: exempt still has to be shown to be the difference you think it is, which is what the null
-#: control is for.
-#:
-#: NO MODE GRANTS THE THIRD ON A DIGEST. `--mode behaviour` is the only one that speaks to it at
-#: all, through `clipboard_carries_the_whole_thread`, which scores the copy against the THREAD
-#: rather than against the other arm. Where a payload carries no readable `select_all_copy` the
-#: gate records that the exemption exists and does not grant it.
-#:
-#: And it scores it BY LENGTH. That is a proxy for completeness, not completeness itself, so the
-#: printed line says so rather than leaving "complete" to be read as a content comparison. A
-#: content comparison is not available here: the base arm's clipboard is the DOM's rendered text
-#: while a store-based copy is markdown SOURCE, so the two are different serialisations of one
-#: conversation and comparing their characters fails on a CORRECT build. The coverage band is what
-#: carries the weight, and it is the band that refused both defects this was written for --
-#: truncation at 0.61 of the thread and substitution at 2.16 -- so it is interpolated from the
-#: live constants by `behaviour_policy()` rather than written out here, where it could drift.
+#: THE POLICY EVERY VERDICT IS JUDGED AGAINST, printed beside the claim, because a bare "PARITY
+#: OK" reads as "the UI is unchanged" and no mode supports that sentence. The exemptions are not
+#: loopholes: the first is a decision made on the record with a number, the second is a
+#: definition, and the third is CONDITIONAL on the copy being complete. NO MODE GRANTS THE THIRD
+#: ON A DIGEST: only `--mode behaviour` speaks to it, through
+#: `clipboard_carries_the_whole_thread`, which scores the copy against the THREAD rather than the
+#: other arm. And it scores BY LENGTH, a proxy for completeness: the base arm's clipboard is
+#: rendered text while a store-based copy is markdown SOURCE, so comparing characters fails on a
+#: CORRECT build. The coverage band carries the weight -- it refused truncation at 0.61 and
+#: substitution at 2.16 -- so it is interpolated from the live constants.
POLICY = (
"UI and UX idempotency is required, with three exemptions: a deliberate difference accepted "
"for a dramatic performance improvement, a difference that exists only OFF SCREEN, and a "
"select-all that does not select all PROVIDED the copy it produces stays complete"
)
-#: What each mode can and cannot decide under that policy. The point of the second half of each
-#: line is that a reader can tell which sentence they are being handed.
+#: What each mode can and cannot decide under that policy; the second half of each line is what
+#: tells a reader which sentence they are being handed.
POLICY_BY_MODE = {
"structural": (
f"{POLICY}. This mode judges the FIRST requirement over the thread root and the declared "
@@ -1320,9 +1096,9 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
}
assert base is not None and treat is not None
- # THE POSITIVE CONTROL. A visibility scan that matched nothing has equal (empty) ordinal sets
- # and no differing digests, so without this it returns the strongest verdict available on the
- # strength of never having seen a single message. Exactly the failure `compare_styles` had.
+ # THE POSITIVE CONTROL: a visibility scan that matched nothing has equal (empty) ordinal sets and
+ # no differing digests, so without this it returns the strongest verdict available on the strength
+ # of never having seen a message. Exactly the failure `compare_styles` had.
bn, tn = len(base.get("ever_visible") or []), len(treat.get("ever_visible") or [])
if bn == 0 or tn == 0:
return {
@@ -1336,29 +1112,17 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
"claim": CLAIM_VISIBLE,
}
- # BEFORE THE TWO SETS ARE COMPARED, because a collision is what makes them untrustworthy.
- # `scene/parity.js` keys its per-message digests by ordinal, so two mounted rows sharing an
- # `aria-posinset` -- a virtualizer renumbering a recycled row wrongly -- leave one digest where
- # two rows were on screen, and `ever_visible` collapses them too. Both quantities below are then
- # short by a row, and whether the verdict came out MATCH or DIFFER would turn only on which of
- # the two survived, which is DOM order and nothing else.
- #
- # Read defensively so a payload recorded before the counter existed compares exactly as it did.
+ # BEFORE THE TWO SETS ARE COMPARED, because a collision is what makes them untrustworthy:
+ # `scene/parity.js` keys per-message digests by ordinal, so two mounted rows sharing an
+ # `aria-posinset` leave one digest where two rows were on screen and `ever_visible` collapses
+ # them too. Read defensively so a payload recorded before the counter existed compares as it did.
collisions = {
label: int(side.get("ordinal_collisions") or 0)
for label, side in (("base", base), ("treatment", treat))
}
- # A SEVERE FINDING OUTRANKS THIS REFUSAL, and exactly one finding qualifies. "One arm's
- # viewport ended EMPTY and the other's did not" is marked NOT SUPPRESSIBLE where it is raised,
- # because losing the thread is a different kind of statement from a capture that could not be
- # read -- and a collision provably cannot manufacture it. A collision needs TWO mounted rows
- # at one position, so the map it corrupts still has at least one entry in it; it can merge two
- # entries and never empty a map. So the one thing a collision cannot have caused is still
- # reported rather than swallowed by a blanket refusal.
- #
- # Only that one. Every other comparison below IS corrupted by a collision: `ever_visible` loses
- # an ordinal on one arm alone, so "the two arms put different messages on screen" can be
- # entirely an artefact of the collision, and the per-ordinal digests are short by a row.
+ # A SEVERE FINDING OUTRANKS THIS REFUSAL, and exactly one qualifies: "one arm's viewport ended
+ # EMPTY and the other's did not" is marked NOT SUPPRESSIBLE, because a collision needs TWO mounted
+ # rows at one position, so it can merge entries but never empty a map.
lost_the_thread = (len(base.get("messages") or {}) == 0) != (
len(treat.get("messages") or {}) == 0
)
@@ -1401,20 +1165,18 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
# never counted as agreement: this is the residue of comparing a windowed arm at one instant.
uncomparable = sorted(int(o) for o in map(str, sorted(bev)) if o not in bmsg or o not in tmsg)
moved = []
- # The subset of `moved` that CANNOT be a point in a stream, kept so the blind-probe refusal
- # below does not take it out with the rest. See each `settled.append` for why that row is
- # provably not the reply being written.
+ # The subset of `moved` that CANNOT be a point in a stream, kept so the blind-probe refusal below
+ # does not take it out with the rest. See each `settled.append` for why.
settled: list[str] = []
for ordinal in sorted(bev):
key = str(ordinal)
b, t = bmsg.get(key), tmsg.get(key)
if b is None or t is None:
continue
- # A ROLE IS NOT A POSITION IN A STREAM. It is captured beside the digest rather than inside
- # it, and how far a reply has arrived says nothing about whether it is the assistant's. So
- # a row that changed role is reported even while that row is in flight, where the digest
- # itself is withheld: a treatment that renders the live assistant row as `data-role="user"`
- # is a structural regression that used to leave here as NOT COMPARABLE.
+ # A ROLE IS NOT A POSITION IN A STREAM: it is captured beside the digest, and how far a reply has
+ # arrived says nothing about whose message it is. So a row that changed role is reported even
+ # while in flight -- a treatment rendering the live assistant row as `data-role="user"` used to
+ # leave here as NOT COMPARABLE.
if b.get("role") != t.get("role"):
claim = f"ordinal {ordinal}:role {b.get('role')}->{t.get('role')}"
moved.append(claim)
@@ -1422,34 +1184,25 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
continue
if b.get("digest") != t.get("digest"):
if b.get("in_flight") or t.get("in_flight"):
- # STILL BEING WRITTEN ON ONE ARM OR THE OTHER, so its digest names a point in a
- # stream. Residue, exactly like an ordinal that was unmounted before the capture:
- # it cannot be a difference, and it cannot be an agreement either, so it joins the
- # list that refuses the verdict below rather than the list that fails it. Same
- # rule as `compare` applies to the structural digest; the two modes must not
- # disagree about which readings have a defined moment.
+ # STILL BEING WRITTEN ON ONE ARM OR THE OTHER, so its digest names a point in a stream: residue,
+ # exactly like an ordinal unmounted before the capture, so it joins the list that refuses the
+ # verdict rather than the one that fails it. Same rule `compare` applies to the structural digest.
uncomparable.append(ordinal)
continue
claim = f"ordinal {ordinal}({b.get('role')}):{b.get('chars')}->{t.get('chars')}c"
moved.append(claim)
- # A USER ROW IS NEVER THE REPLY BEING WRITTEN. Both arms agree this ordinal is the
- # user's, and a stream writes into an assistant message, so this difference survives
- # the blind-probe refusal below. The two arms agreeing is what makes it provable: a
- # row whose role DISAGREES is reported above as a role change, not silently trusted.
+ # A USER ROW IS NEVER THE REPLY BEING WRITTEN, and both arms agreeing on the role is what makes
+ # it provable; a row whose role DISAGREES is reported above as a role change.
if b.get("role") == "user":
settled.append(claim)
- # Ordinals that were unmounted before the capture and ordinals that were still streaming are
- # the same kind of residue and are counted once each.
+ # Ordinals unmounted before the capture and ordinals still streaming are the same kind of residue
+ # and are counted once each.
uncomparable = sorted(set(uncomparable))
- # ONE VIEWPORT ENDED EMPTY AND THE OTHER DID NOT, which is as visible a difference as there is
- # and used to be reported as NOT COMPARABLE -- a refusal, not a finding.
- #
- # This is not hypothetical and it is why the check exists. On the 100K virtualization arm,
- # `model_change` took the thread from 12 mounted messages to 0 and it never came back: the
- # census reads 0 messages and 2,107 elements for the rest of the film, and three later actions
- # could not run at all. Both arms had shown the same ordinals EARLIER in the action, so the
- # union matched and every per-ordinal digest was simply missing on one side. Comparing only the
- # union cannot see that; comparing what each arm could still show at the end can.
+ # ONE VIEWPORT ENDED EMPTY AND THE OTHER DID NOT, as visible a difference as there is, and it used
+ # to be reported as NOT COMPARABLE. Not hypothetical: on the 100K virtualization arm
+ # `model_change` took the thread from 12 mounted messages to 0 and it never came back. Both arms
+ # had shown the same ordinals earlier, so the union matched and every per-ordinal digest was
+ # simply missing on one side -- comparing what each arm could still show at the end sees that.
b_left, t_left = len(bmsg), len(tmsg)
if (b_left == 0) != (t_left == 0):
empty, full = ("treatment", "base") if t_left == 0 else ("base", "treatment")
@@ -1463,34 +1216,22 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
),
"moved": [f"ordinal {o}" for o in sorted(bev)[:8]],
"claim": CLAIM_VISIBLE,
- # NOT SUPPRESSIBLE BY THE NOISE FLOOR. The floor exists for actions whose visible
- # region differs between two runs of the same build, which is a volatile attribute on
- # rows of identical length. Losing the entire thread is a different kind of statement,
- # and an action can easily be BOTH: on the 100K run `model_change` is in the derived
- # unstable set because the null control's copy of it differs on an attribute, and that
- # would have silenced "the treatment arm's viewport ended empty" -- suppressing a lost
- # conversation on the strength of unrelated jitter in the same action.
+ # NOT SUPPRESSIBLE BY THE NOISE FLOOR: the floor exists for actions whose visible region differs
+ # between two runs of one build, and losing the entire thread is a different kind of statement.
+ # An action can be both -- `model_change` is in the derived unstable set for an unrelated
+ # attribute, which would have silenced "the treatment arm's viewport ended empty".
"severe": True,
}
- # THE STREAMING PROBE, on the same footing it has in `compare`, because this mode is scored
- # from its own payload and never sees the structural verdict. `in_flight` above walks the
- # `data-status` / `aria-busy` hooks, so a build that renames them reports every row settled;
- # the two arms are two different builds, so that can be true on ONE of them while the other's
- # reply has genuinely finished, and the differing digests below are then two points in a stream
- # scored as a rendering difference.
- #
- # AFTER the two lost-conversation findings above and BEFORE the digest comparison, for the
- # reason `compare` puts it after `mount_count_mismatch`: different messages on screen, or one
- # viewport emptied, are readings that do not depend on the stream split at all, and a build
- # that loses the thread while a reply runs stays a finding rather than a shrug.
+ # THE STREAMING PROBE, on the same footing it has in `compare`, because this mode is scored from
+ # its own payload: `in_flight` walks the `data-status` / `aria-busy` hooks, so a build that
+ # renames them reports every row settled while the other arm's reply has genuinely finished.
+ # AFTER the two lost-conversation findings and BEFORE the digest comparison, since different
+ # messages on screen do not depend on the stream split at all.
blind = streaming_probe(base, treat)
if blind is not None:
- # SAME RULE AS `compare`: the refusal covers the rows whose meaning depends on where the
- # stream had got to, and nothing else. `settled` holds the rows that provably cannot be the
- # reply being written -- a row both arms call the user's, and a row whose role itself
- # changed -- and those are reported rather than discarded. Without this a changed user
- # message left here as NOT COMPARABLE with an empty `moved`, and `visible_report` buckets
- # a refusal as blind and never consults it for the exit code.
+ # SAME RULE AS `compare`: the refusal covers the rows whose meaning depends on where the stream
+ # had got to and nothing else. `settled` holds the rows that provably cannot be the reply being
+ # written, and without this a changed user message left as NOT COMPARABLE with an empty `moved`.
if settled:
return {
"verdict": DIFFER,
@@ -1519,25 +1260,14 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
"not_digested": uncomparable,
}
if uncomparable:
- # ANY residue refuses the verdict, not only a total one. This branch used to be entered
- # solely when EVERY visible ordinal was undigestable (`len(uncomparable) == len(bev)`), so a
- # scrolling or windowed action that put six messages on screen and unmounted one of them
- # before the capture returned MATCH on the strength of the other five. The claim printed
- # above quantifies over EVERY message the viewport showed; one message missing makes it
- # unsupported, and the rendering difference this mode exists to catch could be in exactly
- # the message that is missing. `visible_report` never printed the `not_digested` residue
- # either, so that pair exited 0 with nothing on screen to say a message went uncompared.
- #
- # WHY THE VERDICT AND NOT MERELY THE PASS COUNT. Demoting it inside `visible_report` alone
- # would leave the row itself reading `match`, and every other consumer of that row counts a
- # match as agreement: `visible_unstable_set` reads verdicts to derive the noise floor,
- # `summarise` tallies them, and the payload keeps them verbatim for whoever reads it next.
- # The honest outcome has to live in the verdict, which is the one thing they all share.
- #
- # THE COST, MEASURED on the two real 100K films rather than guessed. On the windowed arm 4
- # of 39 matching pairs carry a residue, all of them `stop_generation`; on the base-vs-base
- # null control 7 of 43 do. The mode keeps a discriminating majority on both, so this buys
- # honesty at four pairs in sixty-four.
+ # ANY residue refuses the verdict, not only a total one. Entered solely when EVERY visible ordinal
+ # was undigestable, a windowed action that put six messages on screen and unmounted one returned
+ # MATCH on the strength of the other five, while the printed claim quantifies over EVERY message
+ # the viewport showed. WHY THE VERDICT AND NOT MERELY THE PASS COUNT: demoting inside
+ # `visible_report`, which never printed the `not_digested` residue either, alone leaves the row
+ # reading `match`, which everything downstream counts as
+ # agreement. THE COST, MEASURED on two real 100K films: four pairs in sixty-four.
+ # `compare` puts this after `mount_count_mismatch`.
digested = len(bev) - len(uncomparable)
return {
"verdict": NOT_COMPARABLE,
diff --git a/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py b/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py
index 3e0b7469c50..0fc9198f7c8 100644
--- a/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py
+++ b/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py
@@ -181,8 +181,7 @@ def test_a_pair_where_nothing_visible_could_be_digested_is_not_a_pass():
"""Every ordinal the viewport showed had been unmounted by capture time, so the comparison
observed the visibility but none of the content. That is not agreement."""
got = P.compare_visible(_cap({}, ever = [3, 4]), _cap({}, ever = [3, 4]))
- # The zero-length scan control fires first, and either refusal is correct; what must not
- # happen is a MATCH.
+ # The zero-length scan control fires first, and either refusal is correct; what must not happen is a MATCH.
assert got["verdict"] == P.NOT_COMPARABLE, got
@@ -270,16 +269,15 @@ def test_every_mode_names_the_policy_it_is_judging_against():
# The behavioural mode grants neither of the first two and is the only one that speaks to the
# third, so "either" would be the wrong word for it now.
assert "cannot grant the performance or off-screen exemptions" in P.POLICY_BY_MODE["behaviour"]
- # AND IT SAYS HOW IT MEASURES THE CONDITION. "Complete" on its own reads as a comparison of
- # the copied content; what the gate does is divide each arm's clipboard length by the thread's
- # visible text and require the ratio to land in a band. Saying which of those it is decides
- # whether a reader is entitled to conclude the copy was intact, so the weaker wording is not
- # allowed back: the line has to name the measure AND disclaim the one it does not perform.
+ # AND IT SAYS HOW IT MEASURES THE CONDITION. "Complete" alone reads as a comparison of the copied
+ # content; what the gate does is divide each arm's clipboard length by the thread's visible text
+ # and require the ratio to land in a band. Which of those it is decides whether a reader may
+ # conclude the copy was intact, so the line has to name the measure AND disclaim the other.
assert "BY LENGTH" in P.POLICY_BY_MODE["behaviour"]
assert "does not compare the copied characters" in P.POLICY_BY_MODE["behaviour"]
assert "records the exemption rather than granting it" in P.POLICY_BY_MODE["behaviour"]
- # The floor survives the exemption. An exemption changes what counts as a pass; a measurement
- # with no floor under it is not a pass in the first place.
+ # The floor survives the exemption. An exemption changes what counts as a pass; a measurement with
+ # no floor under it is not a pass in the first place.
assert "does not remove the floor" in P.POLICY_BY_MODE["visible"]
@@ -302,8 +300,8 @@ def test_the_policy_line_is_printed_next_to_every_claim_line():
for name in claims:
assert f"P.{name}" in source, f"{name} is never printed"
for mode in P.POLICY_BY_MODE:
- # Either printed straight from the table, or through the per-mode helper that fills in
- # the numbers that mode is actually enforcing.
+ # Either printed straight from the table, or through the per-mode helper that fills in the numbers
+ # that mode is enforcing.
assert (
f"POLICY_BY_MODE['{mode}']" in source or f"{mode}_policy(" in source
), f"the {mode} policy line is never printed"
diff --git a/tests/studio/studiobench/analysis/symbols.py b/tests/studio/studiobench/analysis/symbols.py
index 052deb6e0f5..81daee8d946 100644
--- a/tests/studio/studiobench/analysis/symbols.py
+++ b/tests/studio/studiobench/analysis/symbols.py
@@ -60,28 +60,22 @@ FAILED = "failed"
OK = "ok"
NOT_BUILT = "not_built"
-# A function whose counts are all this small carries almost no information and
-# collides with everything. Excluded from matching, and the exclusion is
-# reported so the coverage of the bridge is visible.
+# A function whose counts are all this small carries almost no information and collides with
+# everything. Excluded from matching, and the exclusion is reported so the bridge's coverage is
+# visible.
MIN_INFORMATIVE_COUNT = 2
-# Above this share of resolved mappings being name-identical (`push` -> `push`),
-# the two arms are almost certainly the same build. Some identity is EXPECTED
-# and healthy: React's release minifier leaves scheduler entry points such as
-# `push`, `peek` and `performWorkUntilDeadline` alone, and `keepNames` preserves
-# others, so a genuine bridge measured on a real React 19.2.4 pair came in at
-# roughly 0.1. A same-build pair comes in at exactly 1.0, so the two regimes are
-# not close and this threshold is not delicate.
+# Above this share of resolved mappings being name-identical (`push` -> `push`), the two arms are
+# almost certainly the same build. Some identity is EXPECTED: React's release minifier leaves
+# scheduler entry points alone and `keepNames` preserves others, so a genuine bridge on a real
+# React 19.2.4 pair came in at roughly 0.1 against exactly 1.0 for a same-build pair.
+# Untouched: `push`, `peek` and `performWorkUntilDeadline`.
MAX_IDENTITY_MAPPING_FRACTION = 0.5
-# NOTE FOR THE NEXT PERSON, because this is the obvious idea and it DOES NOT
-# WORK. You cannot detect "is this the development build?" by looking at how
-# long the function names are. Measured on the real pair: the development bundle
-# had a median react-dom function-name length of 19 with 97.6% of names at four
-# characters or more, and the PRODUCTION bundle had a median of 19 with 95.4%.
-# React's own minifier renames only part of react-dom, `keepNames` preserves the
-# rest, and V8 reports an inferred name for much of what is left. Name shape
-# tells you nothing. The checks below are structural instead.
+# NOTE FOR THE NEXT PERSON: you cannot detect 'is this the development build?' from function-name
+# length. Measured on the real pair, the development bundle had a median react-dom name length of
+# 19 with 97.6% of names four characters or more, and the PRODUCTION bundle a median of 19 with
+# 95.4%. The checks below are structural instead.
@dataclass(frozen = True)
@@ -123,8 +117,8 @@ class Bridge:
ambiguous_prod: list[str] = field(default_factory = list)
ambiguous_dev: list[str] = field(default_factory = list)
unmatched_prod: int = 0
- # Resolved functions whose dev name equals their prod name. Expected to be
- # small and non-zero; near 100% means both arms were the same build.
+ # Resolved functions whose dev name equals their prod name. Expected to be small and non-zero;
+ # near 100% means both arms were the same build.
identity_mappings: int = 0
anchors_checked: int = 0
anchor_failures: list[str] = field(default_factory = list)
@@ -132,9 +126,8 @@ class Bridge:
@staticmethod
def prod_key(url: str, start_offset: int, end_offset: int) -> str:
- # The URL is reduced to its basename because an Unsloth install serves the
- # same bundle from a hashed path that changes per install, while the
- # offsets inside the bundle do not.
+ # The URL is reduced to its basename because an Unsloth install serves the same bundle from a
+ # hashed path that changes per install, while the offsets inside it do not.
return f"{os.path.basename(url)}:{start_offset}:{end_offset}"
def resolve(self, url: str, start_offset: int, end_offset: int) -> str | None:
@@ -408,16 +401,12 @@ def build_bridge(
)
return bridge
+ # The two arms must actually be two different builds. THE FAILURE THIS CLOSES: if both arms are
+ # pointed at the same server, every count vector matches trivially, EVERY ANCHOR PASSES because
+ # an anchor genuinely does map to itself, and the bridge reports `ok` with a mapping of `Zk` to
+ # `Zk`. The anchor check cannot catch it: anchors test invariance and same-build is perfectly
+ # invariant.
# ---- the two arms must actually be two different builds -----------------
- #
- # THE FAILURE THIS CLOSES. If both arms are pointed at the same server (a
- # port collision, a reused base URL, a dev server that quietly served the
- # production dist), every count vector matches trivially, EVERY ANCHOR
- # PASSES because an anchor genuinely does map to itself, and the bridge
- # reports `ok` with a mapping of `Zk` to `Zk`. That is a clean-looking run
- # that proves nothing, and it would put "bridged" next to a minified name.
- # The anchor check cannot catch it; it is the one thing anchors are blind
- # to, because anchors test invariance and same-build is perfectly invariant.
dev_all_keys = {v.key for v in dev_all if v.informative}
prod_all_keys = {v.key for v in prod_all if v.informative}
if dev_all_keys and dev_all_keys == prod_all_keys:
@@ -461,10 +450,8 @@ def build_bridge(
)
return bridge
- # Second guard on the same failure, for the case where the two arms differ
- # in offsets but are still the same code (a rebuild of one tree, say). If
- # nearly every resolved name maps to itself, no minification was undone and
- # the bridge is not doing anything.
+ # Second guard on the same failure, for arms that differ in offsets but are still the same code:
+ # if nearly every resolved name maps to itself, no minification was undone.
fraction = identical_names / matched
if fraction > MAX_IDENTITY_MAPPING_FRACTION:
bridge.status = FAILED
diff --git a/tests/studio/studiobench/analysis/test_analysis.py b/tests/studio/studiobench/analysis/test_analysis.py
index 77e96cf5fed..426fda196e2 100644
--- a/tests/studio/studiobench/analysis/test_analysis.py
+++ b/tests/studio/studiobench/analysis/test_analysis.py
@@ -60,8 +60,8 @@ def test_loads_object_form_trace() -> None:
def test_truncated_trace_fails_the_cell_rather_than_parsing_short() -> None:
- # A stream that was cut off mid-document must not degrade into "a shorter
- # trace"; that reads exactly like the expensive work not happening.
+ # A stream cut off mid-document must not degrade into 'a shorter trace'; that reads exactly like
+ # the expensive work not happening.
try:
Trace.from_json_text('{"traceEvents":[{"ph":"X","ts":1,"dur":2,"name":"RunTask"')
except CellFailure as exc:
@@ -127,8 +127,8 @@ def test_profile_parses_and_deltas_match_wall_clock() -> None:
def test_nodes_accumulate_across_chunks() -> None:
- # Only a handful of chunks carry a `nodes` array; a parser that reads nodes
- # per chunk and forgets them resolves almost nothing.
+ # Only a handful of chunks carry a `nodes` array; a parser that reads nodes per chunk and forgets
+ # them resolves almost nothing.
tr = _trace()
chunks_with_nodes = sum(
1
@@ -178,8 +178,8 @@ def test_stacks_have_ancestry() -> None:
def test_clock_anchor_is_the_event_timestamp() -> None:
- # `args.data.startTime` is in a different clock domain and is kept only to
- # report the skew, never used to anchor samples.
+ # `args.data.startTime` is in a different clock domain and is kept only to report the skew, never
+ # used to anchor samples.
prof = C.main_thread_profile(_trace())
assert prof.declared_start_time
assert abs(prof.clock_skew_us) < 10_000
@@ -228,8 +228,8 @@ def test_timers_and_frames_and_input_are_all_present() -> None:
def test_harness_cost_is_named_not_hidden() -> None:
cls = K.classify_thread(_trace())
- # The devtools pipe running Runtime.evaluate is our own cost. It must have
- # its own column so it can be watched, and must not be inside the app's.
+ # The devtools pipe running Runtime.evaluate is our own cost. It must have its own column so it
+ # can be watched, and must not be inside the app's.
assert K.AGENT_IPC in K.ORIGINS
assert K.AGENT_IPC in K.HARNESS_ORIGINS
@@ -434,9 +434,8 @@ def test_bridge_resolves_a_minified_name_by_count_vector() -> None:
assert b.status == S.OK
assert b.resolve("/assets/index-abc123.js", 0, 5) == "cloneChildFibers"
assert b.resolve("/assets/index-abc123.js", 10, 15) == "beginWork"
- # The anchor legitimately maps to itself, so some identity is expected. What
- # must not happen is identity DOMINATING, which would mean no minification
- # was undone.
+ # The anchor legitimately maps to itself, so some identity is expected; what must not happen is
+ # identity DOMINATING, which would mean no minification was undone.
assert b.identity_mappings / len(b.mapping) <= S.MAX_IDENTITY_MAPPING_FRACTION
@@ -484,10 +483,9 @@ def test_anchor_mismatch_discards_the_whole_bridge() -> None:
def test_same_build_on_both_arms_is_refused() -> None:
- # The one failure anchors are structurally blind to. If both arms are the
- # same build, every anchor maps to itself PERFECTLY, so anchor validation
- # passes and the bridge reports ok while mapping minified names to
- # themselves. Verified against the real implementation before this guard
+ # The one failure anchors are structurally blind to: if both arms are the same build every anchor
+ # maps to itself PERFECTLY, so anchor validation passes and the bridge reports ok while mapping
+ # minified names to themselves. Verified against the real implementation before this guard
# existed: it returned status ok with {"Zk": "Zk"}.
_, prod = _arms(
{"a": [1, 1]},
@@ -512,8 +510,8 @@ def test_same_build_on_both_arms_is_refused() -> None:
def test_an_all_identity_mapping_is_refused() -> None:
- # Different scripts and offsets, but no minification was undone: every
- # resolved name is its own name, so the bridge is doing nothing.
+ # Different scripts and offsets, but no minification was undone: every resolved name is its own
+ # name, so the bridge is doing nothing.
dev, prod = _arms(
{"Zk": [340, 3400], "qi": [17, 170]},
{"Zk": [340, 3400], "qi": [17, 170]},
@@ -645,8 +643,8 @@ def test_missing_page_counters_are_skipped_loudly() -> None:
def test_page_counter_contract_is_documented() -> None:
- # Layer 3 emits against these exact key names; a rename here is a contract
- # change and must be agreed, not discovered at analysis time.
+ # Layer 3 emits against these exact key names; a rename here is a contract change and must be
+ # agreed, not discovered at analysis time.
assert set(O.PAGE_COUNTER_CONTRACT) == {"m2_reparse", "m3_forced_layout"}
assert "chars_rescanned" in O.PAGE_COUNTER_CONTRACT["m2_reparse"]
assert "forced_layouts" in O.PAGE_COUNTER_CONTRACT["m3_forced_layout"]
diff --git a/tests/studio/studiobench/analysis/test_instruments_live.py b/tests/studio/studiobench/analysis/test_instruments_live.py
index 92a5d842de7..1ecf5484485 100644
--- a/tests/studio/studiobench/analysis/test_instruments_live.py
+++ b/tests/studio/studiobench/analysis/test_instruments_live.py
@@ -45,8 +45,8 @@ from studiobench.runtime.types import ( # noqa: E402
new_session_id,
)
-# A page with a known shape: a message-channel loop (the React scheduler's
-# mechanism), a timer loop, and a function whose call count is exactly known.
+# A page with a known shape: a message-channel loop (the React scheduler's mechanism), a timer
+# loop, and a function whose call count is exactly known.
PAGE = """<!doctype html><meta charset=utf-8><body><div id=o></div><script>
window.__N = 0;
window.__junk = [];
@@ -205,8 +205,8 @@ def _check_level(level, wrows, crows, ground_truth) -> int:
except Exception as exc: # noqa: BLE001
fail(f"window row is not JSON-safe: {exc}")
- # Overhead is mandatory from every instrument at level >= 1, and it is what
- # the report layer's overhead_growth_with_length gate consumes.
+ # Overhead is mandatory from every instrument at level >= 1, and it is what the report layer's
+ # overhead_growth_with_length gate consumes.
for name, payload in crows.items():
if "overhead_ms" not in payload or "overhead_ms_attempted" not in payload:
fail(f"{name}.end_cell has no overhead_ms/overhead_ms_attempted")
@@ -225,8 +225,8 @@ def _check_level(level, wrows, crows, ground_truth) -> int:
if by_origin.get("message_channel", 0) < MESSAGE_ITERATIONS * 0.8:
fail(f"expected ~{MESSAGE_ITERATIONS} message-channel tasks, got {by_origin}")
- # THE LADDER. Naming requires the v8 profiler, which only L2+ turns on. At
- # L1 that must be an explicit null with a reason, never an empty list.
+ # THE LADDER. Naming requires the v8 profiler, which only L2+ turns on; at L1 that must be an
+ # explicit null with a reason, never an empty list.
if level == 1:
if tracing.get("named_frames") is not None:
fail("L1 must not report named frames; the profiler category is off")
diff --git a/tests/studio/studiobench/analysis/traceparse.py b/tests/studio/studiobench/analysis/traceparse.py
index e9a0a8e9d34..35f3c6a6cb7 100644
--- a/tests/studio/studiobench/analysis/traceparse.py
+++ b/tests/studio/studiobench/analysis/traceparse.py
@@ -37,18 +37,17 @@ from typing import Any, Iterable, Iterator, Sequence
from . import CellFailure
-# Complete-duration events are the only phase that forms the task tree. Async
-# (`b`/`e`/`n`), flow (`s`/`t`/`f`), instant (`I`/`R`), sample (`P`), counter
-# (`C`) and metadata (`M`) events are carried alongside but never nested, since
-# their timestamps do not describe a stack.
+# Complete-duration events are the only phase that forms the task tree. Async (`b`/`e`/`n`), flow
+# (`s`/`t`/`f`), instant (`I`/`R`), sample (`P`), counter (`C`) and metadata (`M`) events are
+# carried alongside but never nested, since their timestamps do not describe a stack.
_PHASE_COMPLETE = "X"
_PHASE_BEGIN = "B"
_PHASE_END = "E"
-# When two events share both `ts` and `dur`, the trace gives no ordering. On the
-# main thread the devtools view (`RunTask`) is conceptually the outer frame and
-# the scheduler view (`ThreadControllerImpl::RunTask`) the inner one, so pin
-# that order rather than letting dict ordering decide which is the parent.
+# When two events share both `ts` and `dur` the trace gives no ordering. On the main thread the
+# devtools view (`RunTask`) is conceptually the outer frame and the scheduler view
+# (`ThreadControllerImpl::RunTask`) the inner one, so pin that order rather than letting dict
+# ordering decide.
_OUTERMOST_FIRST = {
"RunTask": 0,
"ThreadControllerImpl::RunTask": 1,
@@ -137,7 +136,7 @@ class Trace:
self._thread_names[key] = name # type: ignore[index]
self._threads: dict[tuple[int, int], Thread] = {}
- # ---------------------------------------------------------------- loading
+ # loading
@classmethod
def from_json_text(cls, text: str) -> "Trace":
@@ -154,8 +153,8 @@ class Trace:
try:
doc = json.loads(text)
except json.JSONDecodeError as exc:
- # A truncated JSON document is the signature of a drained stream that
- # was cut short. That is a failed cell, never a short trace.
+ # A truncated JSON document is the signature of a drained stream that was cut short: a failed cell,
+ # never a short trace.
raise CellFailure(
"trace_truncated",
f"trace JSON did not parse ({exc}); {len(text)} bytes drained",
@@ -303,9 +302,9 @@ def build_tree(events: Iterable[dict[str, Any]]) -> list[Task]:
for t in complete:
while stack and t.ts >= stack[-1].end:
stack.pop()
- # An event that starts inside its would-be parent but ends after it is
- # not nested; the trace is inconsistent there. Treat it as a sibling
- # rather than corrupting self-time arithmetic for the whole subtree.
+ # An event that starts inside its would-be parent but ends after it is not nested; the trace is
+ # inconsistent there, so treat it as a sibling rather than corrupting self-time arithmetic for the
+ # whole subtree.
while stack and t.end > stack[-1].end:
stack.pop()
if stack:
diff --git a/tests/studio/studiobench/arms/calibration.py b/tests/studio/studiobench/arms/calibration.py
index 56efe748037..4671b3627ae 100644
--- a/tests/studio/studiobench/arms/calibration.py
+++ b/tests/studio/studiobench/arms/calibration.py
@@ -41,13 +41,13 @@ from ..scoring.schema import Measure
from .manifest import Arm, ArmOutcome, ArmStatus, Invariance, PotencyCounter
#: The spike sizes every batch runs. 0.1 ms is below any plausible per-update mechanism and is
-#: expected to be INVISIBLE; it is included precisely so the detection floor has something to sit
-#: on. 2.0 ms is comfortably above, and must be seen or the instrument is broken.
+#: expected to be INVISIBLE, so the detection floor has something to sit on; 2.0 ms is comfortably
+#: above and must be seen or the instrument is broken.
SPIKE_SIZES_MS: tuple[float, ...] = (0.1, 0.5, 2.0)
-#: A spike is considered recovered when the observed delta is this close to the burned cost.
-#: Wide on purpose: the point is "the instrument sees roughly the right amount", not calibration
-#: to three digits, and a tight band would fail on a machine that is merely noisy.
+#: A spike is considered recovered when the observed delta is this close to the burned cost. Wide on
+#: purpose: the point is "roughly the right amount", not calibration to three digits, and a tight
+#: band would fail on a merely noisy machine.
RECOVERY_MIN = 0.5
RECOVERY_MAX = 2.0
diff --git a/tests/studio/studiobench/arms/content_visibility_probe.js b/tests/studio/studiobench/arms/content_visibility_probe.js
index bb4260be4b9..815164a4de7 100644
--- a/tests/studio/studiobench/arms/content_visibility_probe.js
+++ b/tests/studio/studiobench/arms/content_visibility_probe.js
@@ -1,68 +1,34 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-/* POTENCY PROBE for `content-visibility: auto` on Unsloth's chat message roots.
+/* POTENCY PROBE for `content-visibility: auto` on Unsloth's chat message roots. Installed through
+ * `SBENCH_EXTRA_INIT_SCRIPT`, reporting through `SBENCH_PAGE_CONSOLE`; needs `--password` (and
+ * `--password-b`, which defaults to `--password` and is wrong when two Unsloth instances are
+ * attached, since each mints its own bootstrap password). A run carrying this is a PROBE RUN and
+ * its payload is never scored: the probe forces layout on every sample, including the rendering it
+ * asks about.
+ * Without it the run dies on an HTTP 401 only after the browser has started. The invocation is
+ * SBENCH_EXTRA_INIT_SCRIPT=tests/studio/studiobench/arms/content_visibility_probe.js
+ * SBENCH_PAGE_CONSOLE="CVPOT " python -m tests.studio.studiobench --tier fast --ab probe
+ * --attach URL --attach-b URL --password ... --password-b ...; see the studiobench README.
*
- * Installed through `SBENCH_EXTRA_INIT_SCRIPT`, reporting through `SBENCH_PAGE_CONSOLE`:
+ * WHY A PROBE AT ALL. A null from an arm that never fired measures the cascade, or a selector, or
+ * the fact that nothing was off screen -- a null already produced twice here. So nothing times
+ * anything; it answers only whether the browser genuinely skipped rendering an off-screen subtree.
+ * Three independent answers, weakest first: `cvAuto` (computed `auto`, proving only that the
+ * declaration won the cascade), `skipEvents` (`contentvisibilityautostatechange` with `skipped`,
+ * fired by the engine alone, so a non-zero count is proof), and `offUnrendered` (descendants with
+ * no layout boxes -- the route everybody reaches for first, which DOES NOT WORK; kept only so its
+ * zero is on the record. See `descendantBoxes`).
*
- * SBENCH_EXTRA_INIT_SCRIPT=tests/studio/studiobench/arms/content_visibility_probe.js \
- * SBENCH_PAGE_CONSOLE="CVPOT " \
- * python -m tests.studio.studiobench --tier fast --ab probe --out outputs/probe \
- * --attach http://127.0.0.1:PORT --attach-b http://127.0.0.1:OTHER \
- * --password "$(cat "$STUDIO_HOME_A/auth/.bootstrap_password")" \
- * --password-b "$(cat "$STUDIO_HOME_B/auth/.bootstrap_password")"
+ * IT ALSO WATCHES THE SIZING TRAP. Size containment while skipping means a skipped root's height is
+ * its `contain-intrinsic-size`; with `auto` that is the LAST REMEMBERED SIZE (css-sizing-4 5.2),
+ * and a root never rendered without containment falls back to the <length>. Both wreck scroll
+ * geometry and they differ, so scrollHeight, the roots on the fallback and the roots on their
+ * PADDING ALONE (a remembered size of zero) are reported per sample, read against the unarmed side.
*
- * The credential flags are spelled out rather than trailed off with an ellipsis. A probe run
- * drives a real Unsloth like every other command in the loop, and without `--password` it dies on
- * an HTTP 401 only after the browser has already started. See "You need an Unsloth, and you need
- * its password" in the studiobench README.
- *
- * ONE PASSWORD PER ARM when both arms are attached. Two separately booted Unsloth instances mint two
- * different bootstrap passwords, so reusing the first for the treatment is a 401 on the second
- * arm only, after the browser is up. `--password-b` defaults to `--password`, which is right
- * for the single-Unsloth case and wrong for this one.
- *
- * A run carrying this is a PROBE RUN and its payload is never scored: the probe forces layout on
- * every sample, and one of the things it forces is the very rendering it is asking about. See the
- * warning on `descendantBoxes`.
- *
- * WHY A PROBE AT ALL, AND WHY IT RUNS BEFORE ANY TIMING IS COLLECTED.
- *
- * A null from an arm that never fired is not a measurement of the mechanism; it is a measurement
- * of the cascade, or of a selector, or of the fact that nothing was ever off screen. This exact
- * null has already been produced twice in this codebase for `content-visibility`. So nothing here
- * times anything. It answers one question: did the browser genuinely skip rendering an off-screen
- * subtree, on the elements this arm targets, in the running app.
- *
- * THREE INDEPENDENT ANSWERS, WEAKEST FIRST.
- *
- * 1. cvAuto elements whose COMPUTED content-visibility is `auto`. This proves the
- * declaration won the cascade and nothing more. An element can compute to
- * `auto` and be painted on every single frame because it never stopped
- * being relevant to the user.
- * 2. skipEvents `contentvisibilityautostatechange` events with `skipped === true`. This
- * event is fired by the engine's own relevance machinery and by nothing
- * else: no stylesheet, no selector and no author code can produce one. A
- * non-zero count is proof that a subtree was locked and its rendering
- * skipped.
- * 3. offUnrendered off-screen armed roots whose element descendants generate NO layout
- * boxes. This is the route everybody reaches for first and it DOES NOT
- * WORK for `content-visibility: auto`. It is kept, and reported, only so
- * that its zero is on the record next to a non-zero skip count rather than
- * being rediscovered. See `descendantBoxes`.
- *
- * IT ALSO WATCHES THE SIZING TRAP. `content-visibility: auto` imposes size containment while
- * skipping, so a skipped root's height is its `contain-intrinsic-size`. With the `auto` keyword
- * that is the LAST REMEMBERED SIZE (css-sizing-4 5.2, 5.2.1), and an element that has never been
- * rendered without size containment has none and falls back to the <length>. Both failure modes
- * wreck scroll geometry on a long thread and they are different, so the viewport's scrollHeight,
- * the count of roots sitting on the declared fallback, and the count sitting on their PADDING
- * ALONE (a remembered size of zero) are all reported every sample, to be read against the
- * unarmed side of the same session.
- *
- * OUT OF THE PAGE VIA THE CONSOLE. Unsloth ships `connect-src 'self'`, so a beacon to a collector
- * on another port is blocked by CSP before it is sent. The console is the one channel that costs
- * nothing and cannot be silently dropped.
+ * Reporting goes out via the console: Unsloth ships `connect-src \'self\'`, so a beacon to another
+ * port is blocked by CSP before it is sent.
*/
(function () {
"use strict";
@@ -73,32 +39,25 @@
var SAMPLE_MS = 2000;
var MAX_ROOTS_SCANNED = 60;
var MAX_DESCENDANTS_PER_ROOT = 40;
- /* Per role, the two heights a skipped root can land on, and they mean OPPOSITE things:
- *
- * fallback the declared `contain-intrinsic-size` length, used because no last remembered
- * size exists yet. The arm is working as designed and the length is a guess.
- * padding the root's own padding and nothing else, which is a remembered size of ZERO,
- * recorded while the root was mounted and still empty. That is the trap.
- *
- * They have to be told apart or the probe attributes one to the other, and on this app they
- * are close enough to collide: the user root's fallback is 60px against 40px of padding. So
- * each is matched against its OWN target height, the fallback first so that it wins ties, and
- * a root that is neither is counted as neither. A single `height <= 64` bucket put a user root
- * sitting exactly on its fallback into both counters at once. */
+ /* Per role, the two heights a skipped root can land on, meaning OPPOSITE things: `fallback` is the
+ * declared `contain-intrinsic-size`, used because no last remembered size exists yet (working as
+ * designed); `padding` is the root's own padding alone, a remembered size of ZERO recorded while
+ * the root was mounted and empty, which is the trap. On this app they collide -- the user root's
+ * fallback is 60px against 40px of padding -- so each is matched against its OWN target height,
+ * fallback first so it wins ties, and a root that is neither is counted as neither. A single
+ * `height <= 64` bucket put a user root sitting exactly on its fallback into both counters. */
var ROLE_PX = {
assistant: { fallback: 300, padding: 18 },
user: { fallback: 60, padding: 40 }
};
- /* `getBoundingClientRect().height` is the BORDER BOX, so both targets carry the root's own
- * padding: an assistant root on its 300px fallback measures 318, and one whose remembered
- * size is zero measures 18. Comparing the rect against the bare declared length instead left
- * `fallbackBite` structurally pinned at zero, which reads as "the fallback is never used" no
- * matter what the browser did. */
+ /* `getBoundingClientRect().height` is the BORDER BOX, so both targets carry the root's padding: an
+ * assistant root on its 300px fallback measures 318. Comparing against the bare declared length
+ * pinned `fallbackBite` at zero, reading as "the fallback is never used". */
function targetHeight(px, which) {
return (which === "fallback" ? px.fallback : 0) + px.padding;
}
- /* Half the gap between the two smallest interesting heights, so neither test can reach the
- * other's target. Sub-pixel layout means an exact equality test would miss. */
+ /* Half the gap between the two smallest interesting heights, so neither test can reach the other's
+ * target; sub-pixel layout means an exact equality test would miss. */
var PX_EPS = 2;
if (typeof window === "undefined" || !window.document) {
@@ -112,8 +71,8 @@
var doc = window.document;
var watched = [];
var watchedSet = typeof WeakSet === "function" ? new WeakSet() : null;
- /* element -> its record, so a sample can ask whether THIS root is currently skipped. Weak so
- * that a root the app has thrown away is not held alive by the probe. */
+ /* element -> its record, so a sample can ask whether THIS root is skipped. Weak so a discarded root
+ * is not held alive by the probe. */
var recOf = typeof WeakMap === "function" ? new WeakMap() : null;
var ev = { stateChange: 0, skip: 0, unskip: 0, watchers: 0, listenerErrors: 0 };
var seq = 0;
@@ -156,8 +115,8 @@
return a.bottom > b.top && a.top < b.bottom && a.right > b.left && a.left < b.right;
}
- /* Attached BEFORE anything is read, and exactly once per element. The event is the only
- * signal here that no amount of author CSS can fake. */
+ /* Attached BEFORE anything is read and exactly once per element: the one signal here that no author
+ * CSS can fake. */
function watch(el) {
try {
if (watchedSet) {
@@ -196,13 +155,10 @@
/* How many of this element's first `cap` element descendants generate a layout box.
*
- * A KNOWN FALSE NEGATIVE, KEPT ON PURPOSE. The reasoning is that a skipped subtree is not laid
- * out, so its descendants have no boxes. The reasoning is correct and the measurement is still
- * useless, because ASKING is what breaks it: `getClientRects()` on content inside a locked
- * subtree makes Chromium render that subtree in order to answer, so the probe unlocks exactly
- * what it came to observe. Measured on a 100K thread this returned 0 off-screen unrendered
- * roots while the event counter recorded 22 roots simultaneously in the skipped state. Read
- * alone it is a clean, confident, wrong "the arm did not fire". Use `ev_skip`. */
+ * A KNOWN FALSE NEGATIVE, KEPT ON PURPOSE. A skipped subtree is not laid out, so its descendants
+ * have no boxes -- but ASKING breaks it: `getClientRects()` inside a locked subtree makes Chromium
+ * render it to answer. On a 100K thread this returned 0 off-screen unrendered roots while the event
+ * counter recorded 22 roots simultaneously skipped. Use `ev_skip`. */
function descendantBoxes(el, cap) {
var kids;
try {
@@ -224,10 +180,8 @@
return n;
}
- /* Whether the browser last told us THIS root is skipping its contents.
- *
- * `null` means no transition has been seen, which is not the same as "rendered" and must not
- * be read as either. Callers that care about the difference check for `true` explicitly. */
+ /* Whether the browser last told us THIS root is skipping its contents. `null` means no transition
+ * has been seen, which is not "rendered"; callers that care check for `true` explicitly. */
function skippedState(el) {
var rec = null;
try {
@@ -309,20 +263,15 @@
var r = rect(el);
if (r) {
heights.push(Math.round(r.height));
- /* Two MUTUALLY EXCLUSIVE buckets, in priority order. `content-visibility: auto`
- * imposes size containment while skipping, so a skipped root's height is its
- * padding plus its intrinsic size. Landing on the declared <length> means no
- * remembered size exists yet; landing on the padding alone means one exists and
- * it is ZERO, recorded while the root was mounted and still empty. The fallback
- * is tested first and wins ties, because a root sitting on its fallback is
- * behaving exactly as the declaration asks and must never be charged to the
+ /* Two MUTUALLY EXCLUSIVE buckets, in priority order: a skipped root's height is its padding plus
+ * its intrinsic size, so landing on the declared <length> means no remembered size exists yet
+ * and landing on the padding alone means one exists and is ZERO. The fallback is tested first
+ * and wins ties, since such a root behaves exactly as declared and must not be charged to the
* trap. Anything else is counted as neither. */
- /* ONLY WHILE SKIPPED. `content-visibility: auto` computes to `auto` whether or
- * not the element is currently skipping, and size containment applies only while
- * it is. An armed root that is on screen has its ordinary rendered height, and if
- * that height happens to land within the tolerance of a role target it would be
- * charged to the remembered-size trap. Ordinary geometry must not be able to
- * masquerade as the finding. */
+ /* ONLY WHILE SKIPPED: `content-visibility: auto` computes to `auto` whether or not the element
+ * is skipping, and size containment applies only while it is. An on-screen armed root has its
+ * ordinary rendered height, which must not be able to land in a role target and masquerade as
+ * the finding. */
var px = ROLE_PX[roleOf(el)];
if (cv === "auto" && px && skippedState(el) === true) {
if (Math.abs(r.height - targetHeight(px, "fallback")) <= PX_EPS) {
@@ -332,8 +281,8 @@
}
}
}
- /* The geometry question is asked only of armed roots, and only of as many as can be
- * asked cheaply: this forces layout, and the probe must not become the load. */
+ /* Asked only of armed roots, and only of as many as can be asked cheaply: this forces layout, and
+ * the probe must not become the load. */
if (cv === "auto" && r && vpRect && scanned < MAX_ROOTS_SCANNED) {
scanned += 1;
var hasOwnBox = r.width > 0 || r.height > 0;
@@ -366,13 +315,10 @@
}
}
- /* CONNECTED roots only, and the detached ones are dropped as we go.
- *
- * `thread_reopen` is in the film: it tears the thread down and rebuilds it, so the old
- * roots leave the document. A detached root receives no further transitions, so one whose
- * last event said `skipped` would go on being counted in `skippedNow` for the rest of the
- * session, reporting roots as skipped that are not in the page at all. Pruning here also
- * stops `watched` growing a strong reference per root for the life of a long run. */
+ /* CONNECTED roots only, dropping the detached ones as we go. `thread_reopen` rebuilds the thread,
+ * so old roots leave the document and receive no further transitions; one whose last event said
+ * `skipped` would be counted in `skippedNow` for the rest of the session. Pruning also stops
+ * `watched` growing a strong reference per root. */
var live = [];
for (i = 0; i < watched.length; i++) {
var wel = watched[i].el;
@@ -430,27 +376,20 @@
window.__cvPotSample = sample;
- /* THE LISTENER HAS TO EXIST BEFORE THE ELEMENT'S FIRST TRANSITION, and polling cannot
- * guarantee that.
- *
- * `contentvisibilityautostatechange` fires on a CHANGE of state. A root that is inserted
- * off screen becomes skipped once, at its first lifecycle update, and then never changes
- * again while the user stays where they are. If the listener is attached on a two-second
- * tick, every root mounted and skipped inside that first tick emits its only event into a
- * void, `ev_skip` stays at zero, and the probe reports precisely the false NOT RUN it was
- * written to prevent. A whole thread can mount inside two seconds; a seeded one does.
+ /* THE LISTENER HAS TO EXIST BEFORE THE ELEMENT'S FIRST TRANSITION, and polling cannot guarantee
+ * that. `contentvisibilityautostatechange` fires on a CHANGE: a root inserted off screen becomes
+ * skipped once and never changes again while the user stays put, so on a two-second tick every
+ * root mounted inside the first tick emits its only event into a void and `ev_skip` stays zero --
+ * the false NOT RUN this file exists to prevent. A whole thread can mount inside two seconds.
*
- * So roots are adopted at INSERTION, from a MutationObserver installed before the app boots.
- * The observer callback runs in a microtask after the mutation and before the next lifecycle
- * update, which is early enough. The interval is kept, because a root can also acquire the
- * property later without being re-inserted, and `adoptAll` is idempotent.
+ * So roots are adopted at INSERTION, from a MutationObserver installed before the app boots; its
+ * callback runs in a microtask before the next lifecycle update. The interval is kept because a
+ * root can acquire the property later without being re-inserted, and `adoptAll` is idempotent.
*
- * OBSERVE THE DOCUMENT, NOT `documentElement`. An init script is evaluated after the Document
- * exists but before the page's own scripts run, and at that instant the parser may not have
- * created the root element yet. `observe(null, ...)` throws, the catch below would swallow it,
- * and adoption would silently fall back to the two-second tick: the exact false zero this
- * whole arrangement exists to prevent, reintroduced by the fix for it. A Document node is a
- * valid target and it always exists here. */
+ * OBSERVE THE DOCUMENT, NOT `documentElement`: an init script runs after the Document exists but
+ * possibly before the parser creates the root element, and `observe(null, ...)` would throw into
+ * the catch below, silently falling back to the tick -- the same false zero, reintroduced by the
+ * fix for it. */
function adoptAll() {
var roots = all(MESSAGE_SELECTOR);
for (var i = 0; i < roots.length; i++) {
@@ -458,10 +397,9 @@
}
}
- /* Only the ADDED NODES, never a document-wide re-scan. Streaming produces thousands of
- * mutations a second, and a `querySelectorAll` per mutation would make this probe the load
- * it is trying to observe. `watch` is idempotent, so a node reached twice costs a WeakSet
- * lookup. */
+ /* Only the ADDED NODES, never a document-wide re-scan: streaming produces thousands of mutations a
+ * second and a `querySelectorAll` per mutation would make this probe the load. `watch` is
+ * idempotent, so a node reached twice costs a WeakSet lookup. */
function adoptAdded(records) {
for (var i = 0; i < records.length; i++) {
var added = records[i].addedNodes;
diff --git a/tests/studio/studiobench/arms/dose.py b/tests/studio/studiobench/arms/dose.py
index 892a3edf128..12435d25bc7 100644
--- a/tests/studio/studiobench/arms/dose.py
+++ b/tests/studio/studiobench/arms/dose.py
@@ -40,12 +40,12 @@ from typing import Any, Mapping, Sequence
from ..scoring.schema import Measure
-#: The declared doses. Four points spanning three orders of magnitude, because a two-point
-#: "line" cannot be distinguished from a step and a three-point one cannot show curvature.
+#: The declared doses. Four points spanning three orders of magnitude, because a two-point "line"
+#: cannot be distinguished from a step and a three-point one cannot show curvature.
DOSES: tuple[int, ...] = (4, 40, 400, 4000)
#: A fit this far from the through-origin model is called nonlinear rather than reported as a
-#: slope. Loose, because the question is "is this O(children)", not "what is the exact constant".
+#: slope. Loose, because the question is "is this O(children)", not the exact constant.
MIN_R2_FOR_LINE = 0.90
#: An intercept larger than this fraction of the largest-dose cost means most of the cost is not
@@ -128,8 +128,8 @@ def _r2(xs: Sequence[float], ys: Sequence[float], predict) -> float | None:
ss_tot = sum((y - mean) ** 2 for y in ys)
ss_res = sum((y - predict(x)) ** 2 for x, y in zip(xs, ys))
if ss_tot <= 0:
- # Every reading identical. R2 is undefined, and reporting 1.0 (a perfect fit) for a flat
- # line is exactly the wrong way round: this is the case with the least information.
+ # Every reading identical. R2 is undefined, and reporting 1.0 for a flat line is exactly the wrong
+ # way round: this is the case with the least information.
return None
return 1.0 - (ss_res / ss_tot)
@@ -213,10 +213,10 @@ def fit_dose_response(
largest_cost = max(ys)
floor = detection_floor_ms
- # Detectability is judged on the FREE slope, not the through-origin one. A perfectly flat
- # series still has a large through-origin slope, because forcing the line through zero makes
- # the constant offset look like a per-child cost; judging detectability on that number turns
- # every null result into a positive identification.
+ # Detectability is judged on the FREE slope, not the through-origin one: a perfectly flat series
+ # still has a large through-origin slope, because forcing the line through zero makes the
+ # constant offset look like a per-child cost, which turns every null result into a positive
+ # identification.
detectable = (
slope_free is not None
and floor is not None
@@ -246,10 +246,10 @@ def fit_dose_response(
and largest_cost > 0
and abs(intercept_free) / largest_cost > MAX_INTERCEPT_FRACTION
):
- # Checked BEFORE the through-origin R2, because a straight line with a big offset fits a
- # free line perfectly and a through-origin line badly. Reporting that as NONLINEAR would
- # be exactly backwards: the data is as linear as data gets, it just does not pass through
- # zero, and the offset is the finding.
+ # Checked BEFORE the through-origin R2, because a straight line with a big offset fits a free line
+ # perfectly and a through-origin line badly. Reporting that as NONLINEAR would be backwards: the
+ # data is as linear as data gets, it just does not pass through zero, and the offset is the
+ # finding.
verdict = "MOSTLY FIXED COST"
note = (
f"the free fit puts {abs(intercept_free) / largest_cost:.0%} of the largest-dose cost "
diff --git a/tests/studio/studiobench/arms/knobs.js b/tests/studio/studiobench/arms/knobs.js
index 6d4297dbc35..3b8d8b93ed5 100644
--- a/tests/studio/studiobench/arms/knobs.js
+++ b/tests/studio/studiobench/arms/knobs.js
@@ -4,181 +4,80 @@
/*
* knobs.js -- runtime ablation knobs for the SHIPPED Unsloth Studio build.
*
- * Injected with Playwright's context.add_init_script BEFORE the app boots, so it can patch
- * browser APIs the app is about to use. It needs no compiler, no source edit and no rebuild: the
- * point is that an external tester can run every ablation against the same production bundle the
- * user gets, and that the ablated run and the control run are the same bytes of application code.
+ * Injected with Playwright's context.add_init_script BEFORE the app boots, so an external tester
+ * can run every ablation against the same production bundle the user gets, with the ablated and
+ * control runs sharing the same bytes of application code.
*
- * WHAT AN ARM IS FOR
+ * Each arm removes ONE candidate mechanism and nothing else, so a cheaper thread names the fix:
*
- * Each arm removes one candidate mechanism and nothing else. If removing it makes the thread
- * cheaper, that mechanism was the cost, and the arm names the fix. The arms:
- *
- * A visibility:hidden on completed messages.
- * Removes paint and raster for the prefix. Keeps layout, keeps the DOM, keeps React.
- * IF A IS THE WIN: the cost is paint/raster of off-screen content. The fix is to stop
- * painting the prefix, not to stop building it.
- *
- * B content-visibility:auto plus contain-intrinsic-size on completed messages, and the undo
- * of the index.css override that force-disables content-visibility on code blocks.
- * Removes style, layout AND paint for anything off screen.
- * IF B IS THE WIN: the cost is off-screen style plus layout plus paint, and the rule at
- * studio/frontend/src/index.css:2537 is wrong -- it was kept for a flicker, and the comment
- * above it asserts containment on message roots "was measured as no help", which is exactly
- * the claim this arm re-tests. B beating A says the expensive part is not the pixels.
- *
- * C display:none on completed messages.
- * Removes layout geometry and the sibling count as well as everything B removes.
- * IF C IS THE WIN AND B IS NOT: the cost is layout geometry and the cost of having N
- * siblings in one flow, not per-element style. That points at the thread container's layout
- * mode (flex column over N children) rather than at the messages.
- *
- * D detach the autoscroll subtree MutationObserver.
- * use-intent-aware-autoscroll.tsx:502 observes the viewport with subtree+characterData, and
- * its callback reads el.scrollHeight, which forces synchronous style+layout of the WHOLE
- * thread on every delivery. During streaming those deliveries arrive per streamed character.
- * IF D IS THE WIN: the cost is forced synchronous layout inside the autoscroll observer.
- * The fix is to stop reading scrollHeight per mutation (rAF-coalesce it, or switch to
- * IntersectionObserver / overflow-anchor), not to touch the messages at all.
- *
- * E neutralise the --aui-scroll-stabilizer custom property write.
- * That same callback writes a custom property on the viewport element. A custom property is
- * inherited, so writing it on the scroll container invalidates inherited style for the whole
- * subtree beneath it, which is every message.
- * IF E IS THE WIN: the cost is inherited custom-property subtree style invalidation. The fix
- * is to register the property with syntax and inherits:false via CSS.registerProperty, or to
- * move the write off the ancestor of the thread.
- *
- * F freeze React's scheduler while leaving the DOM exactly as it is.
- * React's browser scheduler drives work through a MessageChannel: port1.onmessage is
- * performWorkUntilDeadline, and port2.postMessage schedules the next slice. Suppressing the
- * delivery stops reconciliation, effects and store subscriptions without touching a node.
- * IF F IS THE WIN: the cost is React subscriptions and reconciliation over the prefix, and
- * the fix is memoisation / virtualisation / narrower context subscriptions, not CSS.
- *
- * G CONTROL. Identical DOM, no knob, thread scrolled so prior turns are actually IN the
- * viewport. Every other arm removes work that only exists when the prefix is off screen and
- * cheap; G is the run where the prefix is on screen and expensive. It is the calibration for
- * "how much does this page cost when nothing is being skipped", and a G that is not slower
- * than the untouched baseline means the baseline was never skipping anything, which would
+ * A visibility:hidden on completed messages -- removes paint/raster only. If A wins, stop
+ * painting the prefix rather than building it.
+ * B content-visibility:auto + contain-intrinsic-size, undoing the index.css:2537 override --
+ * removes off-screen style, layout AND paint. B beating A says the pixels are not the cost,
+ * and re-tests that override's claim that containment on message roots "was no help".
+ * C display:none -- also removes layout geometry and the sibling count. C winning where B does
+ * not points at the thread container's layout mode, not at per-element style.
+ * D detach the autoscroll subtree MutationObserver (use-intent-aware-autoscroll.tsx:502), whose
+ * callback reads el.scrollHeight and so forces synchronous layout of the whole thread per
+ * streamed character. If D wins, rAF-coalesce that read rather than touching the messages.
+ * Observed with subtree+characterData, so it fires per streamed character.
+ * E neutralise the --aui-scroll-stabilizer write: a custom property is inherited, so writing it
+ * on the scroll container invalidates inherited style for every message. If E wins, register
+ * it with inherits:false or move the write off the thread's ancestor.
+ * If E wins, `CSS.registerProperty` with inherits:false is the fix.
+ * F freeze React's scheduler (MessageChannel port1.onmessage = performWorkUntilDeadline),
+ * leaving the DOM untouched. If F wins the fix is memoisation / virtualisation, not CSS.
+ * G CONTROL: identical DOM, no knob, prior turns actually IN the viewport. A G that is not
+ * slower than the baseline means the baseline was never skipping anything, which would
* invalidate the whole comparison.
*
- * THE TWO FAILURE MODES THIS FILE EXISTS TO PREVENT
- *
- * See arms/manifest.py. An ablation lies in exactly two ways and both are silent.
- *
- * 1. THE ARM CHANGED THE OUTPUT. The knob also removed some of the work, so the two sides did
- * not render the same page. The dangerous case is the small invisible difference: an earlier
- * stub fixture in this codebase produced 552 syntax-highlighted spans where the real page
- * produces 2,561, and it read as a clean 4x win. It was not fast, it was rendering a fifth
- * of the content. digest() and counts() exist for that: the canonical serialisation carries
- * the code-block count and the span count per message, so a highlighting difference is
- * caught even when every character of text is identical.
- *
- * TWO THINGS THE DIGEST CANNOT SEE, stated here because they cannot be stated in it. It
- * walks `[data-message-id]` roots and serialises their attributes and text plus the count of
- * code blocks and of spans inside them. It does NOT serialise the attributes of descendants,
- * so an inline style written on a code block is invisible to it (arm B does exactly that,
- * which is one reason B is EQUIVALENT and not EXACT). And it does not know about geometry,
- * so a change that only moves pixels is invisible too. Neither gap is patched by making the
- * digest bigger; they are declared, because a normaliser nobody wrote down is just a bug.
- *
- * Arms A and C therefore go out of their way to mutate NOTHING: they target the completed
- * prefix with `:nth-child(-n+K)` against the viewport's direct children rather than by
- * tagging elements, so the raw digest is byte-identical across the arm and EXACT is a claim
- * they can actually make. When that is impossible they fall back to a marker attribute and
- * say in the returned reason that they have dropped to EQUIVALENT.
- *
- * 2. THE ARM DID NOT FIRE. Selector matched nothing, patch failed to install, run completed,
- * difference was zero, and "no effect" got written down as evidence against the mechanism.
- * It is evidence of nothing. So every arm has a potency counter that the arm CAUSES: not
- * "the stylesheet was appended" but "N elements actually compute to visibility:hidden".
- * A patch that could not be installed lands in unavailable[] and never in available[], so
- * UNAVAILABLE can never be misread as "had no effect".
- *
- * CASCADE FACTS THIS FILE DEPENDS ON, VERIFIED AGAINST THE SPEC
- *
- * The index.css override lives inside `@layer utilities { ... }` (Tailwind v4 emits native
- * cascade layers). Per CSS Cascading and Inheritance Level 5 and MDN:
- *
- * - For IMPORTANT declarations the layer order is REVERSED, and important declarations in ANY
- * layer beat important declarations outside every layer. So appending an unlayered
- * `!important` rule, at any specificity, LOSES to the layered `!important` rule it is trying
- * to undo. The obvious approach silently does nothing, which is failure mode 2.
- * - Element-attached (inline) declarations are sorted BEFORE layers in the cascade, so an
- * inline `!important` declaration beats every other author declaration whatever its layer.
- *
- * Therefore each CSS arm applies its stylesheet in two copies, one unlayered and one inside
- * `@layer utilities` so it competes in the same layer as the rule it must beat (same layer, same
- * importance, higher specificity, later order -> wins), THEN verifies with getComputedStyle, and
- * only if the verification says the rule did not take effect does it escalate to inline
- * !important, counting the escalation. The verification is the whole point: the stylesheet is a
- * hypothesis about the cascade, the computed style is the observation.
- *
- * ARMS ARE APPLIED ONCE, OVER THE PREFIX THAT EXISTS AT THAT MOMENT
- *
- * apply() runs at the start of a measured window and never again. Messages created during the
- * window are NOT ablated. That is deliberate and not a limitation:
- *
- * - the hypothesis is about the PREFIX (the N completed turns that are off screen), not about
- * the turn being streamed, so ablating the in-flight message would change the thing under
- * measurement rather than the thing under test;
- * - a live MutationObserver re-applying the knob to each new message would add per-mutation
- * work to the hot path, which is the exact category of cost the experiment is trying to
- * remove. An ablation that installs its own observer to remove an observer measures itself.
- *
- * For the same reason arm B marks only code blocks inside COMPLETED messages: touching the code
- * block that is still streaming would change how streamdown and shiki finalise it.
- *
- * PREBOOT VERSUS RUNTIME
- *
- * D, E and F patch APIs the app captures during boot, so they must be installed before the app
- * runs and they are read from window.__sbArmConfig.preboot, set by an earlier init script. An arm
- * that is not listed is NOT INSTALLED AT ALL. Installing a patch and leaving it inactive is not
- * free: the wrapper costs a call frame on every MutationObserver.observe or setProperty in the
- * process, and a control run carrying that overhead is not a control. So the preboot list decides
- * per page load, and a run whose config did not arm D reports D as unavailable with a reason,
- * never as an arm that ran and did nothing.
- *
- * D and E are ACTIVE FROM INJECTION when armed; there is no meaningful apply() for them, because
- * the autoscroll observer is installed once during mount and the suppression has to be in place
- * before that. apply("D") therefore reports the arm as already active and returns the suppression
- * count so far. F is different: the port capture is preboot, the freeze is toggled by apply("F").
- *
- * ARM F IS DOM_CHANGING. Freezing React stops the stream from rendering, so the frozen run is not
- * rendering the same page as the control -- it is rendering less of it. Its cost is therefore an
- * upper bound on what React reconciliation costs and can never be quoted as a point estimate.
- * manifest.py prints it as `<= x`. This is stated here as well because the arm is the one most
- * likely to produce a spectacular number that somebody wants to quote.
- *
- * COUNTERS: EVENT COUNTERS VERSUS GAUGES
+ * THE TWO FAILURE MODES THIS FILE EXISTS TO PREVENT (see arms/manifest.py); both are silent.
*
- * potency() returns integers only, in two kinds, and the distinction is load-bearing:
+ * 1. THE ARM CHANGED THE OUTPUT. An earlier stub fixture rendered 552 highlighted spans where
+ * the real page renders 2,561 and read as a clean 4x win. digest() and counts() catch that:
+ * the canonical form carries code-block and span counts per message, so a highlighting
+ * difference is caught even when every character of text matches. Two declared gaps: the
+ * digest does not serialise descendants' attributes (so arm B's inline style on a code block
+ * is invisible, which is why B is EQUIVALENT and not EXACT) and it knows nothing about
+ * geometry. Arms A and C therefore mutate NOTHING, targeting the prefix with
+ * `:nth-child(-n+K)` so EXACT is a claim they can make; when that is impossible they fall
+ * back to a marker attribute and say in the returned reason that they dropped to EQUIVALENT.
+ * 2. THE ARM DID NOT FIRE, and "no effect" got written down as evidence. So every arm has a
+ * potency counter the arm CAUSES ("N elements compute to visibility:hidden", not "the
+ * stylesheet was appended"), and a patch that could not be installed lands in unavailable[]
+ * so UNAVAILABLE can never be misread as "had no effect".
*
- * EVENT COUNTERS are monotonic and never decrease for the life of the page:
- * suppressedViewportObserves, observeCallsTotal, observeCallsPassedThrough,
- * suppressedStabilizerSets, customPropSets, setPropertyCallsTotal,
- * suppressedSchedulerCallbacks, schedulerCallbacksDelivered, capturedSchedulerPorts, and the
- * bookkeeping counters (armsApplied, inlineFallbacks, ...).
+ * CASCADE FACTS, verified against CSS Cascading and Inheritance Level 5. The index.css override
+ * sits inside `@layer utilities`, and for IMPORTANT declarations layer order is REVERSED, so an
+ * unlayered `!important` rule LOSES at any specificity -- the obvious approach silently does
+ * nothing, which is failure mode 2. Inline declarations sort BEFORE layers, so an inline
+ * `!important` beats every author declaration. Each CSS arm therefore emits its sheet twice (once
+ * unlayered, once inside `@layer utilities`), verifies with getComputedStyle, and only escalates
+ * to inline !important when the verification says the rule did not take, counting the escalation.
*
- * GAUGES are point-in-time measurements of the live DOM, re-measured on every potency() call
- * unless {live:false} is passed: visibilityHiddenConfirmed, cvAutoMessages, cvAutoCodeBlocks,
- * contentVisibilityAutoConfirmed, displayNoneConfirmed, controlVisibleMessages,
- * controlVisiblePriorMessages.
+ * ARMS ARE APPLIED ONCE, over the prefix that exists at that moment. Messages created during the
+ * window are NOT ablated, deliberately: the hypothesis is about the completed off-screen prefix,
+ * and a live MutationObserver re-applying the knob would add per-mutation work to the hot path --
+ * an ablation that installs an observer to remove an observer measures itself. For the same reason
+ * arm B marks only code blocks inside COMPLETED messages.
*
- * Gauges are deliberately not monotonic. A stored 0 and a measured 0 are different facts, and
- * this file exists to keep them different: measuring live means the "before" read is a real
- * observation of the untouched page rather than a variable that has not been written yet, and it
- * means arm G is checked against what the page looks like at the END of the window, after the
- * app's own autoscroll has had every chance to undo the scroll position G set. Re-measuring calls
- * getComputedStyle over every message root, which forces style recalc, so potency() must be
- * called OUTSIDE the timed window; pass {live:false} to read the last stored values with no DOM
- * access at all.
+ * PREBOOT VERSUS RUNTIME. D, E and F patch APIs the app captures during boot and are read from
+ * window.__sbArmConfig.preboot. An arm not listed is NOT INSTALLED AT ALL, because an inactive
+ * wrapper still costs a call frame on every observe or setProperty and a control run carrying that
+ * overhead is not a control. D and E are active from injection (apply("D") reports the arm already
+ * active); F captures the port at preboot and toggles the freeze in apply("F").
*
- * HASHING
+ * COUNTERS. potency() returns integers in two kinds. EVENT COUNTERS are monotonic for the life of
+ * the page (suppressedViewportObserves, observeCallsTotal, suppressedStabilizerSets,
+ * suppressedSchedulerCallbacks, capturedSchedulerPorts, the bookkeeping counters, ...). GAUGES are
+ * point-in-time reads of the live DOM, re-measured on every call unless {live:false}
+ * (visibilityHiddenConfirmed, cvAutoMessages, displayNoneConfirmed, controlVisibleMessages, ...).
+ * A stored 0 and a measured 0 are different facts: measuring live keeps the "before" read a real
+ * observation, and checks arm G at the END of the window after the app's autoscroll has had every
+ * chance to undo the scroll position. Re-measuring calls getComputedStyle over every message root,
+ * forcing style recalc, so potency() must be called OUTSIDE the timed window.
*
- * FNV-1a, 32 bit, hex. NOT CRYPTOGRAPHIC and not a security boundary. It is a change detector
- * against an adversary that does not exist, and 32 bits is small enough that a collision on a
+ * HASHING: FNV-1a, 32 bit, hex. Not cryptographic, and small enough that a collision on a
* multi-megabyte canonical form is not impossible, so digest() also returns canonicalLength and
* rawLength. Compare the pair, not the hash alone.
*/
@@ -192,9 +91,8 @@
var W = window;
- // Idempotence. add_init_script runs for every document including iframes, and a driver that
- // reloads the page must not stack patches: a doubly wrapped observe() would double every
- // count, and a doubly wrapped MessageChannel would make the freeze flag ambiguous.
+ // Idempotence: add_init_script runs for every document including iframes, and double
+ // wrapping would double every count and make the freeze flag ambiguous.
if (W.__sbArmsInstalled) {
return;
}
@@ -221,12 +119,8 @@
var DEFAULT_CONTROL_VISIBLE_TARGET = 3;
var MAX_STYLESHEET_DEPTH = 8;
- // ------------------------------------------------------------------------------------
- // Config
- //
- // Read defensively. A missing or malformed __sbArmConfig means no preboot arms, which is the
- // correct default: no config is not permission to install patches into a control run.
- // ------------------------------------------------------------------------------------
+ // A missing or malformed __sbArmConfig means no preboot arms: absent config is not
+ // permission to patch a control run.
var cfg = {};
try {
@@ -302,21 +196,15 @@
doc = null;
}
- // ------------------------------------------------------------------------------------
- // State
- // ------------------------------------------------------------------------------------
function freshEventCounters() {
return {
- // D
observeCallsTotal: 0,
observeCallsPassedThrough: 0,
suppressedViewportObserves: 0,
- // E
setPropertyCallsTotal: 0,
customPropSets: 0,
suppressedStabilizerSets: 0,
- // F
capturedSchedulerPorts: 0,
schedulerCallbacksDelivered: 0,
suppressedSchedulerCallbacks: 0,
@@ -326,7 +214,6 @@
droppedSchedulerEvents: 0,
schedulerListenerAdds: 0,
schedulerCallbacksWithoutHandler: 0,
- // bookkeeping
armsApplied: 0,
armsReverted: 0,
styleSheetsInstalled: 0,
@@ -353,13 +240,13 @@
var gauges = freshGauges();
var state = {
- applied: {}, // armId -> {at, affected, reason}
+ applied: {},
frozen: false,
- markers: [], // {el, prev} for MARKER_ATTR restoration
- inline: [], // {el, prop, prevValue, prevPriority, hadStyleAttr}
- sheets: {}, // armId -> <style> element
- channels: [], // captured MessageChannel records
- scrollRestore: null, // arm G restoration record
+ markers: [],
+ inline: [],
+ sheets: {},
+ channels: [],
+ scrollRestore: null,
originals: {
observe: null,
observeDescriptor: null,
@@ -402,13 +289,8 @@
return { reverted: !!reverted, reason: String(reason || "") };
}
- // ------------------------------------------------------------------------------------
- // DOM helpers
- //
- // All of them tolerate a missing document, a detached node and a selector that matches
- // nothing, and none of them throw. A query that fails returns an empty array, and the caller
- // turns that into a reason string rather than into a zero.
- // ------------------------------------------------------------------------------------
+ // None of these throw on a missing document, detached node or empty match; a failed
+ // query returns [] and the caller turns that into a reason, not a zero.
function toArray(nodeList) {
var out = [];
@@ -454,9 +336,8 @@
}
function isRunning(el) {
- // The ONLY DOM signal of an in-flight message is a descendant (or the root itself)
- // carrying [data-status="running"]; status.type from assistant-ui is one of
- // running|complete|incomplete|requires-action.
+ // The only DOM signal of an in-flight message is [data-status="running"] on the root or a
+ // descendant (assistant-ui status: running|complete|incomplete|requires-action).
try {
if (!el) {
return false;
@@ -466,18 +347,16 @@
}
return queryOne(RUNNING_SELECTOR, el) !== null;
} catch (e) {
- // Unknown status is treated as running, i.e. as NOT ablatable. Erring the other way
- // would ablate the message being streamed and contaminate the measurement.
+ // Unknown status counts as running, i.e. not ablatable: the other way would ablate the
+ // streaming message and contaminate the measurement.
return true;
}
}
- // The completed-message filter is done in JS, one querySelector per root, and NOT with a CSS
- // `:has()` selector. `:has()` is unsupported on older engines and an unsupported compound
- // selector makes the WHOLE selector list invalid, so the rule would be dropped at parse time,
- // match nothing, and the arm would report a clean zero difference. That is precisely the
- // did-not-fire failure this file is built to make impossible, and it is invisible in a
- // screenshot, in a trace and in the console.
+ // Filtering in JS rather than CSS `:has()`: an unsupported compound selector invalidates the
+ // whole selector list, so the rule would be dropped at parse time and the arm would report a
+ // clean zero difference.
+ // One querySelector per root.
function completedMessageRoots() {
var roots = messageRoots();
var out = [];
@@ -567,22 +446,14 @@
} catch (e) {
/* fall through */
}
- // No CSS.supports means we cannot prove support. Treat as supported and let the computed
- // check decide: a false "unavailable" would hide an arm that works.
+ // No CSS.supports means support cannot be proven; assume supported and let the computed
+ // check decide, since a false UNAVAILABLE hides an arm that works.
return true;
}
- // ------------------------------------------------------------------------------------
- // Markers and inline styles
- //
- // Arms A, B and C need a CSS hook on specific elements, and the only way to have a stylesheet
- // address "these elements and no others" is an attribute the stylesheet can select. One
- // attribute name is used for all arms, space separated, so [data-sb-arm~="A"] matches. That
- // attribute IS a change to the DOM, so it shows up in digest().raw. It is meant to: an arm
- // that mutates attributes is EQUIVALENT, not EXACT, and the Python side declares
- // `attr:data-sb-arm` (plus `attr:style` when the inline escalation fires) as its allowed diff.
- // Pretending otherwise by hiding the marker from the raw digest would defeat the check.
- // ------------------------------------------------------------------------------------
+ // One space-separated attribute serves arms A, B and C ([data-sb-arm~="A"]). It IS a DOM
+ // change and appears in digest().raw deliberately: such an arm is EQUIVALENT, not EXACT,
+ // and Python declares `attr:data-sb-arm` (plus `attr:style`) as its allowed diff.
function markElement(el, armId) {
try {
@@ -693,8 +564,7 @@
left = "x";
}
if (left === "") {
- // Leaving style="" behind would be a permanent, invisible difference in
- // every future digest of this page.
+ // Leaving style="" behind would be a permanent, invisible difference in every future digest of this page.
rec.el.removeAttribute("style");
}
}
@@ -705,23 +575,10 @@
state.inline = [];
}
- // ------------------------------------------------------------------------------------
- // Stylesheets
- //
- // Every arm sheet is emitted TWICE: once unlayered, once inside `@layer utilities`.
- //
- // The rule that has to be beaten (index.css:2536-2538) is inside `@layer utilities`, and for
- // important declarations layered beats unlayered no matter the specificity. The layered copy
- // therefore competes in the same layer as its target, where the ordinary rules apply again:
- // same layer, same importance, higher specificity, later in document order, so it wins. The
- // unlayered copy covers the case where the build has no layers at all (then there is nothing
- // layered to lose to, and the unlayered copy is the one that applies). Emitting both is safe
- // because the two copies carry identical declarations, so whichever wins produces the same
- // computed value.
- //
- // Appending to <head> puts the sheet last among same-layer rules, which settles order of
- // appearance. None of this is trusted: applyX() checks getComputedStyle afterwards.
- // ------------------------------------------------------------------------------------
+ // Every arm sheet is emitted twice, unlayered and inside `@layer utilities`. The rule to beat
+ // (index.css:2536-2538) is layered, and for important declarations layered beats unlayered at
+ // any specificity; the unlayered copy covers a build with no layers. Nothing is trusted:
+ // applyX() rechecks getComputedStyle afterwards.
function ensureStyleSheet(armId, cssBody) {
try {
@@ -763,32 +620,13 @@
}
}
- // ------------------------------------------------------------------------------------
- // Two ways to address "the completed messages" from a stylesheet, and why the first one is
- // worth the extra code.
- //
- // PREFIX PATH (preferred). Messages are DIRECT CHILDREN of the viewport; there is no
- // message-list wrapper element. The message being streamed is the last one. So the completed
- // messages are a leading run of the viewport's children and
- //
- // .aui-thread-viewport.aui-stream-viewport > [data-message-id]:nth-child(-n+K)
- //
- // selects exactly them, with K computed in JS as the child index of the first running message
- // root. This mutates NOTHING. No attribute is added, no inline style is written, and the raw
- // digest of the thread is byte-identical before and after, so arms A and C can honestly claim
- // EXACT invariance. The `[data-message-id]` part is not redundant: nth-child alone would also
- // match spacers, sentinels and buttons that share the viewport, and hiding one of those is a
- // change to the rendered page that the digest cannot see because the digest only walks
- // message roots.
- //
- // MARKER PATH (fallback). If the completed set is not a leading prefix (a message somewhere
- // in the middle is still running, which assistant-ui allows in principle), or the viewport
- // cannot be found, the arm falls back to a `data-sb-arm` attribute on each completed root.
- // That is a real DOM mutation, so the arm drops from EXACT to EQUIVALENT and the returned
- // reason string says so in as many words. It is reported rather than hidden because a
- // silently-EQUIVALENT arm quoted as EXACT is a false number, while a loudly-EQUIVALENT one is
- // just a number with a declared diff.
- // ------------------------------------------------------------------------------------
+ // PREFIX PATH (preferred): messages are direct children of the viewport and the streamed one
+ // is last, so a `> [data-message-id]:nth-child(-n+K)` selector picks exactly the completed
+ // run. It mutates nothing, so the raw digest is byte-identical and arms A and C can claim
+ // EXACT. `[data-message-id]` is needed: nth-child alone also matches spacers and buttons.
+ // MARKER PATH (fallback): when the completed set is not a leading prefix, or no viewport is
+ // found, each completed root is marked with `data-sb-arm`; the arm drops to EQUIVALENT and
+ // says so in its reason.
function cssPrefixA(k) {
return (
@@ -833,9 +671,8 @@
"\tdisplay: none !important;\n" +
"}\n";
- // Returns the leading run of viewport children that contains no running message, or null when
- // the prefix path is not usable. `k` is a 1-based nth-child bound; `roots` are the message
- // roots inside it, all of them completed by construction.
+ // Returns the leading run of viewport children with no running message, else null. `k` is a
+ // 1-based nth-child bound; `roots` are the completed roots inside it.
function prefixInfo() {
try {
var vp = viewportEl();
@@ -895,9 +732,6 @@
}
}
- // ------------------------------------------------------------------------------------
- // Hashing and canonical serialisation
- // ------------------------------------------------------------------------------------
var imul =
typeof Math.imul === "function"
@@ -910,8 +744,8 @@
return ((al * bl + (((ah * bl + al * bh) << 16) >>> 0)) | 0);
};
- // FNV-1a, 32 bit, over UTF-16 code units taken low byte first. Non-cryptographic: this is a
- // change detector, not a security boundary. Compare it together with canonicalLength.
+ // FNV-1a 32-bit over UTF-16 code units, low byte first. Non-cryptographic: compare it
+ // together with canonicalLength.
function fnv1a32(str) {
var h = 0x811c9dc5;
for (var i = 0; i < str.length; i++) {
@@ -924,9 +758,8 @@
return ("0000000" + (h >>> 0).toString(16)).slice(-8);
}
- // Field encoding. JSON escapes quotes, backslashes, newlines and control characters; the pipe
- // is escaped on top of that so no encoded field can ever contain the field separator, which
- // makes the canonical form parseable by a plain split() in diffKeys().
+ // JSON escapes quotes, backslashes, newlines and controls; the pipe is escaped on top, so no
+ // field can contain the separator and diffKeys() can split() the canonical form.
function enc(s) {
try {
return JSON.stringify(String(s === null || s === undefined ? "" : s))
@@ -949,8 +782,8 @@
if (!skipStyleProps || !skipStyleProps.length) {
return value;
}
- // Parse with the browser's own CSS parser on a detached element rather than by splitting
- // on ";" and ":", which breaks on url(data:...) and on any value containing a semicolon.
+ // Parsed by the browser's CSS parser on a detached element: splitting on ";" and ":" breaks
+ // on url(data:...) and any value containing a semicolon.
try {
if (doc && typeof doc.createElement === "function") {
var scratch = doc.createElement("div");
@@ -986,9 +819,8 @@
return kept.join("; ");
}
- // One DOM walk, reused for both serialisations. Walking twice would be slower and, worse,
- // would let the page change between the raw pass and the normalised pass, producing a "raw
- // differs, normalised does not" that is an artefact of the instrument.
+ // One DOM walk feeds both serialisations: walking twice lets the page change between the raw
+ // and normalised passes, faking a raw-only difference.
function collectRecords() {
var roots = messageRoots();
var records = [];
@@ -1153,9 +985,8 @@
out.codeSpans = collected.codeSpans;
var rawKeys = {};
- // `raw` skips nothing at all: it is the answer to "did anything about the rendered
- // output change", including the arm's own marker attribute. An EXACT arm that touches
- // an attribute is not an EXACT arm, and this is where that gets caught.
+ // `raw` skips nothing, including the arm's own marker attribute: an EXACT arm that touches an
+ // attribute is not EXACT, and this is where that is caught.
var rawCanonical = serialise(collected, [], [], rawKeys);
var normKeys = {};
@@ -1177,9 +1008,8 @@
if (keepCanonical) {
if (rawCanonical.length > maxCanonicalChars || normCanonical.length > maxCanonicalChars) {
- // A prefix is retained because it is useful when debugging, but `truncated`
- // is set and diffKeys() refuses to answer from a truncated pair. A quiet cut
- // would turn "we did not look at the rest" into "the rest was identical".
+ // A prefix is kept for debugging but `truncated` is set and diffKeys() refuses a truncated
+ // pair: a quiet cut would turn "not looked at" into "identical".
out.truncated = true;
}
out.canonicalRaw = rawCanonical.slice(0, maxCanonicalChars);
@@ -1192,27 +1022,16 @@
return out;
}
- // ------------------------------------------------------------------------------------
- // diffKeys
- //
- // Compares the RAW canonical forms, not the normalised ones. The normalised hashes already
- // answer "are these equivalent"; what manifest.py needs from here is the other question,
- // "what actually differs", so it can check the observed diff against the diff the arm
- // DECLARED it would produce. An arm that declares one difference and produces two is voided,
- // and comparing normalised forms would hide the second one, since the normaliser is exactly
- // the thing that removes the declared difference.
- //
- // Keys are collapsed to a stable vocabulary (`attr:<name>`, `text`, `tag`,
- // `structure:<what>`) rather than per-message paths, because a declared diff has to be
- // writable in advance and `#msg-8fc2.attr.style` is not knowable in advance.
- //
- // Fails closed: if the canonical form was not retained, or was truncated, or cannot be
- // parsed, the returned array contains a key beginning with `__unavailable:` which can never
- // appear in a declared diff, so the arm is voided instead of silently passing.
- // ------------------------------------------------------------------------------------
+ // Compares the RAW canonical forms: the normalised hashes already answer "equivalent?", while
+ // manifest.py needs what actually differs to check it against the DECLARED diff. Normalised
+ // forms would hide a second difference, since the normaliser removes the declared one.
+ // Keys collapse to a stable vocabulary (`attr:<name>`, `text`, `tag`, `structure:<what>`): a
+ // declared diff must be writable in advance, and `#msg-8fc2.attr.style` is not.
+ // Fails closed: an unretained, truncated or unparseable canonical form yields a
+ // `__unavailable:` key, which can never appear in a declared diff, so the arm voids.
function parseCanonical(text) {
- var entries = {}; // entryId -> {key: value}
+ var entries = {};
var indexToId = {};
var lines = String(text).split("\n");
for (var i = 0; i < lines.length; i++) {
@@ -1337,7 +1156,7 @@
}
// Beyond the required API: the same comparison with examples, for a human reading a VOIDED
- // verdict who needs to know WHICH message drifted and by how much.
+ // verdict who needs to know which message drifted and by how much.
function diffDetail(a, b, limit) {
var cap = typeof limit === "number" && limit > 0 ? Math.floor(limit) : 20;
var out = [];
@@ -1397,9 +1216,6 @@
return out;
}
- // ------------------------------------------------------------------------------------
- // counts()
- // ------------------------------------------------------------------------------------
function counts() {
var out = {
@@ -1414,10 +1230,8 @@
chars: 0,
domNodes: 0,
visibleMessages: 0,
- // Beyond the required keys, and required by the zero discipline: visibleMessages == 0
- // because the thread viewport was not found is a different fact from
- // visibleMessages == 0 because nothing is on screen, and the two must not print the
- // same.
+ // Zero discipline: visibleMessages == 0 from a missing thread viewport is a different fact
+ // from nothing being on screen, and must not print the same.
viewportFound: false
};
try {
@@ -1462,10 +1276,8 @@
var spans = queryAll("span", blocks[b]);
out.codeSpans += spans.length;
for (var s = 0; s < spans.length; s++) {
- // A shiki token is a leaf span with text. Both numbers are reported: a
- // fixture that renders a fifth of the highlighting moves both, and a change
- // in span NESTING moves only codeSpans, so keeping them separate says which
- // of the two happened.
+ // A shiki token is a leaf span with text. Both numbers are reported: partial highlighting
+ // moves both, a change in span NESTING moves only codeSpans.
var sp = spans[s];
var hasElementChild = false;
try {
@@ -1500,9 +1312,6 @@
return out;
}
- // ------------------------------------------------------------------------------------
- // potency()
- // ------------------------------------------------------------------------------------
function measureGauges() {
try {
@@ -1565,9 +1374,6 @@
return out;
}
- // ------------------------------------------------------------------------------------
- // Arm A -- visibility:hidden on completed messages
- // ------------------------------------------------------------------------------------
function applyA() {
var info = prefixInfo();
@@ -1602,8 +1408,8 @@
}
var confirmed = countComputed(roots, "visibility", "hidden");
if (confirmed === 0) {
- // The stylesheet lost the cascade. Inline important is sorted before layers and beats
- // every author rule, so this is the escalation that cannot lose.
+ // The stylesheet lost the cascade. Inline important sorts before layers and beats every
+ // author rule, so this escalation cannot lose.
counters.inlineFallbacks += 1;
for (var i = 0; i < roots.length; i++) {
setInline(roots[i], "visibility", "hidden", "important");
@@ -1630,22 +1436,14 @@
);
}
- // ------------------------------------------------------------------------------------
- // Arm B -- content-visibility:auto plus contain-intrinsic-size
- //
- // Part (i) restores content-visibility on code blocks, undoing index.css:2536-2538.
- // Part (ii) puts content-visibility:auto and contain-intrinsic-size:auto <measured>px inline
- // on each completed message root.
- //
- // THE ORDERING BUG THIS FUNCTION IS SHAPED TO AVOID: content-visibility:auto makes an
- // off-screen element skip its contents, so the moment it is set, offsetHeight collapses to the
- // contain-intrinsic-size placeholder. Reading offsetHeight after setting the property captures
- // the placeholder, which is then fed back as the intrinsic size, and every element ends up
- // claiming to be 200px (or 0) tall. The thread's scroll height collapses, the scrollbar jumps,
- // the autoscroll observer fires on the reflow, and the run measures a page that is not the
- // page. So the heights are all read in one pass FIRST and only then written in a second pass.
- // Two passes also avoid read/write layout thrash, but that is a bonus, not the reason.
- // ------------------------------------------------------------------------------------
+ // (i) restores content-visibility on code blocks, undoing index.css:2536-2538; (ii) puts
+ // content-visibility:auto and contain-intrinsic-size inline on each completed message root.
+ // THE ORDERING BUG THIS FUNCTION AVOIDS: content-visibility:auto makes an off-screen element
+ // skip its contents, so an offsetHeight read AFTER setting it returns the
+ // contain-intrinsic-size placeholder, which then feeds back as the intrinsic size: scroll
+ // height collapses, the scrollbar jumps, the autoscroll observer fires and the run measures a
+ // page that is not the page. So heights are read in one pass and written in a second.
+ // The placeholder claims 200px, or 0.
function readHeights(els) {
var heights = [];
@@ -1672,9 +1470,8 @@
);
}
- // Only code blocks inside COMPLETED messages. The block that is still streaming must not
- // be touched: changing containment on it would change how streamdown and shiki finalise
- // it, which is a change to the output, not an ablation of cost.
+ // Only code blocks in COMPLETED messages: changing containment on the streaming block would
+ // change how streamdown and shiki finalise it, altering the output rather than its cost.
var blocks = [];
for (var r = 0; r < roots.length; r++) {
var bs = codeBlocksIn(roots[r]);
@@ -1684,13 +1481,15 @@
}
// READ PASS. Every height is captured before any property is written.
+ // THE ORDERING BUG THIS AVOIDS: content-visibility:auto makes an off-screen element skip its
+ // contents, so offsetHeight collapses to the contain-intrinsic-size placeholder the moment it
+ // is set. Read afterwards, that placeholder feeds back as the intrinsic size.
var blockHeights = readHeights(blocks);
var rootHeights = readHeights(roots);
- // WRITE PASS. B writes inline styles on the roots either way, so it is EQUIVALENT
- // whatever happens here; the prefix selector is still preferred because it keeps the
- // code blocks free of a marker attribute that the digest cannot see (the canonical form
- // serialises message-root attributes and DESCENDANT COUNTS, not descendant attributes).
+ // B writes inline styles either way, so it is EQUIVALENT regardless; the prefix selector is
+ // still preferred because it keeps a marker attribute off the code blocks, which the canonical
+ // form cannot see (it serialises root attributes and descendant COUNTS).
if (info) {
ensureStyleSheet("B", cssPrefixB(info.k));
} else {
@@ -1699,8 +1498,7 @@
}
var i;
for (i = 0; i < blocks.length; i++) {
- // contain-intrinsic-size has to be inline anyway because it is per element, and the
- // stylesheet only carries content-visibility.
+ // contain-intrinsic-size has to be inline because it is per element; the stylesheet only carries content-visibility.
setInline(
blocks[i],
"contain-intrinsic-size",
@@ -1711,10 +1509,9 @@
var cvBlocks = countComputed(blocks, "content-visibility", "auto");
if (blocks.length > 0 && cvBlocks === 0) {
- // Expected on this build: index.css:2537 sets content-visibility:visible !important
- // from inside @layer utilities, and for important declarations a layered rule beats
- // an unlayered one at any specificity. The layered copy of CSS_B should win; if it
- // did not, inline important is the escalation that the cascade guarantees.
+ // Expected here: index.css:2537 sets content-visibility:visible !important inside @layer
+ // utilities, and a layered important rule beats an unlayered one at any specificity, so the
+ // layered copy of CSS_B should win; if not, inline important is the guaranteed escalation.
counters.inlineFallbacks += 1;
for (i = 0; i < blocks.length; i++) {
setInline(blocks[i], "content-visibility", "auto", "important");
@@ -1754,9 +1551,6 @@
);
}
- // ------------------------------------------------------------------------------------
- // Arm C -- display:none on completed messages
- // ------------------------------------------------------------------------------------
function applyC() {
var info = prefixInfo();
@@ -1815,26 +1609,15 @@
);
}
- // ------------------------------------------------------------------------------------
- // Arm D -- detach the autoscroll subtree observer (PREBOOT)
- //
- // Two discriminators identify the autoscroll observer, and both come from
- // use-intent-aware-autoscroll.tsx:502: the target carries the aui-stream-viewport class, and
- // the options carry "aria-expanded" in attributeFilter. Nothing else in the app observes with
- // aria-expanded, and .aui-stream-viewport marks only real streaming-thread viewports.
- //
- // EVERY OTHER observe() CALL MUST PASS THROUGH UNCHANGED. reasoning.tsx,
- // research-activity-panel.tsx, animated-theme-toggler.tsx, tooltip-modal-layer.ts (twice),
- // settings-dialog.tsx, monitor-frame-store.ts and use-composer-pill-fit.ts all install
- // observers, and breaking any of them would change what the app renders, which converts an
- // ablation into a different page.
- //
- // The wrapper forwards `arguments` verbatim to the original, so calls with missing or invalid
- // arguments throw the same TypeError from the same function they always did. The matching
- // logic cannot throw: if reading options.attributeFilter fails for any reason, the call is
- // treated as not-matching and passes through, because passing an observe() through is always
- // safe and suppressing one that should not have been suppressed is not.
- // ------------------------------------------------------------------------------------
+ // Two discriminators identify the autoscroll observer, both from
+ // use-intent-aware-autoscroll.tsx:502: target has .aui-stream-viewport and options carry
+ // "aria-expanded" in attributeFilter. Nothing else in the app observes with aria-expanded.
+ // EVERY OTHER observe() CALL MUST PASS THROUGH UNCHANGED (reasoning, research panel, theme
+ // toggler, tooltip layer, settings dialog, monitor store, composer pill fit): breaking one
+ // turns an ablation into a different page.
+ // The wrapper forwards `arguments` verbatim, so invalid calls throw the same TypeError from
+ // the same function. The matcher cannot throw: an unreadable options.attributeFilter counts as
+ // not-matching and passes through, since passing an observe() through is always safe.
function targetIsStreamViewport(target) {
try {
@@ -1945,15 +1728,9 @@
return true;
}
- // ------------------------------------------------------------------------------------
- // Arm E -- neutralise the --aui-scroll-stabilizer write (PREBOOT)
- //
- // One custom property, by exact name. Every other setProperty call, custom property or not,
- // passes through with its arguments forwarded verbatim so that coercion and throwing are
- // unchanged. customPropSets counts every custom property write including the suppressed ones,
- // which is the context that tells a reader whether the stabilizer was one write in ten or one
- // in ten thousand.
- // ------------------------------------------------------------------------------------
+ // One custom property, by exact name; every other setProperty call forwards its arguments
+ // verbatim so coercion and throwing are unchanged. customPropSets counts every
+ // custom-property write including suppressed ones, so a reader can see the rate.
function installE() {
var proto = null;
@@ -1989,8 +1766,8 @@
counters.setPropertyCallsTotal += 1;
var name = null;
try {
- // A Symbol argument makes String() throw. In that case name stays null, the call
- // passes through, and the original throws exactly the TypeError it would have.
+ // A Symbol argument makes String() throw; name stays null, the call passes through, and the
+ // original throws exactly the TypeError it would have.
name = typeof property === "string" ? property : String(property);
} catch (e) {
name = null;
@@ -2033,35 +1810,19 @@
return true;
}
- // ------------------------------------------------------------------------------------
- // Arm F -- freeze the React scheduler, keep the DOM (PREBOOT capture, apply() freezes)
- //
- // THIS ARM IS DOM_CHANGING. While frozen, React renders nothing, so the stream stops
- // appearing on screen and the frozen side is not rendering the same page as the control. Its
- // cost is an UPPER BOUND on what React reconciliation costs and can never be quoted as a point
- // estimate; manifest.py prints it as `<= x`. It is here because a bound on React is still
- // worth having when the alternative is guessing, not because the number is clean.
- //
- // React's browser scheduler is the only runtime handle available. The ReactDOMRoot object is
- // not exposed on window, so root.unmount() is not reachable, and #root is just an element.
- // The scheduler does: const channel = new MessageChannel(); channel.port1.onmessage =
- // performWorkUntilDeadline; ... port2.postMessage(null). Intercepting the assignment to
- // port1.onmessage puts a dispatcher in front of performWorkUntilDeadline, and the freeze flag
- // decides whether the dispatcher forwards.
- //
- // capturedSchedulerPorts is what separates "React never used MessageChannel on this build"
- // (0 ports: NOT RUN, the treatment never happened) from "we froze it and nothing changed"
- // (ports captured, callbacks suppressed).
- //
- // SUPPRESSING SCHEDULER MESSAGES CAN WEDGE REACT PERMANENTLY. The message loop only posts the
- // next message from inside the handler it just ran, so a swallowed delivery ends the loop and
- // unfreezing alone does not restart it. revert() therefore posts one fresh message per
- // channel that had suppressions (schedulerResumeKicks) unless config.redeliverOnUnfreeze is
- // false, and says so in the returned reason. The suppressed events themselves are NOT
- // replayed: React's performWorkUntilDeadline ignores the event object and reads its own
- // queue, so one kick resumes exactly the work the swallowed messages would have done, and
- // replaying N events would run N slices that no longer correspond to anything.
- // ------------------------------------------------------------------------------------
+ // THIS ARM IS DOM_CHANGING: while frozen React renders nothing, so it is not rendering the
+ // control's page. Its cost is an UPPER BOUND on reconciliation, never a point estimate;
+ // manifest.py prints it as `<= x`.
+ // React's browser scheduler is the only runtime handle: ReactDOMRoot is not on window, and the
+ // scheduler assigns performWorkUntilDeadline to channel.port1.onmessage, so intercepting that
+ // assignment puts a dispatcher in front of it.
+ // capturedSchedulerPorts separates "React never used MessageChannel on this build" (0 ports,
+ // NOT RUN) from "we froze it and nothing changed".
+ // SUPPRESSING SCHEDULER MESSAGES CAN WEDGE REACT PERMANENTLY: the loop posts the next message
+ // only from inside the handler it just ran, so revert() posts one fresh kick per channel that
+ // had suppressions unless config.redeliverOnUnfreeze is false. Suppressed events are not
+ // replayed: performWorkUntilDeadline reads its own queue, so one kick resumes the work.
+ // Counted as `schedulerResumeKicks`.
function instrumentSchedulerPort(channel) {
var port = channel.port1;
@@ -2085,15 +1846,13 @@
return undefined;
}
counters.schedulerCallbacksDelivered += 1;
- // Called without a try/catch on purpose: an exception thrown by React must propagate
- // exactly as it would have. Swallowing it here would make the frozen run behave
- // differently from the control in a way that has nothing to do with the ablation.
+ // Deliberately not wrapped in try/catch: an exception thrown by React must propagate exactly
+ // as it would have, or the frozen run differs for reasons unrelated to the ablation.
return fn.call(port, event);
}
- // Assigning through the prototype accessor first, so the real port ends up calling the
- // dispatcher. This implicitly starts port1, which the scheduler does a moment later
- // anyway when it assigns its own handler.
+ // Assigned through the prototype accessor first so the real port calls the dispatcher; this
+ // implicitly starts port1, which the scheduler does a moment later anyway.
port.onmessage = dispatch;
Object.defineProperty(port, "onmessage", {
@@ -2107,9 +1866,9 @@
}
});
- // Some builds attach with addEventListener instead of onmessage. Without this, such a
- // build would report captured ports and zero suppressions, which reads as "we froze it
- // and React did not care" when the truth is that the freeze never reached the handler.
+ // Some builds attach with addEventListener instead of onmessage; without this, such a build
+ // reports captured ports and zero suppressions, reading as "React did not care" when the
+ // freeze never reached the handler.
try {
var origAdd = port.addEventListener;
var origRemove = port.removeEventListener;
@@ -2183,9 +1942,8 @@
return false;
}
- // Probe before committing: if the port instance refuses a redefined onmessage there is no
- // way to intercept, and the arm must read UNAVAILABLE rather than run and suppress
- // nothing.
+ // Probe before committing: a port that refuses a redefined onmessage cannot be intercepted,
+ // and the arm must read UNAVAILABLE rather than run and suppress nothing.
try {
var probe = new Original();
Object.defineProperty(probe.port1, "onmessage", {
@@ -2212,8 +1970,8 @@
try {
instrumentSchedulerPort(channel);
} catch (e) {
- // A channel we failed to instrument is still a working channel. The app must not
- // notice, and capturedSchedulerPorts simply does not count it.
+ // A channel we failed to instrument is still a working channel: the app must not notice, and
+ // capturedSchedulerPorts simply does not count it.
debug("could not instrument a MessageChannel", e);
}
return channel;
@@ -2250,8 +2008,8 @@
}
state.frozen = true;
if (counters.capturedSchedulerPorts === 0) {
- // Not an error yet: React may create its channel later. It is recorded in the reason
- // so that a run which ends with zero captured ports is read as NOT RUN.
+ // Not an error yet (React may create its channel later); recorded in the reason so a run
+ // ending with zero captured ports reads as NOT RUN.
return result(
true,
"freeze flag set, but no MessageChannel has been created yet; if " +
@@ -2313,19 +2071,12 @@
);
}
- // ------------------------------------------------------------------------------------
- // Arm G -- control: identical DOM, prior turns in the viewport
- //
- // The viewport carries the `scroll-smooth` class (thread.tsx:1728), so a scrollTop assignment
- // would animate. An animated scroll would still be running when the measured window starts,
- // which would put scroll animation cost inside the control and make the control the most
- // expensive arm. scrollBehavior is forced to auto for the assignment and restored afterwards.
- //
- // The app's own autoscroll may pull the thread back to the bottom at any time, which is why
- // controlVisibleMessages is a gauge that is re-measured on every potency() call rather than a
- // number recorded once at apply time: the honest question is whether the prior turns were
- // visible for the WINDOW, not whether they were visible for an instant.
- // ------------------------------------------------------------------------------------
+ // The viewport carries `scroll-smooth` (thread.tsx:1728), so assigning scrollTop would animate
+ // and put scroll-animation cost inside the control; scrollBehavior is forced to auto for the
+ // assignment and restored afterwards.
+ // The app's own autoscroll may pull the thread back at any time, so controlVisibleMessages is
+ // a gauge re-measured on every potency() call: the honest question is whether prior turns were
+ // visible for the WINDOW, not for an instant.
function applyG() {
var vp = viewportEl();
@@ -2401,9 +2152,9 @@
}
seen = visibleCounts();
- // Walk upwards until enough prior turns are on screen or the top is reached. Bounded
- // iterations: a loop that cannot terminate would hang the measured window, and reporting
- // "could not reach the target" is a usable result while a hang is not.
+ // Walk up until enough prior turns are on screen or the top is reached. Bounded iterations: a
+ // non-terminating loop would hang the measured window, and "could not reach the target" is a
+ // usable result while a hang is not.
while (seen.prior < target && attempts < 16) {
attempts += 1;
var before = 0;
@@ -2488,9 +2239,6 @@
}
}
- // ------------------------------------------------------------------------------------
- // apply / revert
- // ------------------------------------------------------------------------------------
function apply(armId) {
var id;
@@ -2506,9 +2254,8 @@
return result(false, unavailable[id], 0);
}
if (Object.prototype.hasOwnProperty.call(state.applied, id) && id !== "F") {
- // `applied` means "this call applied it". Arms are applied once per measured window
- // by design, so a second call is a no-op and says so rather than re-marking elements
- // and inflating the counters.
+ // `applied` means "this call applied it": arms apply once per measured window, so a second
+ // call is a no-op instead of re-marking elements and inflating the counters.
return result(
false,
"arm " + id + " was already applied at " + state.applied[id].at +
@@ -2547,8 +2294,8 @@
out = applyG();
}
} catch (e) {
- // An arm that throws has to be reported, not propagated: the app is mid-stream and an
- // exception here would end the run with no data at all.
+ // An arm that throws is reported, not propagated: the app is mid-stream and an exception here
+ // would end the run with no data at all.
return result(false, "arm " + id + " threw while applying: " + e, 0);
}
@@ -2575,9 +2322,8 @@
try {
if (id === "A" || id === "B" || id === "C") {
removeStyleSheet(id);
- // The inline and marker ledgers are global rather than per arm: a run applies one
- // arm, and unwinding everything is the behaviour least likely to leave a stray
- // !important behind on a page that is about to be measured again.
+ // The inline and marker ledgers are global, not per arm: a run applies one arm, and unwinding
+ // everything is least likely to leave a stray !important on a page about to be re-measured.
restoreInline();
unmarkAll();
out = revertResult(true, "stylesheet removed, inline properties and markers restored");
@@ -2646,9 +2392,6 @@
return out;
}
- // ------------------------------------------------------------------------------------
- // Install the preboot arms, then publish
- // ------------------------------------------------------------------------------------
if (armedPreboot.indexOf("D") !== -1) {
installD();
@@ -2685,8 +2428,8 @@
);
}
- // Feature checks for the runtime arms. These are capability questions about the browser, not
- // about the page, so a false here is UNAVAILABLE and never a zero difference.
+ // Feature checks for the runtime arms: capability questions about the browser, not the page,
+ // so a false here is UNAVAILABLE and never a zero difference.
if (!doc) {
markUnavailable("A", "no document");
markUnavailable("B", "no document");
diff --git a/tests/studio/studiobench/arms/ladder.py b/tests/studio/studiobench/arms/ladder.py
index 8323b526123..2967d4df516 100644
--- a/tests/studio/studiobench/arms/ladder.py
+++ b/tests/studio/studiobench/arms/ladder.py
@@ -50,8 +50,8 @@ from typing import Any, Iterable, Mapping, Sequence
from ..scoring.schema import Measure
from .manifest import ArmOutcome, ArmStatus
-#: The named mechanisms this ladder can remove. A step must name mechanisms from this set, so a
-#: typo becomes an error rather than a new, unexplained row in the report.
+#: The named mechanisms this ladder can remove. A step must name mechanisms from this set, so a typo
+#: becomes an error rather than a new, unexplained row in the report.
MECHANISMS: tuple[str, ...] = (
"paint_raster",
"offscreen_style_layout",
@@ -365,8 +365,8 @@ def differences(
and result.sum_of_steps
):
result.residual_ms = float(result.total.value) - float(result.sum_of_steps.value)
- # This is an arithmetic identity, not an empirical claim: the interior terms cancel. A
- # non-zero residual here means a bug in this function, not a finding about the app.
+ # An arithmetic identity, not an empirical claim: the interior terms cancel, so a non-zero residual
+ # means a bug in this function, not a finding about the app.
result.identity_holds = abs(result.residual_ms) < 1e-9
result.identity_note = (
"top minus floor equals the sum of the adjacent differences, identically, because the "
@@ -495,14 +495,14 @@ def interaction_terms(
return terms
-# ---------------------------------------------------------------------------------------
# the two declared routes
+
# ---------------------------------------------------------------------------------------
_SHIPPING: frozenset[str] = frozenset()
-#: Route 1 walks down the rendering pipeline first: stop painting, then stop laying out
-#: off-screen, then stop occupying layout at all, and only then touch the observers and React.
+#: Route 1 walks down the rendering pipeline first: stop painting, then stop laying out off-screen,
+#: then stop occupying layout at all, and only then touch the observers and React.
ROUTE_VISUAL_FIRST = LadderRoute(
route_id = "visual_first",
name = "paint, then off-screen layout, then geometry, then observers, then React",
@@ -546,9 +546,9 @@ ROUTE_VISUAL_FIRST = LadderRoute(
),
)
-#: Route 2 reaches the identical floor from the other end: kill the observers and React first,
-#: then walk down the rendering pipeline. If the mechanisms are additive both routes report the
-#: same per-mechanism numbers. Where they do not, that gap is the interaction term.
+#: Route 2 reaches the identical floor from the other end: kill the observers and React first, then
+#: walk down the rendering pipeline. If the mechanisms are additive both routes report the same
+#: per-mechanism numbers; where they do not, that gap is the interaction term.
ROUTE_SCHEDULER_FIRST = LadderRoute(
route_id = "scheduler_first",
name = "observers, then React, then paint, then off-screen layout, then geometry",
diff --git a/tests/studio/studiobench/arms/layoutcost.py b/tests/studio/studiobench/arms/layoutcost.py
index 918d1be003a..0b4a291e1b8 100644
--- a/tests/studio/studiobench/arms/layoutcost.py
+++ b/tests/studio/studiobench/arms/layoutcost.py
@@ -35,10 +35,10 @@ from ..scoring.schema import Measure
LAYOUTCOST_JS_PATH = Path(__file__).resolve().parents[1] / "instruments" / "layoutcost.js"
-#: The instrument level at which this may run. Headline numbers come from level 0 only.
+#:The instrument level at which this may run. Headline numbers come from level 0 only.
LAYOUTCOST_LEVEL = 3
-#: Counter families the browser side reports, each with its own `attempted` flag.
+#:Counter families the browser side reports, each with its own `attempted` flag.
COUNTER_FAMILIES = (
"scrollHeightReads",
"scrollTopWrites",
@@ -187,8 +187,8 @@ class LayoutCostInstrument:
self._ctx = ctx
def start_cell(self, cell: Any) -> None:
- # `ctx.page` may be replaced between cells when a crashed renderer is recovered, so the
- # page is re-read here rather than cached in attach().
+ # `ctx.page` may be replaced between cells when a crashed renderer is recovered, so the page is
+ # re-read here rather than cached in attach().
self._page = getattr(self._ctx, "page", None)
def open(self, window: Any) -> None:
@@ -214,9 +214,9 @@ class LayoutCostInstrument:
return reading_from_snapshot(snapshot).to_json()
def end_cell(self, cell: Any) -> dict[str, Any] | None:
- # The harness contract requires every instrument at level >= 1 to declare its own cost.
- # This one declares the LOWER BOUND it can measure itself, and says so, because the real
- # number comes from the paired with/without cell that only the deep tier runs.
+ # The harness contract requires every instrument at level >= 1 to declare its own cost. This one
+ # declares the LOWER BOUND it can measure itself, because the real number comes from the paired
+ # with/without cell that only the deep tier runs.
if self._page is None:
return {"overhead_ms": None, "overhead_attempted": False}
try:
diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py b/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py
index 66d5099e242..3fa395ec485 100644
--- a/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py
+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py
@@ -72,8 +72,9 @@ def _arm(
)
-# ---------------------------------------------------------------------------------------
# manifest: invariance and potency
+
+
# ---------------------------------------------------------------------------------------
@@ -200,8 +201,9 @@ def test_an_equivalent_arm_must_declare_its_diff_and_an_exact_one_may_not():
_arm(declared_diff = DeclaredDiff(normaliser = "n", keys = ("style",)))
-# ---------------------------------------------------------------------------------------
# ladder
+
+
# ---------------------------------------------------------------------------------------
@@ -385,8 +387,9 @@ def test_required_rungs_covers_both_routes():
assert "D" in keys and "A" in keys
-# ---------------------------------------------------------------------------------------
# calibration
+
+
# ---------------------------------------------------------------------------------------
@@ -485,8 +488,9 @@ def test_a_batch_with_no_null_reading_has_no_noise_floor():
assert "no measured noise floor" in verdict.reason
-# ---------------------------------------------------------------------------------------
# dose-response
+
+
# ---------------------------------------------------------------------------------------
@@ -549,8 +553,9 @@ def test_two_points_do_not_make_a_line():
assert fit.verdict == "NO FIT"
-# ---------------------------------------------------------------------------------------
# armpack
+
+
# ---------------------------------------------------------------------------------------
@@ -612,8 +617,9 @@ def test_a_matching_armpack_resolves(tmp_path: Path):
assert resolution.require().target_dist_digest == "digest-1"
-# ---------------------------------------------------------------------------------------
# recovery
+
+
# ---------------------------------------------------------------------------------------
@@ -671,8 +677,9 @@ def test_worse_after_delete_points_at_the_delete_path():
assert result.classification == "WORSE AFTER DELETE"
-# ---------------------------------------------------------------------------------------
# knobs
+
+
# ---------------------------------------------------------------------------------------
diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py b/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py
index f5601f162af..05778cc7ef7 100644
--- a/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py
+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py
@@ -76,8 +76,8 @@ def _outcome(
)
-#: One coherent synthetic world: the autoscroll observer is the dominant cost on both routes,
-#: and the two routes disagree about it, which is the interaction the design exists to surface.
+#: One coherent synthetic world: the autoscroll observer is the dominant cost on both routes, and
+#: the two routes disagree about it, which is the interaction the design exists to surface.
_VISUAL = {
"shipping": 40.0,
"A": 34.0,
@@ -140,8 +140,9 @@ def _blind_calibration():
)
-# ---------------------------------------------------------------------------------------
# planning
+
+
# ---------------------------------------------------------------------------------------
@@ -166,8 +167,9 @@ def test_no_scene_durations_at_all_is_a_refusal_not_a_pass():
assert_equal_scene_duration({})
-# ---------------------------------------------------------------------------------------
# judging and rendering
+
+
# ---------------------------------------------------------------------------------------
@@ -250,8 +252,8 @@ def test_fix_implications_rank_the_largest_step_and_name_its_fix():
)
rendered = render_fix_implications(result)
top = rendered.splitlines()[1].strip()
- # the two routes disagree about the autoscroll observer, so the headline is a RANGE and both
- # routes are named; quoting only the larger would let a reader pick the flattering route
+ # the two routes disagree about the autoscroll observer, so the headline is a RANGE and both routes
+ # are named; quoting only the larger would let a reader pick the flattering route
assert top.startswith("9.000 to 18.000 ms")
assert "autoscroll_forced_layout" in top
assert "routes disagree: scheduler_first 18.000, visual_first 9.000" in rendered
@@ -288,8 +290,9 @@ def test_a_batch_with_a_dose_fit_an_armpack_refusal_and_a_recovery_renders_all_t
assert "RETAINED STRUCTURE" in rendered
-# ---------------------------------------------------------------------------------------
# the harness layer's row stream
+
+
# ---------------------------------------------------------------------------------------
diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py b/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py
index 582b6630abb..713a8d43254 100644
--- a/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py
+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py
@@ -61,9 +61,9 @@ def test_the_hooks_are_off_unless_asked_for(main_src: str, monkeypatch):
assert os.environ.get("SBENCH_EXTRA_INIT_SCRIPT") is None
assert os.environ.get("SBENCH_PAGE_CONSOLE") is None
- # Both call sites are guarded by a plain truthiness check on the variable, so an unset
- # variable cannot reach `add_init_script` or `page.on`. Pinned as source, because the failure
- # this guards is a refactor that hoists either call out of its `if`.
+ # Both call sites are guarded by a plain truthiness check on the variable, so an unset variable
+ # cannot reach `add_init_script` or `page.on`. Pinned as source, because the failure this guards
+ # is a refactor that hoists either call out of its `if`.
for guarded in ("if extra_init:", "if console_prefix:"):
assert guarded in main_src, f"{guarded!r} is gone; the hook may no longer be opt-in"
@@ -83,11 +83,10 @@ def test_the_probe_path_is_validated_before_anything_is_started(main_src: str):
assert "except (OSError, UnicodeDecodeError) as exc:" in main_src
# Read once, used later. A second read at launch time would reintroduce the window.
assert main_src.count("Path(extra_init).read_text") == 1
- # AND ahead of the archive, which is the other thing a refusal must not arrive after. Reusing
- # an --out without --resume moves the payload aside, so reading the probe after that call let a
- # path typo take payload.jsonl off its standard path while running no benchmark at all. The
- # behaviour is pinned in runtime/selftest/test_studiobench_run_acquisition.py; the order is
- # pinned here because that is what makes it true.
+ # AND ahead of the archive, the other thing a refusal must not arrive after: reusing an --out
+ # without --resume moves the payload aside, so reading the probe after that call let a path typo
+ # take payload.jsonl off its standard path while running no benchmark at all. The behaviour is
+ # pinned in runtime/selftest/test_studiobench_run_acquisition.py; the order is pinned here.
assert read_at < main_src.index("prepare_payload(")
@@ -136,9 +135,9 @@ def test_a_probe_run_records_the_gate_that_makes_it_unscorable(main_src: str):
"""
assert '"probe_init_script": extra_init or None' in main_src
- # Matched without pinning the indent. `run()` puts side acquisition under a cleanup guard, so
- # everything from the acquisition loop to the end of setup sits one level deeper than it did,
- # and the gate moving with it is not the thing this test is here to notice.
+ # Matched without pinning the indent: `run()` puts side acquisition under a cleanup guard, so
+ # everything from the acquisition loop to the end of setup sits one level deeper, and the gate
+ # moving with it is not what this test notices.
assert re.search(r"rec\.gate\(\s*\n\s*\"probe_free\",", main_src)
@@ -152,8 +151,8 @@ def test_roots_are_adopted_at_insertion_not_on_the_sample_tick(probe_src: str):
assert "MutationObserver" in probe_src
assert "adoptAdded" in probe_src
- # The DOCUMENT, not documentElement. An init script can run before the parser has created
- # the root element, and `observe(null, ...)` throws into the catch that hides it.
+ # The DOCUMENT, not documentElement: an init script can run before the parser has created the
+ # root element, and `observe(null, ...)` throws into the catch that hides it.
assert "observe(doc, {" in probe_src
# Added nodes only. A document-wide re-scan per mutation would make the probe the load.
assert "addedNodes" in probe_src
@@ -171,8 +170,8 @@ def test_the_fallback_and_padding_buckets_cannot_both_count_one_root(probe_src:
assert "ROLE_PX" in probe_src
assert "out.fallbackBite += 1;" in probe_src
assert 'targetHeight(px, "padding")' in probe_src
- # getBoundingClientRect() is the border box, so BOTH targets carry the root's own padding.
- # Comparing against the bare declared length pinned fallbackBite at zero whatever happened.
+ # getBoundingClientRect() is the border box, so BOTH targets carry the root's own padding;
+ # comparing against the bare declared length pinned fallbackBite at zero whatever happened.
assert 'targetHeight(px, "fallback")' in probe_src
assert 'return (which === "fallback" ? px.fallback : 0) + px.padding;' in probe_src
@@ -254,8 +253,8 @@ def test_the_probe_source_is_the_first_thing_in_its_script():
"semantics than the file it was read from"
)
assert "window.__sbExtraInitScript" in scripts[0]
- # Appended after an explicit statement boundary, so neither ASI nor an unterminated
- # expression can join it to the probe's last line.
+ # Appended after an explicit statement boundary, so neither ASI nor an unterminated expression can
+ # join it to the probe's last line.
assert (
scripts[0][len(source) :]
.lstrip("\n")
@@ -379,8 +378,8 @@ def test_the_event_counter_is_the_one_potency_rests_on(probe_src: str):
assert "contentvisibilityautostatechange" in probe_src
assert "ev_skip" in probe_src
- # The geometry route is a documented false negative and must stay documented rather than
- # quietly removed: someone will reimplement it otherwise.
+ # The geometry route is a documented false negative and must stay documented rather than quietly
+ # removed: someone will reimplement it otherwise.
assert "KNOWN FALSE NEGATIVE" in probe_src
@@ -398,9 +397,9 @@ def _node_parses(source: str):
if node is None:
pytest.skip("no node on PATH; this assertion needs a real JS parser, not a regex")
with tempfile.TemporaryDirectory() as tmp:
- # `--check` reads a FILE, so the IIFE is written out rather than passed with `-e`: an
- # argument would have to survive the shell, and the point of this helper is that nothing
- # between the producer and the parser is allowed to edit the bytes.
+ # `--check` reads a FILE, so the IIFE is written out rather than passed with `-e`: an argument
+ # would have to survive the shell, and the point of this helper is that nothing between the
+ # producer and the parser is allowed to edit the bytes.
script = Path(tmp) / "init_script.js"
script.write_text("(() => {\n" + source + "\n})();", encoding = "utf-8")
done = subprocess.run([node, "--check", str(script)], capture_output = True, text = True)
diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py b/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py
index 03c5e674967..7040b0497d8 100644
--- a/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py
+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py
@@ -35,8 +35,9 @@ from studiobench.instruments.selfcheck import ( # noqa: E402
from studiobench.scoring.schema import EXCLUSION_REASONS, check_exclusion_reasons # noqa: E402
-# ---------------------------------------------------------------------------------------
# the injected stall
+
+
# ---------------------------------------------------------------------------------------
@@ -58,8 +59,9 @@ def test_no_stall_reading_at_all_is_a_failure_not_a_pass():
assert gate.measured.has_reading is False
-# ---------------------------------------------------------------------------------------
# the injected input delay
+
+
# ---------------------------------------------------------------------------------------
@@ -73,8 +75,9 @@ def test_an_input_path_that_does_not_move_is_not_measuring_input():
assert "will not move when the app gets slower" in gate.detail
-# ---------------------------------------------------------------------------------------
# scene contrast: the blindness check
+
+
# ---------------------------------------------------------------------------------------
@@ -88,8 +91,9 @@ def test_an_instrument_that_cannot_tell_heavy_from_trivial_is_blind():
assert "BLIND" in gate.detail
-# ---------------------------------------------------------------------------------------
# longtask support
+
+
# ---------------------------------------------------------------------------------------
@@ -109,8 +113,9 @@ def test_an_unreadable_support_list_is_recorded_and_not_guessed():
assert gate.fatal is False
-# ---------------------------------------------------------------------------------------
# the clock-pair control ratio
+
+
# ---------------------------------------------------------------------------------------
@@ -124,8 +129,9 @@ def test_a_moving_control_ratio_means_the_measurement_moved():
assert "the MEASUREMENT moved, not the page" in gate.detail
-# ---------------------------------------------------------------------------------------
# three-clock agreement
+
+
# ---------------------------------------------------------------------------------------
@@ -188,8 +194,9 @@ def test_clock_disagreement_is_a_declared_exclusion_reason():
assert "clock_disagreement" in EXCLUSION_REASONS
-# ---------------------------------------------------------------------------------------
# the whole gate set
+
+
# ---------------------------------------------------------------------------------------
@@ -291,7 +298,7 @@ def test_the_injection_script_burns_only_on_sse_chunks():
js = stream_cost_injection_init_script(3.0)
assert 'indexOf("data:")' in js
- # Queued, not inline: the burn has to land inside the measured chain whichever TextDecoder
- # wrapper ended up outermost.
+ # Queued, not inline: the burn has to land inside the measured chain whichever TextDecoder wrapper
+ # ended up outermost.
assert "queueMicrotask" in js
assert "3.0" in js
diff --git a/tests/studio/studiobench/attribution/vite.studiobench.config.ts b/tests/studio/studiobench/attribution/vite.studiobench.config.ts
index 0d2a76ec3b2..0af98566183 100644
--- a/tests/studio/studiobench/attribution/vite.studiobench.config.ts
+++ b/tests/studio/studiobench/attribution/vite.studiobench.config.ts
@@ -1,51 +1,32 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// studiobench attribution build.
-//
-// Identical to `studio/frontend/vite.config.ts` in every respect that affects
-// what the app DOES, and different only in what it lets us SEE:
-//
-// 1. `react-dom/client` is aliased to `react-dom/profiling`, which is the
-// only build where React's `<Profiler>` `onRender` callback exists at all.
-// 2. Hidden sourcemaps, so our own frames get real names without shipping a
-// `//# sourceMappingURL` comment.
-// 3. Minifier name preservation, so our own functions and classes keep
-// their source names instead of being mangled to single letters.
-//
-// The resulting dist is injected into a REAL Unsloth install through the
-// existing `unsloth studio --frontend <dir>` flag (see `studio/backend/run.py`
-// around line 2820), so the backend, auth and storage stay real and only the
-// frontend bundle is swapped. That matters: the failure this whole tool exists
-// to correct was a fixture that did not run the real path.
-//
-// WHAT THIS BUILD DOES NOT FIX, and cannot. React's production and profiling
-// dists are minified by React's own release pipeline BEFORE Vite ever sees
-// them. Name preservation and sourcemaps therefore recover OUR names and
-// nothing whatsoever inside `react-dom`: the original identifier really is
-// `Zk`. Recovering react-dom names is the job of the symbol bridge in
-// `tests/studio/studiobench/analysis/symbols.py`, not of this config, and no
-// build flag here can do it.
+// studiobench attribution build. Identical to `studio/frontend/vite.config.ts` in every respect
+// that affects what the app DOES, differing only in what it lets us SEE: `react-dom/client`
+// aliased to `react-dom/profiling` (the only build where React's `<Profiler>` `onRender` exists),
+// hidden sourcemaps, and minifier name preservation.
+// The resulting dist is injected into a REAL Unsloth install through `unsloth studio --frontend
+// <dir>`, so backend, auth and storage stay real and only the frontend bundle is swapped: the
+// failure this tool exists to correct was a fixture that did not run the real path.
+// WHAT THIS BUILD CANNOT FIX: React's production and profiling dists are minified by React's own
+// release pipeline before Vite sees them, so name preservation and sourcemaps recover OUR names
+// and nothing inside `react-dom` (the original identifier really is `Zk`). That is the job of the
+// symbol bridge in `analysis/symbols.py`.
+// That is `tests/studio/studiobench/analysis/symbols.py`.
import { createRequire } from "node:module";
import path from "node:path";
const FRONTEND_ROOT = path.resolve(__dirname, "../../../../studio/frontend");
-// The plugins are resolved from the FRONTEND package, not from this directory,
-// and that is not a stylistic preference. Vite bundles a `--config` file with
-// its bare imports left external (`bundleConfigFile` sets `root =
-// path.dirname(fileName)`) and then loads the bundle under the config file's
-// own path: CJS via `module._compile(code, <config path>)`, ESM via a temp file
-// in the nearest ancestor `node_modules`, of which this directory has none. So
-// Node starts its `node_modules` walk here and climbs to the repository root,
-// and NOTHING on that path has a `node_modules` -- only
-// `studio/frontend/node_modules` does. Written with plain top-level imports
-// this config could not be loaded from any working directory, by any `vite`
-// binary: `Cannot find module '@tailwindcss/vite'`, and the same for the React
-// plugin and for `vite` itself. Anchoring the lookup at the frontend package's
-// `package.json` is the whole fix, and it points at the same installed copies
-// the shipping `studio/frontend/vite.config.ts` uses.
+// The plugins are resolved from the FRONTEND package, not this directory. Vite bundles a
+// `--config` file with bare imports left external and loads it under the config file's own path,
+// so Node starts its `node_modules` walk here and climbs to the repository root, where nothing
+// has a `node_modules`: only `studio/frontend/node_modules` does. With plain top-level imports
+// this config could not be loaded from any working directory by any `vite` binary. Anchoring the
+// lookup at the frontend package's `package.json` points at the same installed copies the
+// shipping config uses.
+// Vite's `bundleConfigFile` sets root = path.dirname(fileName).
const requireFromFrontend = createRequire(path.join(FRONTEND_ROOT, "package.json"));
const tailwindcss = requireFromFrontend("@tailwindcss/vite").default;
const react = requireFromFrontend("@vitejs/plugin-react").default;
@@ -54,8 +35,8 @@ const { defineConfig } = requireFromFrontend("vite");
export default defineConfig({
root: FRONTEND_ROOT,
plugins: [react(), tailwindcss()],
- // Keep an unrelated PostCSS config in an ancestor directory from leaking
- // into Unsloth installs. Tailwind is provided by its dedicated Vite plugin.
+ // Keep an unrelated PostCSS config in an ancestor directory from leaking into Unsloth installs.
+ // Tailwind is provided by its dedicated Vite plugin.
css: {
postcss: {
plugins: [],
@@ -99,22 +80,14 @@ export default defineConfig({
},
},
resolve: {
- // An ARRAY of regex aliases, not the object form, and the react-dom entry
- // is deliberately anchored. Two traps, both of which produce a build that
- // looks fine and is not:
- //
- // - Aliasing the bare specifier `react-dom` recurses forever. The
- // profiling bundle's own first lines are
- // `require("react"), require("react-dom")`, so rewriting `react-dom`
- // points it at itself.
- // - A plain string `find: "react-dom/client"` is prefix matching in
- // Vite, so it would also rewrite `react-dom/profiling` and produce the
- // same recursion by a different route.
- //
- // `/^react-dom\/client$/` matches exactly the specifier the app imports and
- // nothing else. Bare `react-dom` is left alone on purpose: it is a small
- // shim that dispatches through the injected internals object, so it drives
- // whichever renderer actually loaded.
+ // An ARRAY of regex aliases, not the object form, and the react-dom entry is deliberately
+ // anchored. Aliasing the bare specifier `react-dom` recurses forever, because the profiling
+ // bundle's own first lines are `require("react"), require("react-dom")`; and a plain string
+ // `find: "react-dom/client"` is prefix matching in Vite, so it would also rewrite
+ // `react-dom/profiling` into the same recursion.
+ // `/^react-dom\/client$/` matches exactly the specifier the app imports. Bare `react-dom` is left
+ // alone on purpose: it is a small shim dispatching through the injected internals object, so it
+ // drives whichever renderer actually loaded.
alias: [
{ find: /^react-dom\/client$/, replacement: "react-dom/profiling" },
{ find: /^@\//, replacement: `${path.resolve(FRONTEND_ROOT, "./src")}/` },
@@ -130,70 +103,44 @@ export default defineConfig({
build: {
outDir: path.resolve(__dirname, "dist"),
emptyOutDir: true,
- // `hidden` emits the .map files but suppresses the sourceMappingURL
- // comment, so the bundle is byte-comparable to a shipping one apart from
- // the profiling renderer, and devtools does not silently start resolving
- // maps mid-measurement.
+ // `hidden` emits the .map files but suppresses the sourceMappingURL comment, so the bundle is
+ // byte-comparable to a shipping one apart from the profiling renderer, and devtools does not
+ // silently start resolving maps mid-measurement.
sourcemap: "hidden",
- // Vite 8 minifies with oxc by default, and without a name-preservation
- // knob the minifier renames our own components too: the app-side frames
- // become single letters, which would leave the symbol bridge with no
- // anchors to validate against. Measured on this tree, of the 403
- // `export function <Component>` names in `studio/frontend/src` only 37
- // survive an unconfigured build, against 390 with the option below.
- //
- // The knob is the OXC MINIFIER's `mangle.keepNames`, reached through
- // `output.minify`, and NOT the bundler-level `output.keepNames`. That
- // distinction is the whole reason this block looks the way it does.
- // `output.keepNames: true` on the rolldown that Vite 8.0.16 pins
- // (`[email protected]`, an exact pin in Vite's own `dependencies`) fails the
- // build outright:
- //
- // [MISSING_EXPORT] "toString" is not exported by
- // "node_modules/underscore/modules/_setup.js"
- //
- // and 23 more like it. That is rolldown#9973: under `keepNames` the
- // bundler splits a multi-declarator `export var a = 1, b = 2` into
- // separate declarations and drops the `export` keyword while doing it, so
- // every name after the split silently stops being exported. Underscore's
- // `_setup.js` is written almost entirely in that style
- // (`export var push = ..., slice = ..., toString = ...`) and we reach it
- // through `@dagrejs/graphlib`, so the build aborts. Fixed upstream by
- // rolldown#9974, first released in the 1.1.x line; unavailable to us
- // without overriding the rolldown that Vite pins, which would change the
- // bundler under the shipping UI and is not something an attribution-only
- // config gets to do.
- //
- // `mangle.keepNames` is the option that actually matters here anyway.
- // Bundler-level renaming only appends deconflicting suffixes and leaves
- // names readable; it is the minifier's mangler that turns `AppSidebar`
- // into a letter. `compress.keepNames` is set alongside it because oxc
- // documents the pair as belonging together -- DCE has to stop treating
- // the name-carrying binding as dead. Vite spreads the user `output` last
- // (`buildOutputOptions`), so `minify` here replaces Vite's default
- // `minify: true` rather than being clobbered by it; compression and
- // mangling both stay on, and the bundle grows only ~3.9% from carrying
- // the real names.
+ // Vite 8 minifies with oxc by default, and without a name-preservation knob the minifier renames
+ // our own components too, leaving the symbol bridge with no anchors: of the 403 `export function
+ // <Component>` names in `studio/frontend/src` only 37 survive an unconfigured build, against 390
+ // with the option below.
+ // The knob is the OXC MINIFIER's `mangle.keepNames`, reached through `output.minify`, and NOT the
+ // bundler-level `output.keepNames`, which on the rolldown Vite 8.0.16 pins fails the build
+ // outright with `[MISSING_EXPORT] "toString" is not exported by underscore/modules/_setup.js` and
+ // 23 more. That is rolldown#9973: under `keepNames` the bundler splits a multi-declarator `export
+ // var a = 1, b = 2` and drops the `export` keyword, so every name after the split stops being
+ // exported. Underscore's `_setup.js` is written that way and we reach it through
+ // `@dagrejs/graphlib`. Fixed upstream by rolldown#9974 in the 1.1.x line, unavailable without
+ // overriding the rolldown Vite pins, which an attribution-only config does not get to do.
+ // The pin is `[email protected]`.
+ // `mangle.keepNames` is the option that matters anyway: bundler-level renaming only appends
+ // deconflicting suffixes, while the minifier's mangler is what turns `AppSidebar` into a letter.
+ // `compress.keepNames` is set alongside it because oxc documents the pair as belonging together
+ // (DCE has to stop treating the name-carrying binding as dead). Vite spreads the user `output`
+ // last, so `minify` here replaces Vite's default rather than being clobbered; compression and
+ // mangling stay on and the bundle grows only ~3.9%.
+ // In `buildOutputOptions`.
rolldownOptions: {
output: {
minify: {
compress: { keepNames: { function: true, class: true } },
mangle: { keepNames: { function: true, class: true } },
},
- // Marks the bundle so the harness can prove it is measuring the
- // attribution build and not a stale shipping dist left in the
- // directory handed to `unsloth studio --frontend`. Read by
+ // Marks the bundle so the harness can prove it is measuring the attribution build and not a stale
+ // shipping dist left in the directory handed to `unsloth studio --frontend`. Read by
// `../analysis/bridge_build.py:assert_attribution_build`.
- //
- // This is a BANNER and not a `define`, and the difference is not
- // cosmetic. `define` performs identifier SUBSTITUTION in source: it
- // rewrites occurrences of a token that already appear in the code. No
- // Unsloth source file mentions `__STUDIOBENCH_ATTRIBUTION_BUILD__`, so
- // a `define` entry substitutes nothing, emits nothing, and the marker
- // is simply absent at runtime. Built that way first; the assertion
- // could never pass and the bundle was byte-identical with and without
- // the option. A banner is prepended unconditionally, so the global
- // genuinely exists.
+ // This is a BANNER and not a `define`: `define` performs identifier SUBSTITUTION in source, and no
+ // Unsloth source file mentions `__STUDIOBENCH_ATTRIBUTION_BUILD__`, so a `define` entry
+ // substitutes nothing and the marker is simply absent at runtime. Built that way first, the
+ // assertion could never pass and the bundle was byte-identical with and without the option. A
+ // banner is prepended unconditionally, so the global genuinely exists.
banner: "globalThis.__STUDIOBENCH_ATTRIBUTION_BUILD__ = true;",
},
},
diff --git a/tests/studio/studiobench/build.py b/tests/studio/studiobench/build.py
index 5000b67d166..2673f388308 100644
--- a/tests/studio/studiobench/build.py
+++ b/tests/studio/studiobench/build.py
@@ -45,7 +45,7 @@ def _copy_package(staging: Path) -> None:
target.parent.mkdir(parents = True, exist_ok = True)
shutil.copytree(PKG, target, ignore = shutil.ignore_patterns(*EXCLUDE_DIRS, "*.pyc", "*.pyo"))
# `tests` and `tests/studio` must be importable packages inside the archive, and they are not
- # packages in the repository -- the harnesses there are standalone scripts.
+ # packages in the repository: the harnesses there are standalone scripts.
for pkg_dir in (staging / "tests", staging / "tests" / "studio"):
init = pkg_dir / "__init__.py"
if not init.exists():
@@ -86,8 +86,8 @@ def build(out: Path = DEFAULT_OUT, compressed: bool = True) -> Path:
"artifact ships a benchmark with no content."
)
- # A __main__.py at the archive root is what `python foo.pyz` executes. It re-exports the
- # package's CLI rather than duplicating it.
+ # A __main__.py at the archive root is what `python foo.pyz` executes. It re-exports the package's
+ # CLI rather than duplicating it.
(staging / "__main__.py").write_text(
"import sys\n"
"from tests.studio.studiobench.__main__ import main\n"
diff --git a/tests/studio/studiobench/fixture/corpus.py b/tests/studio/studiobench/fixture/corpus.py
index 3139de8678e..03ad263c82a 100644
--- a/tests/studio/studiobench/fixture/corpus.py
+++ b/tests/studio/studiobench/fixture/corpus.py
@@ -48,60 +48,51 @@ UNITS_JSONL = FROZEN_DIR / "units.jsonl"
MANIFEST_JSON = FROZEN_DIR / "manifest.json"
CORPUS_SEED = 20260819
-# 2 added math. A number taken on v1 and a number taken on v2 are measurements of two different
-# films, so they are not comparable and `floor_table` refuses to pool them. Bump this whenever the
-# generated text changes at all, and never edit the generator without bumping it.
+# 2 added math. A number taken on v1 and one on v2 measure two different films, so
+# `floor_table` refuses to pool them. Bump this whenever the generated text changes.
CORPUS_VERSION = 2
-# Span density, calibrated against the field capture. See the module docstring. These are now the
-# MEANS of a jittered distribution rather than fixed sizes -- see `_jitter`.
+# Span density, calibrated against the field capture (see the module docstring). These are
+# MEANS of a jittered distribution rather than fixed sizes; see `_jitter`.
PROSE_CHARS = 1250
FENCE_CHARS = 1800
PREAMBLE_FRACTION = 0.25
-# How far each block's size may wander from its mean, as a fraction. Real replies do not emit
-# identically sized paragraphs and fences, and a fixture that does lets a cost that is really
-# per-block masquerade as per-character (and vice versa): with every block the same size the two
-# are perfectly collinear and no measurement can separate them. Jitter breaks that.
-#
-# Applied through the PER-UNIT rng, so unit 40 is still the same text whether it was reached by
-# generating 41 units or by asking for that one. Randomised is not the same as nondeterministic:
-# the corpus hash still pins every byte.
+# How far each block's size may wander from its mean. With every block the same size,
+# per-block and per-character cost are perfectly collinear and no measurement can separate
+# them; jitter breaks that. Applied through the PER-UNIT rng, so unit 40 is the same text
+# however it was reached, and the corpus hash still pins every byte.
BLOCK_JITTER = 0.55
-# Unit sizes wander too, around the escalating nominal. Same reason at the turn level: a thread of
-# turns that are all exactly 10,000 characters is not the shape of a real session, and the rung
-# planner then has no short turns to land its target on.
+# Unit sizes wander too, around the escalating nominal: a thread of turns all exactly 10,000
+# characters is not the shape of a real session, and the rung planner then has no short turns
+# to land its target on.
UNIT_JITTER = 0.35
-# Tool calls per unit, as a range. The thread renders a tool group for each, which is a component
-# with its own mount and update cost and which nothing in the corpus previously exercised.
+# Tool calls per unit, as a range. The thread renders a tool group for each, a component with
+# its own mount and update cost that nothing in the corpus previously exercised.
TOOL_CALLS_PER_UNIT = (0, 3)
-#: The stored part shape that actually renders a tool block. VERIFIED, not assumed: seeding a flat
-#: `{"type": "tool-call", ...}` and an assistant-ui style `{"type": "tool-invocation", ...}` both
-#: rendered their sibling text and NO tool UI, while this shape produced a "Used tool" group. A
-#: corpus full of parts the app ignores looks richer and measures nothing.
+#: The stored part shape that actually renders a tool block. VERIFIED, not assumed: a flat
+#: `{"type": "tool-call"}` and an assistant-ui `{"type": "tool-invocation"}` both rendered
+#: their sibling text and NO tool UI, while this shape produced a "Used tool" group.
TOOL_NAMES = ("web_search", "code_execution", "python", "terminal", "search_knowledge_base")
# The escalating cycle. Reasoning first because a turn's reasoning is what the pane holds open.
CYCLE_BASE = ((("reasoning", 10_000), ("code", 8_000)),)
-# Doubling stops here. Without a cap the eighth cycle is a single 2.5M-character reply, which is
-# not a thread, it is one document -- and it makes the 1M rung one turn deep, where every
-# per-message cost this tool exists to measure is multiplied by one.
+# Doubling stops here: without a cap the eighth cycle is a single 2.5M-character reply, making
+# the 1M rung one turn deep, where every per-message cost this tool measures is x1.
MAX_UNIT_CHARS = 320_000
-# Text below `units.jsonl` ships for every rung up to and including this many characters. Beyond
-# it the units are regenerated on the tester's machine and checked against the manifest hash, so
-# drift is still impossible; only the bytes are not carried in the artifact.
+# Text below `units.jsonl` ships for every rung up to this many characters; beyond it units
+# are regenerated on the tester's machine and checked against the manifest hash.
SHIPPED_CHARS_BUDGET = 460_000
-# ── the vocabulary ──────────────────────────────────────────────────
-#
-# Deliberately mundane. The point of the corpus is span density and uniqueness, not realism of
-# meaning, and a vocabulary a reader can skim is a vocabulary a reader can spot a bug in.
+# Deliberately mundane vocabulary. The point of the corpus is span density and uniqueness, not
+# realism of meaning, and a vocabulary a reader can skim is one a reader can spot a bug in.
+# ── the vocabulary ──────────────────────────────────────────────────
_NOUNS = (
"buffer",
"scheduler",
@@ -204,18 +195,12 @@ _SHORT = (
"tmp",
)
+# Corpus v1 contained not one dollar sign across 519,859 characters, making every math cost
+# unmeasurable by construction: `preprocessLaTeX` measured as a real cost in isolation and as
+# an exact NULL in the browser, because the fixture gave it nothing to do. Both delimiter
+# families are here on purpose: `$...$` is what remark-math consumes directly, while `\(...\)`
+# is what `preprocessLaTeX` exists to REWRITE.
# ── math ────────────────────────────────────────────────────────────
-#
-# Corpus v1 contained not one dollar sign across 519,859 characters, which made every math cost in
-# the app unmeasurable by construction. That is not a hypothetical gap: `preprocessLaTeX` was
-# measured as a real cost in isolation and then as an exact NULL in the browser, and the reason was
-# that the fixture gave it nothing to do. A benchmark that cannot see a cost cannot be used to
-# argue the cost is not there.
-#
-# Both delimiter families are here on purpose. `$...$` and `$$...$$` are what remark-math consumes
-# directly; `\(...\)` and `\[...\]` are what `preprocessLaTeX` exists to REWRITE into that form, so
-# a corpus carrying only the first kind exercises the renderer while leaving the preprocessor's
-# actual work uncovered.
_MATH_OPS = ("+", "-", "\\cdot", "\\times", "\\oplus")
_MATH_RELS = ("=", "\\le", "\\ge", "\\approx", "\\equiv")
_MATH_FUNCS = ("\\log", "\\exp", "\\sin", "\\cos", "\\tanh")
@@ -232,25 +217,20 @@ _MATH_GREEK = (
"\\omega",
)
-# Chance that a non-preamble prose BLOCK is a display-math block instead.
-#
-# Math takes prose's slot rather than being appended alongside it, and is drawn from the same
-# jittered size distribution. That keeps the prose/fence alternation exactly as it was: same number
-# of fences, same fence sizes, so the Shiki span density the whole corpus is calibrated to is
-# untouched by construction rather than by a re-tuning nobody would re-check.
+# Chance that a non-preamble prose BLOCK is a display-math block instead. Math takes prose's
+# slot rather than being appended alongside it, drawn from the same jittered size
+# distribution, so the fence count and sizes, and the Shiki span density the corpus is
+# calibrated to, are untouched by construction.
MATH_BLOCK_PROB = 0.16
-# Chance that a sentence in a non-preamble prose block carries an inline expression. Inline math is
-# the common case in real chat replies, and it is the one that interleaves with text rather than
-# sitting in its own block, which is a different path through the markdown pipeline.
-#
-# Inline math is spent from the prose block's own character budget, for the same reason as above.
+# Chance that a sentence in a non-preamble prose block carries an inline expression. Inline
+# math is the common case in real chat replies and interleaves with text rather than sitting
+# in its own block, a different path through the markdown pipeline. Spent from the prose
+# block's own character budget.
INLINE_MATH_PROB = 0.22
-# The PREAMBLE stays pure prose, deliberately. Its whole job is to be the stretch that builds no
-# spans and holds 60 fps, so the onset of cost has somewhere to be visible against. Putting math in
-# it would give the preamble a rendering cost and destroy the only fence-free, math-free baseline
-# the film has.
+# The PREAMBLE stays pure prose, deliberately: its job is to be the stretch that builds no
+# spans and holds 60 fps, so the onset of cost has somewhere to be visible against.
@dataclass(frozen = True)
@@ -258,14 +238,14 @@ class Unit:
"""One assistant turn's worth of content."""
index: int
- kind: str # "reasoning" | "code"
- reasoning: str # goes out as delta.reasoning_content
- content: str # goes out as delta.content
+ kind: str
+ reasoning: str
+ content: str
chars: int
sha256: str
- #: Stored tool-call parts for this turn, in the shape the app renders. Not counted in `chars`:
- #: `chars` is the rung's size axis and must stay comparable with every earlier measurement,
- #: while a tool call's cost is a mount, not a character count. Reported separately.
+ #: Stored tool-call parts for this turn, in the shape the app renders. Not counted in `chars`,
+ #: which is the rung's size axis and must stay comparable with every earlier measurement,
+ #: while a tool call's cost is a mount.
tool_calls: tuple = ()
@property
@@ -310,8 +290,8 @@ class Unit:
size += len(b) + 2
return "\n\n".join(out)
- # Keep the unit's own reasoning/content split, so a clipped turn has the same shape as a
- # whole one and the smallest rung is not accidentally all-visible or all-reasoning.
+ # Keep the unit's own reasoning/content split, so a clipped turn has the same shape as a whole
+ # one and the smallest rung is not accidentally all-visible or all-reasoning.
reasoning_budget = max(1, int(chars * (len(self.reasoning) / self.chars)))
reasoning = take(self.reasoning, reasoning_budget)
content = take(self.content, max(1, chars - len(reasoning)))
@@ -322,13 +302,13 @@ class Unit:
reasoning = reasoning,
content = content,
chars = len(text),
- # A DIFFERENT digest, deliberately. A clipped unit is not the frozen unit and must not
- # be checked against its hash or reported under it.
+ # A DIFFERENT digest, deliberately: a clipped unit is not the frozen unit and must not be
+ # checked against its hash.
sha256 = "clip:"
+ hashlib.sha256(f"{self.sha256}\x00{chars}".encode("utf-8")).hexdigest(),
- # Tool calls survive clipping. They carry no `chars` weight, so dropping them would
- # silently remove a whole component from exactly the rung -- the smallest -- that every
- # growth ratio is taken against.
+ # Tool calls survive clipping: they carry no `chars` weight, so dropping them would silently
+ # remove a whole component from exactly the rung, the smallest, that every growth ratio is
+ # taken against.
tool_calls = self.tool_calls,
)
@@ -477,23 +457,20 @@ def _fence(
size = len(lines[0]) + 1
i = 0
# SHORT identifiers and dense punctuation, because the density target is a SPAN count, not a
- # character count. Long descriptive names are cheap to highlight per character: the field
- # capture ran at 5.6 characters per span, and a first version of this generator with
- # `cumulative_buffer_0007_12`-style names measured 10.1, i.e. half the highlighter work per
- # character of the content it is standing in for. Real code is mostly operators.
+ # character count: the field capture ran at 5.6 characters per span, and a first generator
+ # with `cumulative_buffer_0007_12`-style names measured 10.1, half the work per character.
r = rng.randint
def v() -> str:
- # A MIX of short and long names. All-short measured 3.95 characters per span against the
- # field's 5.6 and all-long measured 10.1, so neither extreme stands in for real code. The
- # mix is the tuning knob for the offline proxy; the number that actually gets reported is
- # the span density MEASURED in the DOM at run time, because Shiki merges adjacent
- # same-scope tokens and no offline count can predict that exactly.
+ # A MIX of short and long names: all-short measured 3.95 characters per span against the
+ # field's 5.6 and all-long 10.1, so neither extreme stands in for real code. The mix tunes the
+ # offline proxy; the reported number is the span density MEASURED in the DOM at run time,
+ # because Shiki merges adjacent same-scope tokens.
return rng.choice(_SHORT) if rng.random() < 0.55 else rng.choice(_NOUNS)
while size < target:
- # ONE salted identifier per line is all the uniqueness Shiki's source-keyed cache needs,
- # and it leaves the rest of the line free to be dense.
+ # ONE salted identifier per line is all the uniqueness Shiki's source-keyed cache needs, and
+ # it leaves the rest of the line free to be dense.
u = f"{rng.choice(_SHORT)}{salt}{i}"
if lang == "python":
line = (
@@ -553,9 +530,9 @@ def _body(rng: random.Random, target: int, salt: str, *, preamble: bool) -> str:
parts.append(head)
size += len(head) + 2
while size < target:
- # Math takes the prose slot rather than being added alongside it, and is drawn from the
- # same size distribution, so the fence blocks keep their count and their sizes and the
- # span-density calibration in the module docstring still holds.
+ # Math takes the prose slot rather than being added alongside it, drawn from the same size
+ # distribution, so the fence blocks keep their count and sizes and the span-density
+ # calibration still holds.
want = _jitter(rng, PROSE_CHARS)
if rng.random() < MATH_BLOCK_PROB:
p = _math_block(rng, want, f"{salt}{len(parts)}")
@@ -674,11 +651,10 @@ def freeze(
"""Write `units.jsonl` and `manifest.json`. Run deliberately; never on a benchmark run."""
out_dir.mkdir(parents = True, exist_ok = True)
shipped = units_for_chars(max_chars, seed)
- # The manifest covers every unit any rung can reach, including the ones whose text is not
- # shipped, so a regenerated unit at the 1M rung is still checked byte for byte. SIZED FROM THE
- # LADDER, by `manifest_unit_count`, rather than from a bare character budget: the streamed turn
- # and its follow-ups live PAST the seeded prefix, and a manifest that stops where the prefix
- # stops leaves them nothing to point at.
+ # The manifest covers every unit any rung can reach, including ones whose text is not shipped,
+ # so a regenerated unit at 1M is still checked byte for byte. SIZED FROM THE LADDER by
+ # `manifest_unit_count` rather than from a character budget: the streamed turn and its
+ # follow-ups live PAST the seeded prefix.
all_units = [generate_unit(i, seed) for i in range(manifest_unit_count(seed))]
with (out_dir / "units.jsonl").open("w", encoding = "utf-8") as fh:
for u in shipped:
@@ -742,9 +718,8 @@ class Corpus:
@classmethod
def load(cls, frozen_dir: Optional[Path] = None) -> "Corpus":
- # Through the resource loader, NOT a filesystem path. Inside `studiobench.pyz` the frozen
- # corpus is a zip member and every Path built from __file__ points at something that
- # cannot exist; the built artifact's own --doctor is what caught it.
+ # Through the resource loader, NOT a filesystem path: inside `studiobench.pyz` the frozen
+ # corpus is a zip member and every Path built from __file__ points at nothing.
from ..runtime import resources
if frozen_dir is not None:
@@ -833,42 +808,37 @@ RUNGS: dict[str, int] = {
"1M": 1_000_000,
}
-# The PROVISIONAL ratio used to size the corpus before anything has been tokenised. It is never
-# reported: `chars_per_token` in every row is the MEASURED one, and it is measured per rung.
+# The PROVISIONAL ratio used to size the corpus before anything has been tokenised. Never
+# reported: `chars_per_token` in every row is the MEASURED one, per rung.
PROVISIONAL_CHARS_PER_TOKEN = 4.0
-# How much of the last turn STREAMS, at every rung. See the long note in `plan_rung`.
-#
-# Sized from the film, not chosen round: the quick scene's during-generation slots occupy the
-# first 16 s and the first after-generation slot opens at 22.5 s, so the stream has to finish
-# inside roughly 20 s. At the field cadence of 24 characters every 73 ms -- 328.8 chars/s -- that
-# is about 6,500 characters. Rounded DOWN so the stream reliably drains before the first action
-# that claims the reply is complete, rather than finishing exactly as it starts.
+# How much of the last turn STREAMS, at every rung (see the long note in `plan_rung`). Sized
+# from the film: the quick scene's first after-generation slot opens at 22.5 s, so the stream
+# must finish inside roughly 20 s, which at the field cadence of 328.8 chars/s is about 6,500
+# characters. Rounded DOWN so the stream drains before the first action that claims the reply
+# is complete.
STREAM_TAIL_CHARS = 6_000
-# How many turns actually stream per cell: the opening one plus the follow-ups the `send_turn`
-# action sends mid-film. They SHARE the budget above rather than each getting it, so three turns
-# and one turn take the same wall clock. Three because the corpus alternates reasoning-heavy and
-# code-heavy units, so three consecutive turns are guaranteed to cover both kinds.
+# How many turns stream per cell: the opening one plus the `send_turn` follow-ups. They SHARE
+# the budget above rather than each getting it, so three turns and one turn take the same wall
+# clock. Three because the corpus alternates reasoning-heavy and code-heavy units.
STREAM_TURNS = 3
-# Each follow-up's size. Small on purpose: the follow-ups exist to sample "what does a chunk cost
-# given what is already on screen" at two more points inside the cell, not to add mass.
+# Each follow-up's size. Small on purpose: the follow-ups sample "what does a chunk cost given
+# what is already on screen" at two more points, not to add mass.
FOLLOW_UP_CHARS = 1_500
-# Below this the rung streams ONCE. Three turns need about 9,000 characters of budget and the 1K
-# rung is 4,000 characters in total, so splitting it three ways produced an opening stream that
-# drained in four seconds -- shorter than the during-generation slots timed against it -- and a
-# rung 40% over its own target. A small rung is allowed to be a single exchange, which is also
-# what a 1K-token conversation actually is.
+# Below this the rung streams ONCE: three turns need about 9,000 characters of budget while
+# the 1K rung is 4,000 in total, so splitting it three ways gave an opening stream draining in
+# four seconds, shorter than the during-generation slots timed against it, and a rung 40% over
+# target.
MULTI_TURN_MIN_CHARS = 20_000
-# The largest characters-per-token the FROZEN CORPUS is sized for, which is not the same number as
-# the provisional ratio used to plan a rung. `plan_rung` takes a measured ratio, and a corpus sized
-# at exactly the provisional 4.0 has no material left the moment a machine measures 4.1. A quarter
-# of headroom costs a handful of extra manifest entries -- the text of an unshipped unit is
-# regenerated only when a rung actually reaches it -- and buys every ratio the seeder can plausibly
-# report. Past it `plan_rung` refuses rather than degrades.
+# The largest characters-per-token the FROZEN CORPUS is sized for, which is not the
+# provisional ratio used to plan a rung: a corpus sized at exactly 4.0 has no material left
+# the moment a machine measures 4.1. A quarter of headroom costs a handful of extra manifest
+# entries and buys every ratio the seeder can plausibly report; past it `plan_rung` refuses
+# rather than degrades.
MANIFEST_CHARS_PER_TOKEN = 5.0
@@ -902,8 +872,8 @@ class RungPlan:
target_chars: int
seeded_units: list[Unit] = field(default_factory = list)
streamed_unit: Optional[Unit] = None
- #: Further turns streamed DURING the film by the `send_turn` action. They come out of the same
- #: fixed streaming budget as the first, split between them, so more turns cost no wall clock.
+ #: Further turns streamed DURING the film by `send_turn`. They come out of the same fixed
+ #: streaming budget as the first, split between them, so more turns cost no wall clock.
follow_up_units: list[Unit] = field(default_factory = list)
@property
@@ -920,9 +890,9 @@ class RungPlan:
@property
def total_chars(self) -> int:
- # The follow-ups are part of the rung's mass: they are streamed INTO the thread during the
- # film and are on screen for most of it. Leaving them out understated every rung by the
- # follow-up budget, which at the 1K rung is most of the rung.
+ # The follow-ups are part of the rung's mass: they are streamed INTO the thread and are on
+ # screen for most of it, so leaving them out understated every rung by the follow-up budget,
+ # which at 1K is most of the rung.
return self.seeded_chars + self.streamed_chars + self.follow_up_chars
@@ -974,9 +944,9 @@ def dollarise(text: str, salt: str) -> str:
fenced = not fenced
out.append(line)
continue
- # Inside a fence, shell-shaped lines. Outside it, prices in prose. Both are what a real
- # answer contains, and they exercise different branches: the code-region scan has to
- # EXCLUDE the first and the currency heuristic has to escape the second.
+ # Inside a fence, shell-shaped lines; outside it, prices in prose. They exercise different
+ # branches: the code-region scan must EXCLUDE the first and the currency heuristic must escape
+ # the second.
if fenced and index % 11 == 0:
out.append(f"{line} # $HOME/{salt}{index} costs $1{index % 10}.99")
elif not fenced and line and index % 17 == 0:
@@ -1109,44 +1079,33 @@ def plan_rung(
target_chars = int(tokens * chars_per_token)
# THE STREAMED TAIL IS THE SAME SIZE AT EVERY RUNG, and the whole size ladder lives in the
- # seeded prefix. This is not tidiness, it is what makes the film mean anything above 10K.
- #
- # The scene is a FIXED-DURATION film whose slots are wall-clock times: three actions run
- # "during generation" and ten run "after the reply is complete". At the field's own cadence
- # of 24 characters every 73 ms, a streamed tail that grows with the rung takes 11 s at 1K,
- # 54 s at 10K, 354 s at 100K and 811 s at 1M -- against a film 135 s long. Above 10K the
- # stream outlasts the entire film, so every action labelled "after the reply is complete"
- # runs mid-generation. The run still completes and still prints a full table; the labels are
- # simply false, which is worse than a crash.
- #
- # Holding the tail constant also isolates the variable under investigation. The question is
- # what a streamed chunk costs AS A FUNCTION OF THE THREAD ALREADY ON SCREEN, and that needs
- # the chunk workload held fixed while the thread grows, not both moving together.
- # `stream_tail_chars` overrides the constant, and is the ONLY way to vary reply length in this
- # tool. The paragraphs above explain why the tail is pinned: the rung ladder exists to ask what
- # a chunk costs as a function of the THREAD, and that needs the chunk workload held fixed. The
- # consequence, which cost a whole investigation to rediscover, is that a cost scaling with the
- # length of the reply BEING STREAMED is constant across every rung and reads as a floor. Such a
- # mechanism is real and this ladder cannot see it at any effect size.
- #
- # Raising this makes the film's labels false in the way the note above describes, because the
- # stream then outlasts the after-generation slots. That is a trade a reply-length investigation
- # has to make deliberately, so it is a parameter and not a second constant, and `--assert-
- # liveness` on the resulting payload is how the cost of making it shows up.
+ # seeded prefix. The scene is a FIXED-DURATION film whose slots are wall-clock times, and at
+ # the field cadence a tail that grew with the rung would take 54 s at 10K, 354 s at 100K and
+ # 811 s at 1M against a 135 s film, so above 10K every action labelled "after the reply is
+ # complete" would run mid-generation while the run still printed a full table under false
+ # labels. Holding the tail constant also isolates the variable: what a streamed chunk costs
+ # AS A FUNCTION OF THE THREAD ALREADY ON SCREEN.
+ # The field's own cadence is 24 characters every 73 ms.
+ # `stream_tail_chars` overrides the constant and is the ONLY way to vary reply length in this
+ # tool. The consequence, which cost a whole investigation to rediscover, is that a cost
+ # scaling with the length of the reply BEING STREAMED is constant across every rung and reads
+ # as a floor: such a mechanism is real and this ladder cannot see it at any effect size.
+ # Raising this makes the film's labels false as described above, which is why it is a
+ # parameter rather than a second constant.
turns = STREAM_TURNS if target_chars >= MULTI_TURN_MIN_CHARS else 1
tail_budget = STREAM_TAIL_CHARS if stream_tail_chars is None else max(1, stream_tail_chars)
tail_target = min(tail_budget, target_chars) if stream_tail_chars is None else tail_budget
follow_budget = FOLLOW_UP_CHARS * (turns - 1)
seed_target = max(0, target_chars - tail_target - follow_budget)
- # Size the SEEDED PREFIX to the remainder, then trim its last unit to land on the target.
- # Growing it a whole unit at a time instead overshoots by up to one turn, which at the 1K
- # rung means 13,333 characters against a 4,000-character budget.
+ # Size the SEEDED PREFIX to the remainder, then trim its last unit to land on the target:
+ # growing it a whole unit at a time overshoots by up to one turn, which at 1K means 13,333
+ # characters against a 4,000-character budget.
last_index = max(e["index"] for e in corpus.manifest["units"])
try:
seeded = corpus.units_for_chars(seed_target) if seed_target > 0 else []
except KeyError as exc:
- # The prefix alone outgrew the manifest, before any streamed turn was even asked for.
+ # The prefix alone outgrew the manifest, before any streamed turn was asked for.
raise _too_small(rung, chars_per_token, last_index, "its seeded prefix alone") from exc
if seeded:
overshoot = sum(u.chars for u in seeded) - seed_target
@@ -1154,15 +1113,10 @@ def plan_rung(
seeded[-1] = seeded[-1].clipped_to(max(1, seeded[-1].chars - overshoot))
# The streamed turn is the next one after the prefix, clipped to the fixed tail, and each
- # follow-up is the one after that.
- #
- # NO CLAMP HERE, deliberately. This used to be `min(index, last_index)`, so a rung whose prefix
- # reached the end of the manifest silently got the final unit three times over: the opening
- # stream re-sent text the seeded prefix already held, both follow-ups were byte-identical, and
- # every fence in all three hit Shiki's source-keyed cache. Running off the end is a corpus that
- # is too small for the ladder, and the only honest answer is to say so and stop. `--freeze`
- # sizes the manifest from the ladder (`manifest_unit_count`) precisely so this cannot be
- # reached by any supported configuration.
+ # follow-up is the one after that. NO CLAMP HERE, deliberately: `min(index, last_index)`
+ # silently gave a rung whose prefix reached the end of the manifest the final unit three times
+ # over, so both follow-ups were byte-identical and every fence hit Shiki's source-keyed cache.
+ # `--freeze` sizes the manifest from the ladder precisely so this cannot be reached.
needed = len(seeded) + turns - 1
if needed > last_index:
raise _too_small(
@@ -1174,25 +1128,17 @@ def plan_rung(
)
# The streaming budget is SPLIT across the opening turn and the follow-ups, so a cell that
- # streams three times takes the same wall clock as one that streams once. Turns are taken from
- # consecutive corpus units, which alternate reasoning-heavy and code-heavy, so the follow-ups
- # exercise the `<think>` re-parse and the Streamdown path rather than repeating one of them.
- # The opening turn keeps the FULL tail budget and the follow-ups are extra. The three
- # during-generation slots are timed against the opening stream, so it has to outlast them;
- # splitting the budget between turns left it draining in four seconds and those slots then ran
- # against a finished reply while still being labelled "during generation".
+ # streams three times takes the same wall clock as one that streams once. Turns are
+ # consecutive corpus units, which alternate reasoning-heavy and code-heavy. The opening turn
+ # keeps the FULL tail budget and the follow-ups are extra: the three during-generation slots
+ # are timed against the opening stream, and splitting the budget left it draining in four
+ # seconds.
source = corpus.unit(len(seeded))
- # THE REQUESTED TAIL HAS TO BE DELIVERABLE. Keyed on `clipped_to`'s own early return
- # (`if chars >= self.chars: return self`) rather than on a tolerance, because the two ways a
- # tail lands under its budget are different in kind and only one of them is a defect: clipping
- # at a whole BLOCK boundary loses at most one block and is what keeps a prefix from ending
- # inside a fence, while exceeding the unit loses everything above it and is unbounded. A
- # percentage bound cannot tell those apart, and a bound nobody can derive is an unacknowledged
- # bias rather than a tolerance.
- #
- # The DEFAULT path is exempt on purpose: `STREAM_TAIL_CHARS` is a ceiling the small rungs are
- # legitimately under (the 1K rung is 4,000 characters in total, less than one tail), and that
- # shortfall is the rung being small rather than the corpus failing to answer.
+ # THE REQUESTED TAIL HAS TO BE DELIVERABLE, keyed on `clipped_to`'s own early return rather
+ # than a tolerance: clipping at a whole BLOCK boundary loses at most one block and is what
+ # keeps a prefix from ending inside a fence, while exceeding the unit loses everything above
+ # it and is unbounded, and a percentage bound cannot tell those apart. The DEFAULT path is
+ # exempt on purpose, since `STREAM_TAIL_CHARS` is a ceiling the small rungs are under.
if stream_tail_chars is not None and tail_target >= source.chars:
raise _tail_not_deliverable(
rung,
@@ -1205,7 +1151,7 @@ def plan_rung(
streamed = source.clipped_to(tail_target)
follow_ups = [corpus.unit(len(seeded) + i).clipped_to(FOLLOW_UP_CHARS) for i in range(1, turns)]
if dollars:
- # The streamed turns only. The seeded prefix is rendered once at mount and never
+ # The streamed turns only: the seeded prefix is rendered once at mount and never
# re-preprocessed, so dollars there would change the corpus without changing what the
# per-frame path is asked to do.
streamed = replace(
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py
index d0951d0e4ea..821cca875f3 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py
@@ -35,14 +35,12 @@ FENCE = re.compile(r"```.*?```", re.S)
DISPLAY = (re.compile(r"\$\$\n(.*?)\n\$\$", re.S), re.compile(r"\\\[\n(.*?)\n\\\]", re.S))
INLINE = (re.compile(r"\$ ([^$\n]+?) \$"), re.compile(r"\\\( (.+?) \\\)"))
-# The v1 corpus, measured before math was added.
-#
-# The tolerance is set from measurement rather than taste. With MATH_BLOCK_PROB at 0 the share is
-# 0.4754, i.e. EXACTLY v1, which is the substitution design working: inline math is spent from the
-# prose block's own budget and costs the fence share nothing. At the shipped 0.16 it reads 0.4779,
-# a drift of 0.0025. At 0.60 it reads 0.4689, a drift of 0.0065. So 0.005 leaves the shipped value
-# a factor of two of headroom while failing well before the parameter could be raised far enough to
-# re-weight the film.
+# The v1 corpus, measured before math was added. The tolerance is set from measurement: with
+# MATH_BLOCK_PROB at 0 the share is 0.4754, EXACTLY v1, which is the substitution design working,
+# since inline math is spent from the prose block's own budget. At the shipped 0.16 it reads
+# 0.4779 and at 0.60 it reads 0.4689, so 0.005 leaves the shipped value a factor of two of
+# headroom while failing well before the parameter could re-weight the film.
+# Drifts of 0.0025 and 0.0065 against v1.
V1_FENCE_SHARE = 0.4754
FENCE_SHARE_TOLERANCE = 0.005
@@ -65,10 +63,9 @@ def test_the_corpus_contains_math_at_all():
def test_both_delimiter_families_are_present():
- # `$...$` is what remark-math consumes directly. `\(...\)` and `\[...\]` reach the renderer
- # ONLY if preprocessLaTeX rewrites them first, so a corpus carrying only the first kind
- # exercises the renderer while leaving the preprocessor uncovered, which is the situation this
- # corpus version exists to end.
+ # `$...$` is what remark-math consumes directly. `\(...\)` and `\[...\]` reach the renderer ONLY
+ # if preprocessLaTeX rewrites them first, so a corpus carrying only the first kind leaves the
+ # preprocessor uncovered.
joined = "\n".join(_texts())
assert joined.count("$$") > 0
assert joined.count("\\[") > 0
@@ -76,8 +73,8 @@ def test_both_delimiter_families_are_present():
def test_math_is_spread_across_the_thread_not_pooled_into_one_turn():
- # A single maths-heavy turn measures one large KaTeX tree. The cost this is meant to expose
- # accumulates over a long thread, so it has to be present in most turns.
+ # A single maths-heavy turn measures one large KaTeX tree; the cost this exposes accumulates over
+ # a long thread, so it has to be present in most turns.
texts = _texts()
with_math = [t for t in texts if _count(t, DISPLAY) or _count(t, INLINE)]
assert len(with_math) >= max(2, int(len(texts) * 0.8))
@@ -95,10 +92,10 @@ def test_there_is_both_display_and_inline_math():
def test_the_fence_share_is_what_it_was_before_math_existed():
- # The span-density calibration in the module docstring is a statement about how much of the
- # corpus is fenced code. Math takes the PROSE slot and is drawn from the prose size
- # distribution precisely so this number does not move; if it has moved, the 5.6 chars/span
- # target no longer describes this film and the docstring is lying.
+ # The span-density calibration in the module docstring is a statement about how much of the corpus
+ # is fenced code. Math takes the PROSE slot and is drawn from the prose size distribution
+ # precisely so this number does not move; if it has moved, the 5.6 chars/span target no longer
+ # describes this film.
texts = _texts()
total = sum(len(t) for t in texts)
fenced = sum(len(m.group(0)) for t in texts for m in FENCE.finditer(t))
@@ -106,9 +103,9 @@ def test_the_fence_share_is_what_it_was_before_math_existed():
def test_the_preamble_stays_free_of_math_and_fences():
- # The field capture held a flat 60 fps and exactly zero spans for its first 33,348 characters.
- # The preamble stands in for that. Give it math and the film loses the one stretch against
- # which the onset of cost is visible, and every rung starts with a rendering cost instead.
+ # The field capture held a flat 60 fps and exactly zero spans for its first 33,348 characters, and
+ # the preamble stands in for that. Give it math and the film loses the one stretch against which
+ # the onset of cost is visible.
for index in range(4):
unit = generate_unit(index)
head = unit.reasoning[: int(len(unit.reasoning) * 0.20)]
@@ -119,8 +116,8 @@ def test_the_preamble_stays_free_of_math_and_fences():
def test_prose_without_the_math_flag_has_none():
- # `_prose` is also what builds tool arguments and results, where a stray dollar sign would be
- # rendered by a different component than the one under test.
+ # `_prose` also builds tool arguments and results, where a stray dollar sign would be rendered by
+ # a different component than the one under test.
import random
text = _prose(random.Random(7), 4_000, "x")
@@ -168,8 +165,8 @@ def test_the_shipped_corpus_matches_the_generator_byte_for_byte():
def test_the_manifest_records_the_math_parameters():
- # The corpus hash covers the generator's parameters as well as its bytes, so two corpora with
- # the same text and different declared densities cannot compare as identical.
+ # The corpus hash covers the generator's parameters as well as its bytes, so two corpora with the
+ # same text and different declared densities cannot compare as identical.
manifest = Corpus.load().manifest
assert "math_block_prob" in manifest
assert "inline_math_prob" in manifest
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py
index 288d98a0a1f..6af622d701e 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py
@@ -37,11 +37,11 @@ from studiobench.analysis import parity as P # noqa: E402
PARITY_JS = Path(__file__).resolve().parents[2] / "scene" / "parity.js"
-# A DOM shim, not a DOM. `signature()` touches exactly six things on an element, so those six are
-# what the harness provides; anything richer would be a second browser to keep correct. `capture()`
-# is deliberately NOT exercised here because it needs querySelectorAll, getComputedStyle and the
-# real selector adapter -- that is the live null and spike controls' job, and pretending to cover
-# it with a mock is how a test suite comes to assert its own fixtures.
+# A DOM shim, not a DOM: `signature()` touches exactly six things on an element, so those six
+# are what the harness provides; anything richer would be a second browser to keep correct.
+# `capture()` is deliberately NOT exercised here because it needs querySelectorAll,
+# getComputedStyle and the real selector adapter, which is the live null and spike controls'
+# job.
HARNESS_JS = r"""
const fs = require("fs");
const src = fs.readFileSync(process.argv[2], "utf8");
@@ -148,8 +148,8 @@ def test_relative_and_absolute_times_collapse():
def test_backend_minted_uuids_collapse():
- # The volatile that made the FIRST null control fail on all eighteen actions: every message
- # root carries `data-message-id`, and the two arms are two installs with two databases.
+ # The volatile that made the FIRST null control fail on all eighteen actions: every message root
+ # carries `data-message-id`, and the two arms are two installs with two databases.
a, b = norm_text(
"id 71ad5735-ede4-464d-a36b-44309ef67624", "id f44017dd-f5f7-45dd-9f53-475c115e61ac"
)
@@ -181,12 +181,11 @@ def test_urls_lose_their_origin_but_keep_their_path():
assert urls[3] == "#BLOB" and urls[4].startswith("#DATA:"), urls
-# ── the normaliser: things that MUST SURVIVE ────────────────────────
-#
# Every test above widens the set of things the digest cannot see. These are the counterweight:
# a normaliser that erased them would pass a null control perfectly and detect nothing.
+# ── the normaliser: things that MUST SURVIVE ────────────────────────
def test_a_bare_number_is_not_a_duration():
a, b = norm_text("3 files changed", "4 files changed")
assert a != b, (a, b)
@@ -237,8 +236,8 @@ def test_adding_or_removing_a_boolean_attribute_moves_the_signature():
def test_a_volatile_attribute_keeps_its_presence_even_though_its_value_is_dropped():
- # Dropping the value must not drop the fact that the attribute is there: an element that
- # gains an `id` has changed, even though which id it gained is noise.
+ # Dropping the value must not drop the fact that the attribute is there: an element that gains
+ # an `id` has changed, even though which id it gained is noise.
plain, with_id = sigs(
{"tag": "div", "attrs": {}}, {"tag": "div", "attrs": {"id": "radix-:r1a:"}}
)
@@ -286,8 +285,8 @@ def test_added_and_removed_elements_move_the_signature():
def test_reordered_siblings_move_the_signature():
- # Two elements with identical content in the other order. A digest built from a SET rather
- # than a sequence would read these as equal, and a list that renders backwards is a real bug.
+ # Two elements with identical content in the other order. A digest built from a SET rather than
+ # a sequence would read these as equal, and a list that renders backwards is a real bug.
one, two = sigs(
{
"tag": "ul",
@@ -311,8 +310,8 @@ def test_nesting_moves_the_signature():
def test_attribute_order_does_not_move_the_signature():
- # React can emit attributes in either order for the same render. Sorting them is what makes
- # the digest a property of the DOM rather than of the serialiser.
+ # React can emit attributes in either order for the same render; sorting them makes the digest a
+ # property of the DOM rather than of the serialiser.
one, two = sigs(
{"tag": "div", "attrs": {"class": "a", "data-state": "open"}},
{"tag": "div", "attrs": {"data-state": "open", "class": "a"}},
@@ -337,8 +336,7 @@ def test_the_depth_cap_leaves_a_visible_marker():
def test_content_below_the_depth_cap_is_not_compared():
- # The honest statement of the limit: past 40 levels the digest stops looking, and this test
- # records that as a KNOWN hole rather than leaving somebody to discover it.
+ # The honest statement of the limit: past 40 levels the digest stops looking, recorded here as a KNOWN hole.
def wrap(inner, n):
for _ in range(n):
inner = {"tag": "div", "children": [inner]}
@@ -385,8 +383,8 @@ def test_identical_captures_match():
def test_a_failed_capture_is_never_a_match():
- # The single most dangerous confusion in the whole instrument: a capture that threw and a
- # capture that agreed both produce no complaint unless they are told apart here.
+ # The single most dangerous confusion in the instrument: a capture that threw and a capture that
+ # agreed both produce no complaint unless they are told apart here.
failed = {"parity_attempted": False, "reason": "threadRoot is not a function"}
got = P.compare(failed, capture())
assert got["verdict"] == P.NOT_COMPARABLE
@@ -395,7 +393,7 @@ def test_a_failed_capture_is_never_a_match():
def test_two_different_roots_are_not_comparable():
- # A body-root capture carries the sidebar and its relative timestamps. Comparing it with a
+ # A body-root capture carries the sidebar and its relative timestamps; comparing it with a
# thread-root one produces two plausible hashes and a meaningless verdict.
got = P.compare(capture(root = "thread"), capture(root = "body"))
assert got["verdict"] == P.NOT_COMPARABLE
@@ -439,7 +437,7 @@ def test_an_added_message_is_localised_as_one_sided():
def test_an_overlay_that_changes_without_changing_count_is_still_localised():
# The bug this pins: comparing only the NUMBER of overlays passes an open menu whose contents
- # were rewritten, which is exactly the popover regression the overlay walk was added for.
+ # were rewritten, which is the popover regression the overlay walk was added for.
one = capture("aaaa", overlays = [{"sel": '[role="menu"]', "digest": "o1", "chars": 40}])
two = capture("zzzz", overlays = [{"sel": '[role="menu"]', "digest": "o2", "chars": 44}])
got = P.compare(one, two)
@@ -448,7 +446,7 @@ def test_an_overlay_that_changes_without_changing_count_is_still_localised():
def test_an_overlay_change_alone_is_a_difference():
# THE FALSE NEGATIVE THE SPIKE CONTROL FOUND. An overlay lives outside the thread root, so a
- # menu that mounts when it should not leaves the whole-thread digest untouched. Testing only
+ # menu that mounts when it should not leaves the whole-thread digest untouched; testing only
# that digest made the entire overlay walk unreachable and reported a clean pass.
one = capture("aaaa", overlays = [])
two = capture("aaaa", overlays = [{"sel": '[role="menu"]', "digest": "o1", "chars": 40}])
@@ -459,7 +457,7 @@ def test_an_overlay_change_alone_is_a_difference():
def test_a_message_change_alone_is_a_difference():
# The same shape one level down: if a per-message digest moves while the whole-thread digest
- # somehow does not, the pair still differs. Belt and braces on the aggregate hash.
+ # somehow does not, the pair still differs.
moved = capture(
"aaaa",
messages = [
@@ -471,7 +469,7 @@ def test_a_message_change_alone_is_a_difference():
def test_a_difference_outside_every_message_is_reported_as_such():
- # An empty `moved` list would read as "nothing differs" next to a DIFFER verdict.
+ # An empty `moved` list would read as 'nothing differs' next to a DIFFER verdict.
got = P.compare(capture("aaaa"), capture("zzzz"))
assert got["verdict"] == P.DIFFER
assert got["moved"] and "scaffolding" in got["moved"][0]
@@ -508,23 +506,24 @@ def test_mutation_detected_does_not_claim_a_detection_it_did_not_make():
def test_an_action_that_never_ran_is_not_a_matching_surface():
- # MEASURED, not imagined: on a 100K fast-tier null control, five of the eighteen actions did
- # not run on either arm -- no attachments button, no Copy button, a missed slot. The window
- # still closes and the digest is still captured, so both arms agreed and `image_upload` was
- # reported as a stable, matching surface. Nobody had opened it.
+ # MEASURED, not imagined: on a 100K fast-tier null control, five of eighteen actions did not run
+ # on either arm (no attachments button, no Copy button, a missed slot). The window still closes
+ # and the digest is still captured, so both arms agreed and `image_upload` was reported as a
+ # stable, matching surface that nobody had opened.
idle = {"ran": False, "reason": "no visible attachments button", "parity": capture()}
got = P.compare_rows(idle, idle)
assert got["verdict"] == P.NOT_EXERCISED
assert "nothing touched" in got["reason"]
# NEITHER arm ran it, so nobody opened the surface on either build and the only thing lost is
- # coverage. `one_sided` says so, and the caller needs it to keep that apart from the case below.
+ # coverage. `one_sided` says so, and the caller needs it to keep that apart from the case
+ # below.
assert got["one_sided"] == ""
def test_an_action_only_one_arm_could_perform_is_named_as_such():
- # A control that stops opening leaves NO digest to differ: the arm that cannot reach it
- # records `ran: false` and the pair carries no comparison at all. Folding that into the
- # missed-slot case is how a button that no longer works reads as lost coverage.
+ # A control that stops opening leaves NO digest to differ: the arm that cannot reach it records
+ # `ran: false` and the pair carries no comparison. Folding that into the missed-slot case is
+ # how a button that no longer works reads as lost coverage.
idle = {"ran": False, "reason": "the control never became visible", "parity": capture()}
got = P.compare_rows({"ran": True, "parity": capture()}, idle)
assert got["verdict"] == P.NOT_EXERCISED
@@ -573,8 +572,8 @@ def test_an_action_that_differs_against_itself_is_derived_as_unstable():
def test_a_blind_action_is_counted_as_blind_and_not_as_stable():
- # An action whose digest could never be captured has an observation count of zero. Reporting
- # it as "stable" would be the instrument certifying a surface it never looked at.
+ # An action whose digest could never be captured has an observation count of zero, and reporting
+ # it as stable would be the instrument certifying a surface it never looked at.
got = P.derive_unstable(
[
("image_upload", {"verdict": P.NOT_COMPARABLE}),
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py
index 09c9a767940..76fe8b918ce 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py
@@ -68,13 +68,12 @@ from tests.studio.studiobench.fixture.selftest.test_studiobench_parity_digest im
run_js,
)
-# ── building a capture out of the SHIPPED signature ──────────────────
-#
-# Not a hand-written digest. The whole point of the node harness is that the thing under test is
-# `scene/parity.js` as it ships, so these fixtures are DOM trees and every digest below is produced
-# by walking them with the real `signature()`.
+# Building a capture out of the SHIPPED signature. Not a hand-written digest: the thing under
+# test is `scene/parity.js` as it ships, so these fixtures are DOM trees and every digest is
+# produced by walking them with the real `signature()`.
+# ── building a capture out of the SHIPPED signature ──────────────────
def message(
index: int,
*,
@@ -141,8 +140,8 @@ def capture(tree: dict, *, streaming_fields: bool = True) -> dict:
"""
messages = tree["children"][0]["children"]
overlays = tree.get("_overlays") or []
- # `elide` marks EVERY message, which is what `capture()` in parity.js does: the scaffold has to
- # be the same walk on both arms, and which message is in flight is not.
+ # `elide` marks EVERY message, as `capture()` in parity.js does: the scaffold has to be the same
+ # walk on both arms, and which message is in flight is not.
scaffold_tree = _mark_elided(tree)
got = run_js(
{
@@ -187,10 +186,10 @@ def capture(tree: dict, *, streaming_fields: bool = True) -> dict:
return out
-# The body of the reply, at four points in one stream. Not four different documents: the same one,
-# with the trailing construct repaired differently at each point, which is what the renderer
-# actually does. The `title` is the KaTeX parse error rehype-katex writes while the formula will
-# not parse, and its character offset moves with every arriving character.
+# The body of the reply at four points in one stream: the same document with the trailing
+# construct repaired differently at each point, which is what the renderer does. The `title` is
+# the KaTeX parse error rehype-katex writes while the formula will not parse, and its character
+# offset moves with every arriving character.
def streamed_body(chars: int) -> list:
text = "The bounded shard coalesces the retained layout, except that the fibre stays inter"
return [
@@ -226,7 +225,7 @@ def streaming_arm(
)
-#: Four points in one stream. Adjacent pairs are what two arms one paint apart look like.
+#:Four points in one stream. Adjacent pairs are what two arms one paint apart look like.
STREAM_POINTS = (12, 24, 48, 81)
@@ -242,8 +241,7 @@ def test_the_same_document_at_two_points_in_one_stream_moves_the_raw_digest():
"""
a, b = streaming_arm(24), streaming_arm(48)
assert a["digest"] != b["digest"], "no drift to fix; the fixture is not reproducing the defect"
- # ...and the settled thread is byte-identical, which is what makes it a false alarm rather
- # than a finding.
+ # ...and the settled thread is byte-identical, which makes it a false alarm rather than a finding.
assert a["digest_scaffold"] == b["digest_scaffold"]
@@ -270,9 +268,9 @@ def test_two_genuinely_different_documents_still_differ_while_a_reply_streams():
def test_a_real_difference_survives_the_streamed_message_drifting_at_the_same_time():
- # The realistic case: the arms are at different points in the stream AND something real
- # changed. Excusing the first must not excuse the second, and the streamed message must not
- # appear in `moved` and drown the finding.
+ # The realistic case: the arms are at different points in the stream AND something real changed.
+ # Excusing the first must not excuse the second, and the streamed message must not appear in
+ # `moved` and drown the finding.
got = P.compare(
streaming_arm(24, settled_body = "settled text"),
streaming_arm(81, settled_body = "settled text, rewritten"),
@@ -300,7 +298,7 @@ def test_the_null_score_is_zero():
differing = [r for r in results if r["verdict"] == P.DIFFER]
assert len(results) == 6
assert not differing, f"NULL SCORE {len(differing)}/{len(results)}: {differing}"
- # And they are refused, not passed. A null that reports MATCH on a pair whose digests plainly
+ # And they are refused, not passed: a null reporting MATCH on a pair whose digests plainly
# disagree would be the instrument certifying a surface it could not look at.
assert all(r["verdict"] == P.NOT_COMPARABLE for r in results)
@@ -314,15 +312,15 @@ def test_the_null_battery_scores_the_old_instrument_too():
"""
before = null_battery(streaming_fields = False)
assert all(r["verdict"] == P.DIFFER for r in before), before
- # Every one of them localised to the streamed message and to nothing else, which is what made
- # them read as a UI change rather than as a clock.
+ # Every one localised to the streamed message and nothing else, which is what made them read as
+ # a UI change rather than a clock.
for r in before:
assert [m for m in r["moved"] if m.startswith("msg2(")] == r["moved"], r["moved"]
def test_two_arms_that_landed_on_the_same_point_in_the_stream_still_match():
- # The coverage this must NOT cost. Two arms at the same point serialise identically, which is
- # exactly the claim this mode makes, so it stays a pass.
+ # The coverage this must NOT cost: two arms at the same point serialise identically, which is
+ # exactly the claim this mode makes.
got = P.compare(streaming_arm(24), streaming_arm(24))
assert got["verdict"] == P.MATCH, got
assert got["in_flight"] == [2]
@@ -332,21 +330,20 @@ def test_a_settled_thread_is_scored_exactly_as_it_was():
settled = capture(thread([message(0, role = "user", body = "hi"), message(1)]))
assert settled["in_flight"] == [] and settled["streaming"] is False
assert P.compare(settled, settled)["verdict"] == P.MATCH
- # The scaffold is a SHORTER walk than the whole thread, not an equal one, and it has to be:
- # every message is elided from it whether or not anything was streaming. What makes the reading
- # complete again is the per-message rows, and the mutant battery is what shows they do.
+ # The scaffold is a SHORTER walk than the whole thread and has to be, since every message is
+ # elided whether or not anything was streaming. The per-message rows are what make the reading
+ # complete again, and the mutant battery is what shows they do.
assert settled["chars_scaffold"] < settled["chars"]
-# ── 3. the mutant score ──────────────────────────────────────────────
-#
-# Every mutant is a real, visible rendering difference, injected WHILE A REPLY IS IN FLIGHT and at
-# the SAME point in the stream on both arms, so the only thing the comparison can be reacting to is
-# the mutation. A normaliser that reports "identical" without ever having been shown a difference
-# is worthless, and one that stopped seeing these because a stream happened to be running would be
-# worse than the drift it was written to fix.
+# The mutant score. Every mutant is a real, visible rendering difference injected WHILE A REPLY
+# IS IN FLIGHT at the SAME point in the stream on both arms, so the comparison can only be
+# reacting to the mutation. A normaliser never shown a difference is worthless, and one that
+# stopped seeing these because a stream happened to be running would be worse than the drift it
+# was written to fix.
+# ── 3. the mutant score ──────────────────────────────────────────────
def mutants() -> list[tuple[str, dict, dict]]:
at = 24
@@ -428,8 +425,8 @@ def mutants() -> list[tuple[str, dict, dict]]:
)
)
# An overlay lives outside the thread root, so it has its own walk and its own way of going
- # unnoticed. Both shapes, because a menu that mounts and a menu that is rewritten are
- # different regressions.
+ # unnoticed. Both shapes, because a menu that mounts and a menu that is rewritten are different
+ # regressions.
menu = {
"sel": '[role="menu"]',
"tree": {
@@ -474,17 +471,16 @@ def test_the_mutant_score_is_total():
def test_no_mutant_is_ever_reported_as_a_match():
- # The weaker bar, held separately, because it is the one that decides whether a change can ship
- # green. DIFFER is the outcome that names the change; NOT COMPARABLE is the outcome that
- # refuses to say; only MATCH lets it through, and nothing here may reach it.
+ # The weaker bar, held separately, because it decides whether a change can ship green: DIFFER
+ # names the change, NOT COMPARABLE refuses to say, and only MATCH lets it through.
for name, before, after in mutants():
assert P.compare(before, after)["verdict"] != P.MATCH, name
def test_the_mutant_score_is_unchanged_by_the_streaming_fields():
# The same battery scored as the OLD instrument would score it. Identical, which is the claim:
- # nothing that used to be caught stopped being caught. If this ever diverges from the test
- # above, the elision has started hiding something.
+ # nothing that used to be caught stopped being caught. Divergence from the test above means the
+ # elision has started hiding something.
for name, before, after in mutants():
old_before = {
k: v
@@ -573,7 +569,7 @@ def test_a_real_change_inside_the_streamed_message_is_refused_not_caught():
def test_a_message_that_is_in_flight_on_one_arm_only_is_still_excused():
- # The ordinary case, and it must not read as a build difference: one arm finished the reply
+ # The ordinary case, which must not read as a build difference: one arm finished the reply
# before its digest was taken and the other did not.
a = streaming_arm(81)
b = capture(
@@ -589,15 +585,14 @@ def test_a_message_that_is_in_flight_on_one_arm_only_is_still_excused():
def test_a_message_that_vanished_is_never_excused_by_being_in_flight():
- # The elision withholds a subtree, never an element. A streamed message that is absent on one
- # arm is a lost message, and no amount of "it was still arriving" makes that not a difference.
+ # The elision withholds a subtree, never an element: a streamed message absent on one arm is a
+ # lost message, and 'it was still arriving' does not make that not a difference.
a = streaming_arm(24)
b = capture(thread([message(0, role = "user", body = "the prompt"), message(1)]))
got = P.compare(a, b)
assert got["verdict"] == P.DIFFER
# Caught one level earlier than `localise`, by the mount-count check: neither arm is windowing
- # and they mounted different numbers of messages, which is a lost conversation and is reported
- # as such rather than as a row that moved.
+ # and they mounted different numbers of messages, which is a lost conversation.
assert "different numbers of messages (3 vs 2)" in got["reason"], got
@@ -814,7 +809,7 @@ def test_the_streamed_row_itself_is_still_withheld_when_the_probe_is_blind():
def test_a_user_row_flagged_in_flight_by_the_other_arm_is_still_withheld():
"""The arm that COULD place its stream is believed about which rows have no defined moment."""
base = _blind_arm(24)
- # The non-blind arm names index 0 as in flight. Whatever role it carries, its digest is a point
+ # The non-blind arm names index 0 as in flight; whatever role it carries, its digest is a point
# in a stream on that arm, so it is not a settled row.
treat = dict(_blind_arm(24, prompt = "a different prompt"), in_flight = [0])
treat["in_flight_unplaced"] = True
@@ -824,8 +819,8 @@ def test_a_user_row_flagged_in_flight_by_the_other_arm_is_still_withheld():
def test_an_old_payload_without_the_streaming_fields_is_scored_as_it_always_was():
- # Not silently refused, and not silently excused: a capture recorded before the fields existed
- # carries no claim about a stream, and falls back to the digest it does carry.
+ # Not silently refused and not silently excused: a capture recorded before the fields existed
+ # carries no claim about a stream and falls back to the digest it does carry.
a = streaming_arm(24, streaming_fields = False)
b = streaming_arm(24, streaming_fields = False)
assert P.compare(a, b)["verdict"] == P.MATCH
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py
index cacc5a70cff..91f79483e25 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py
@@ -44,8 +44,8 @@ def reasoning_trigger(seconds: str) -> dict:
def test_a_duration_split_across_text_nodes_is_normalised():
- # The exact regression. Before the fix these two differed by one character and the digest
- # moved; the two arms are the same build and the only difference is wall clock.
+ # The exact regression. Before the fix these two differed by one character and the digest moved;
+ # the two arms are the same build and the only difference is wall clock.
assert sig(reasoning_trigger("3")) == sig(reasoning_trigger("2"))
@@ -77,8 +77,8 @@ def test_a_split_relative_time_is_normalised():
def test_a_bare_number_with_no_unit_still_moves_the_signature():
- # A message count, a row count, a badge. No time unit follows it, so it is content and the
- # digest has to see it change. This is the false-negative direction and it is the worse one.
+ # A message count, a row count, a badge. No time unit follows it, so it is content and the digest
+ # has to see it change. This is the false-negative direction and it is the worse one.
def badge(n: str) -> dict:
return {"tag": "span", "children": [{"tag": "b", "children": [n, " messages"]}]}
@@ -97,8 +97,8 @@ def test_an_element_boundary_still_breaks_a_text_run():
}
joined = {"tag": "div", "children": [{"tag": "span", "children": ["3", " seconds"]}]}
assert sig(split) != sig(joined)
- # And the genuinely-split-across-elements case keeps its number, because nothing there proves
- # the two spans are one rendered phrase.
+ # And the genuinely-split-across-elements case keeps its number, because nothing there proves the
+ # two spans are one rendered phrase.
other = {
"tag": "div",
"children": [
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py
index 3591b2e05cb..f6e59f0b266 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py
@@ -77,8 +77,8 @@ def test_the_tail_override_moves_the_reply_and_not_the_thread(corpus: Corpus):
for tail in (24_000, 96_000):
plan = plan_rung(corpus, "100K", stream_tail_chars = tail)
assert abs(plan.streamed_chars - tail) < tail * 0.1, (tail, plan.streamed_chars)
- # Within a few percent: the seeded prefix is trimmed to compensate, so the cell measures
- # a different SPLIT of the same total rather than a bigger thread.
+ # Within a few percent: the seeded prefix is trimmed to compensate, so the cell measures a different
+ # SPLIT of the same total rather than a bigger thread.
assert abs(plan.total_chars - base.total_chars) < base.total_chars * 0.05, (
tail,
plan.total_chars,
@@ -133,10 +133,10 @@ def test_dollars_reach_the_streamed_turn_and_nothing_else(corpus: Corpus):
plain = plan_rung(corpus, "100K", stream_tail_chars = 24_000)
salted = plan_rung(corpus, "100K", stream_tail_chars = 24_000, dollars = True)
assert _streamed_text(salted).count("$") > _streamed_text(plain).count("$")
- # The seeded prefix is rendered once at mount and never re-preprocessed, so dollars there
- # would change the corpus without changing what the per-frame path is asked to do. v2's own
- # math is expected there; what must not appear is anything THIS added, so the prefix has to
- # be byte-identical with the flag on and off.
+ # The seeded prefix is rendered once at mount and never re-preprocessed, so dollars there would
+ # change the corpus without changing what the per-frame path is asked to do. v2's own math is
+ # expected; what must not appear is anything THIS added, so the prefix has to be byte-identical
+ # with the flag on and off.
assert [(u.reasoning, u.content) for u in salted.seeded_units] == [
(u.reasoning, u.content) for u in plain.seeded_units
]
@@ -233,8 +233,7 @@ class _FakeKeyboard:
def press(self, key: str) -> None:
self.pressed.append(key)
if key == "Enter" and "one more" in self._page.filled:
- # Sending the throwaway turn starts a generation, which is what the own-turn path
- # then waits for and stops.
+ # Sending the throwaway turn starts a generation, which is what the own-turn path then waits for and stops.
self._page.running = True
@@ -314,8 +313,8 @@ def test_stop_refuses_to_truncate_the_cell_s_own_reply():
from studiobench.scene.actions import stop_generation
page = _FakePage(running = True)
- # 200 ms: too small to hold the throwaway turn at all, so the drain wait is zero and the
- # refusal below is the one this test is about rather than the budget check beside it.
+ # 200 ms: too small to hold the throwaway turn at all, so the drain wait is zero and the refusal
+ # below is the one this test is about rather than the budget check beside it.
result = stop_generation(_stop_ctx(page, budget_ms = 200))
assert result.ran is False, "a stop that would truncate the measured reply must not run"
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py
index 73e612dfe0a..faa8248f916 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py
@@ -82,10 +82,9 @@ def test_the_stream_drains_before_the_first_after_generation_slot():
def test_during_generation_slots_actually_fall_during_generation():
from studiobench.scene.schedule import SCENES
- # The SHORTEST stream on the ladder above 1K, since a slot has to be inside every rung's
- # stream to deserve the name.
- # The OPENING turn only: the follow-ups are sent later by `send_turn`, so a slot that has to
- # fall during generation has to fall inside the first stream, not inside their sum.
+ # The SHORTEST stream on the ladder above 1K, since a slot has to be inside every rung's stream to
+ # deserve the name. The OPENING turn only: the follow-ups are sent later by `send_turn`, so a slot
+ # that must fall during generation has to fall inside the first stream, not their sum.
shortest = min(p.streamed_chars for r, p in _plans().items() if r != "1K") / FIELD_CHARS_PER_SEC
for name, scene in SCENES.items():
during = [s for s in scene.slots if s.action == "scroll_during_generation"]
@@ -134,11 +133,10 @@ def test_the_ladder_is_strictly_increasing_in_seeded_mass():
# Actions that need a SETTLED reply. The action bar's More and Copy buttons are not rendered while
# a turn is streaming, select-all copies a moving target, and delete is hidden on a running
-# message. Each one reports an honest `NOT RUN` rather than a wrong number, which is why this was
-# invisible until somebody counted: on a 36 job sweep the fast film recorded
-# `message_menu: NOT RUN -- no More button` on 312 of 312 attempts, and the four actions it
-# silently stopped exercising include the one carrying the largest measured effect in this
-# codebase (the message menu, 1,164.9 ms to 60.7 ms across the merged campaign).
+# message. Each reports an honest `NOT RUN`, which is why this was invisible until somebody
+# counted: on a 36 job sweep the fast film recorded `message_menu: NOT RUN -- no More button` on
+# 312 of 312 attempts, and the four actions it silently stopped exercising include the one
+# carrying the largest measured effect in this codebase (1,164.9 ms to 60.7 ms).
SETTLED_ACTIONS = ("message_menu", "copy_markdown", "select_all_copy", "delete_message")
@@ -165,31 +163,24 @@ def test_settled_actions_open_after_the_follow_up_drains():
last_send = None
for slot in sorted(scene.slots, key = lambda s: s.t_start_ms):
if slot.action == "send_turn":
- # THE LATEST THE SEND CAN FIRE, not the earliest. The drain starts when the send
- # actually happens, and a send slot is a WINDOW: the action may legitimately begin
- # anywhere inside its own budget, which is precisely what it does when the slot
- # before it overran and pushed it.
- #
- # Measured from the start instead, this check reported 1,538 ms of margin on the
- # fast film where 38 ms existed, and CI then failed the way the arithmetic says it
- # must: `reasoning_toggle` overran its 3,500 ms budget by 934 ms, `send_turn` was
- # pushed 1,373 ms late but stayed inside its 1,500 ms budget so nothing recorded a
- # miss, the send-to-menu gap collapsed from a nominal 5,300 ms to an actual
- # 3,691 ms, and `message_menu` found a reply still streaming and recorded NOT RUN.
- # Every slot was individually within budget and the film was still unrunnable.
- #
- # This is the same defect the rest of this branch is about, in the check rather
- # than in the instrument: a quantity computed at a moment whose meaning is not the
- # one the reader assumes. A margin measured from the earliest possible send is not
- # a margin.
+ # THE LATEST THE SEND CAN FIRE, not the earliest. The drain starts when the send actually happens,
+ # and a send slot is a WINDOW: the action may legitimately begin anywhere inside its own budget,
+ # which is what it does when the slot before it overran.
+ # Measured from the start instead, this check reported 1,538 ms of margin on the fast film where
+ # 38 ms existed, and CI then failed the way the arithmetic says it must: `reasoning_toggle`
+ # overran its 3,500 ms budget by 934 ms, `send_turn` was pushed 1,373 ms late but stayed inside
+ # its own budget so nothing recorded a miss, the send-to-menu gap collapsed from a nominal 5,300
+ # ms to 3,691 ms, and `message_menu` found a reply still streaming and recorded NOT RUN. Every
+ # slot was individually within budget and the film was still unrunnable.
+ # Its own budget is 1,500 ms.
+ # The same defect the rest of this branch is about, in the check rather than the instrument: a
+ # quantity computed at a moment whose meaning is not the one the reader assumes.
last_send = slot.t_start_ms + slot.budget_ms
elif slot.action in SETTLED_ACTIONS and last_send is not None:
- # The slot's WINDOW, not its start. A slot may legitimately open a little before
- # the follow-up finishes and wait inside its own budget -- the quick film opens
- # `message_menu` at 4.5 s against a 4.56 s drain and it runs, because it has 3 s
- # of budget to wait in. What is fatal is a window that CLOSES before the reply
- # settles, which is what the first fast film did: 1.7 s gap plus 0.8 s budget
- # against a 4.56 s drain, so the button had not been rendered yet and never
- # would be inside that window.
+ # The slot's WINDOW, not its start. A slot may open a little before the follow-up finishes and
+ # wait inside its own budget: the quick film opens `message_menu` at 4.5 s against a 4.56 s drain
+ # and it runs, because it has 3 s to wait in. What is fatal is a window that CLOSES before the
+ # reply settles, which is what the first fast film did.
+ # A 1.7 s gap plus a 0.8 s budget against a 4.6 s drain.
window_end = (slot.t_start_ms + slot.budget_ms - last_send) / 1000.0
assert window_end >= drain_s, (name, slot.action, window_end, drain_s)
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py
index d6849dc58a4..0790e0370a8 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py
@@ -50,15 +50,15 @@ def test_every_surface_declares_a_reach_a_restore_and_a_root():
f"{surface.id} declares no reach, so it must also declare no restore: a surface that "
f"is reached by doing nothing cannot need undoing"
)
- # Anything that navigates or opens an overlay has to say how to get back, or the next
- # surface is reached from a state nobody declared.
+ # Anything that navigates or opens an overlay has to say how to get back, or the next surface is
+ # reached from a state nobody declared.
opens = any(step[0] in {"goto", "click", "fill", "press"} for step in surface.reach)
assert not opens or surface.restore, f"{surface.id} opens something and never closes it"
def test_every_surface_declares_a_settle_condition():
- # A sweep with no settle condition is a sweep that digests whatever is on screen when the
- # sleep expires, which is the one result this tool may not produce.
+ # A sweep with no settle condition digests whatever is on screen when the sleep expires, which
+ # is the one result this tool may not produce.
for surface in surfaces.surfaces():
assert surface.settle is not None, surface.id
assert isinstance(surface.settle, dict)
@@ -88,9 +88,8 @@ def test_every_step_uses_a_known_verb_with_the_right_arity():
def test_the_sweep_can_execute_every_verb_the_registry_uses():
- # The registry and the interpreter are in different files, and a verb added to one and not the
- # other fails only at run time, on the surface that uses it, as a reach failure that reads
- # like a broken selector.
+ # The registry and the interpreter are in different files, so a verb added to one and not the
+ # other fails only at run time, on the surface that uses it, and reads like a broken selector.
used = {
step[0] for s in surfaces.surfaces() for steps in (s.reach, s.restore) for step in steps
}
@@ -125,9 +124,9 @@ def test_the_registry_covers_every_major_group(group):
def test_the_settings_dialog_covers_every_shipped_tab():
- # The twelve panels are lazily imported chunks, so a tab that stopped rendering fails only
- # for the tab. Pinned here so a panel added to the app without a surface is visible as a
- # missing id rather than as a coverage figure that quietly stayed at 100%.
+ # The twelve panels are lazily imported chunks, so a tab that stopped rendering fails only for
+ # the tab. Pinned so a panel added without a surface shows up as a missing id rather than a
+ # coverage figure that quietly stayed at 100%.
shipped = {
"general",
"profile",
@@ -147,9 +146,8 @@ def test_the_settings_dialog_covers_every_shipped_tab():
def test_the_registry_covers_every_routed_path():
- # From app/router.tsx. `/` and `/settings` are not page surfaces -- both redirect on
- # beforeLoad -- and the two auth-flow routes are in KNOWN_UNCOVERED with the guard that
- # redirects them.
+ # From app/router.tsx. `/` and `/settings` are not page surfaces (both redirect on beforeLoad)
+ # and the two auth-flow routes are in KNOWN_UNCOVERED with the guard that redirects them.
routed = {
"/chat",
"/projects",
@@ -216,8 +214,8 @@ def test_the_surface_row_type_is_registered():
def test_a_surface_row_must_carry_its_reason():
- # The single defect this project cares most about: a check that cannot distinguish "did not
- # run" from "passed". A surface row without `reason` is exactly that row.
+ # The single defect this project cares most about: a check that cannot distinguish 'did not run'
+ # from 'passed'. A surface row without `reason` is exactly that row.
assert "reason" in ROW_REQUIRED["surface"]
assert "reached" in ROW_REQUIRED["surface"]
assert "parity" in ROW_REQUIRED["surface"]
@@ -261,14 +259,14 @@ def test_surface_rows_land_in_their_own_payload_section(tmp_path):
)
payload = assemble_rows(path)
assert len(payload["surfaces"]) == 1
- # NOT in unknown_rows. A new row type that lands there is carried through the whole report
+ # NOT in unknown_rows: a new row type that lands there is carried through the whole report
# without anyone noticing it is not being read.
assert payload["unknown_rows"] == []
def test_a_payload_carrying_surface_rows_has_no_bare_zeros():
# The schema bans a naked numeric zero, because a zero outside a measure is indistinguishable
- # from "we never ran that". Surface rows carry counts, so they have to attest.
+ # from 'we never ran that'. Surface rows carry counts, so they have to attest.
rows = surface_sweep.sweep(_FakePage(reach_ok = True), "http://x")[0]
validate_payload({"excluded_cells": [], "surfaces": rows})
@@ -318,8 +316,8 @@ def test_a_conditional_miss_is_counted_apart_from_a_hard_miss():
def test_a_volatile_surface_is_reached_but_not_counted_as_comparable():
- # Reached and volatile are different facts and the manifest has to keep them apart. Folding
- # them together is how a live memory gauge ends up quoted as a UI change.
+ # Reached and volatile are different facts the manifest must keep apart; folding them together
+ # is how a live memory gauge ends up quoted as a UI change.
entries = [_one(id = "a"), _one(id = "b", volatile = "shows a live memory gauge")]
rows = _rows_for(entries, {"a", "b"})
manifest = surface_sweep.build_manifest(rows, entries, {"scoped": True})
@@ -337,10 +335,9 @@ def test_every_volatile_surface_states_its_mechanism():
def test_the_measured_volatile_surfaces_are_declared():
- # Three consecutive sweeps against one Unsloth agreed on 44 of 53 surfaces. These are the ones
- # that did not, each with a mechanism established from source or from a text diff. Pinned so
- # a settle condition that is tightened later shows up as a flag that can now be dropped,
- # rather than as a flag nobody revisits.
+ # Three consecutive sweeps against one Unsloth agreed on 44 of 53 surfaces; these are the ones
+ # that did not, each with a mechanism established from source or a text diff. Pinned so a
+ # later-tightened settle condition shows up as a flag that can now be dropped.
declared = {s.id for s in surfaces.surfaces() if s.volatile}
assert {
"route:chat",
@@ -358,8 +355,8 @@ def test_the_measured_volatile_surfaces_are_declared():
def test_the_unexplained_movers_say_so_rather_than_offering_a_cause():
- # A plausible-sounding mechanism that has not been established is worse than an admitted gap:
- # it is the sentence somebody uses to wave away a real difference.
+ # A plausible-sounding mechanism that has not been established is worse than an admitted gap: it
+ # is the sentence somebody uses to wave away a real difference.
unexplained = [
s
for s in surfaces.surfaces()
@@ -397,9 +394,8 @@ def test_the_rendered_manifest_names_every_failure_and_every_known_gap():
def test_an_unscoped_sweep_says_so_in_the_rendered_manifest():
- # An unscoped digest is a whole-page reading taken forty times, and every one of them agrees
- # with every other for reasons that have nothing to do with the surfaces. That has to be at
- # the top of the artefact, not inferable from a field somebody might read.
+ # An unscoped digest is a whole-page reading taken forty times, every one agreeing for reasons
+ # that have nothing to do with the surfaces. That has to be at the top of the artefact.
entries = surfaces.surfaces()[:1]
manifest = surface_sweep.build_manifest(
_rows_for(entries, {entries[0].id}),
@@ -511,8 +507,8 @@ def test_a_reach_that_fails_records_a_reason_and_never_a_digest():
for row in interactive:
assert row["reached"] is False
assert "the reach failed" in row["reason"]
- # Never a digest it did not take. A parity block that claimed `attempted` here would make
- # a failed sweep read as a passing parity check downstream.
+ # Never a digest it did not take: a parity block claiming `attempted` here would make a failed
+ # sweep read as a passing parity check downstream.
assert row["parity"]["parity_attempted"] is False
@@ -559,8 +555,8 @@ def test_the_sweep_streams_rows_to_a_recorder_as_it_goes():
def test_the_sweep_returns_to_the_known_state_before_every_surface():
- # Not "after every surface": an after-only reset trusts each surface's own restore, and the
- # one that fails is the one whose restore did not run.
+ # Not 'after every surface': an after-only reset trusts each surface's own restore, and the one
+ # that fails is the one whose restore did not run.
page = _FakePage()
surface_sweep.sweep(page, "http://x")
assert surfaces.KNOWN_STATE_PATH == "/chat"
@@ -570,9 +566,9 @@ def test_the_sweep_returns_to_the_known_state_before_every_surface():
def test_the_sweep_takes_its_digest_through_parity_capture():
- # If surfaces.js ever grew its own DOM walk, surface digests and action digests would stop
- # being comparable -- and nothing downstream would notice, because both would still be hex
- # strings of the same length.
+ # If surfaces.js grew its own DOM walk, surface digests and action digests would stop being
+ # comparable and nothing downstream would notice, because both would still be hex strings of
+ # the same length.
text = (Path(__file__).resolve().parents[2] / "scene" / "surfaces.js").read_text(
encoding = "utf-8"
)
@@ -581,8 +577,8 @@ def test_the_sweep_takes_its_digest_through_parity_capture():
def test_surfaces_js_does_not_edit_the_shared_chat_adapter():
- # dom.js is the film's adapter and every action reads it. The surface layer restores the root
- # it moved, so a sweep cannot leave the film digesting the wrong element.
+ # dom.js is the film's adapter and every action reads it. The surface layer restores the root it
+ # moved, so a sweep cannot leave the film digesting the wrong element.
text = (Path(__file__).resolve().parents[2] / "scene" / "surfaces.js").read_text(
encoding = "utf-8"
)
diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py
index 135f0ef3312..657a5d26102 100644
--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py
+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py
@@ -40,10 +40,10 @@ def corpus() -> Corpus:
return Corpus.load()
-#: Every pair the axis is plausibly asked for, and what the corpus can actually do with it.
-#: `True` means deliverable, `False` means the request has to be refused. Derived by running the
-#: reply-axis test's own two assertions across the ladder, not chosen: every `False` here is a
-#: pair that silently under-delivered before the guard, and every `True` is one that did not.
+#: Every pair the axis is plausibly asked for, and what the corpus can actually do with it. `True`
+#: means deliverable, `False` means the request has to be refused. Derived by running the
+#: reply-axis test's own two assertions across the ladder, not chosen: every `False` is a pair that
+#: silently under-delivered before the guard.
MATRIX: list[tuple[str, int, bool]] = [
("1K", 24_000, False),
("1K", 96_000, False),
diff --git a/tests/studio/studiobench/instruments/coverage.py b/tests/studio/studiobench/instruments/coverage.py
index 26754e6b782..1eea1d12366 100644
--- a/tests/studio/studiobench/instruments/coverage.py
+++ b/tests/studio/studiobench/instruments/coverage.py
@@ -121,9 +121,8 @@ def _parse(result: dict[str, Any]) -> CoverageSnapshot:
ranges = fn.get("ranges") or []
if not ranges:
continue
- # With `detailed: false` there is one range per function and it
- # spans the function; even with block coverage the FIRST range is
- # the function-level one, so this stays correct either way.
+ # With `detailed: false` there is one range per function spanning the function; even with block
+ # coverage the FIRST range is the function-level one, so this stays correct either way.
head = ranges[0]
snap.functions.append(
FunctionCount(
@@ -288,19 +287,13 @@ def ambiguity(snapshot: CoverageSnapshot, name: str) -> int:
return len(snapshot.find(name))
-# ═══════════════════════════════════════════════════════════════════════════
# Harness adapter (INTERFACES.md section 3)
-# ═══════════════════════════════════════════════════════════════════════════
-#
-# Level 3, and every window this instrument touches is TIMING-VOID by
-# construction. Precise coverage keeps count-collecting bytecode alive, which
-# disables TurboFan and Maglev for the whole isolate, so the durations that
-# `tracing` reports in the same cell describe a program nobody ships.
-#
-# The payload therefore carries `timings_void: true` at both window and cell
-# level. That flag exists so the report layer can refuse to quote a duration
-# from this cell without having to know why. Only integers cross the boundary,
-# enforced by `assert_integers_only` on the way out rather than by convention.
+# Level 3, and every window this instrument touches is TIMING-VOID by construction: precise
+# coverage keeps count-collecting bytecode alive, which disables TurboFan and Maglev for the whole
+# isolate, so the durations `tracing` reports in the same cell describe a program nobody ships.
+# The payload therefore carries `timings_void: true` at both window and cell level, so the report
+# layer can refuse to quote a duration from this cell without knowing why. Only integers cross the
+# boundary, enforced by `assert_integers_only`.
import time # noqa: E402
from typing import Any # noqa: E402
@@ -338,10 +331,9 @@ class CoverageInstrument:
return
t0 = time.perf_counter()
try:
- # Started ONCE per cell, never per window. Restarting coverage
- # re-runs V8's DeoptimizeAll, which changes what gets compiled and
- # therefore what gets counted, so per-window restarts would make the
- # counts depend on the window boundaries.
+ # Started ONCE per cell, never per window: restarting coverage re-runs V8's DeoptimizeAll, which
+ # changes what gets compiled and therefore counted, so per-window restarts would make the counts
+ # depend on the window boundaries.
self.cov = PreciseCoverage(self.cdp)
self.cov.start()
except Exception as exc: # noqa: BLE001
@@ -394,8 +386,8 @@ class CoverageInstrument:
),
},
)
- # The boundary guard. A float here would be a category error, not a
- # rounding problem, so it raises rather than warns.
+ # The boundary guard. A float here would be a category error, not a rounding problem, so it raises
+ # rather than warns.
assert_integers_only(
{
k: v
diff --git a/tests/studio/studiobench/instruments/cpuprofile.py b/tests/studio/studiobench/instruments/cpuprofile.py
index a1553291c51..f883737f126 100644
--- a/tests/studio/studiobench/instruments/cpuprofile.py
+++ b/tests/studio/studiobench/instruments/cpuprofile.py
@@ -34,12 +34,10 @@ from typing import Any
from ..analysis import CellFailure
from ..analysis.cpuprofile import CallFrame, CpuProfile, Sample
-# V8's own default is 1000 us. 100 us resolves a sub-millisecond frame without
-# the sampler itself becoming the workload.
+# V8's own default is 1000 us. 100 us resolves a sub-millisecond frame without the sampler becoming the workload.
DEFAULT_SAMPLING_INTERVAL_US = 100
-# Minimum non-synthetic samples on BOTH sides before the two extraction routes
-# may be declared to agree or disagree.
+# Minimum non-synthetic samples on BOTH sides before the two extraction routes may be declared to agree or disagree.
MIN_JS_SAMPLES_FOR_COMPARISON = 200
@@ -160,13 +158,10 @@ def compare_with_trace_profile(
1 for s in p.samples if (f := p.nodes.get(s.node_id)) is not None and not f.is_synthetic
)
- # A comparison built on a handful of JS samples cannot distinguish the two
- # parsers from sampling noise. Saying "agrees" there would be worse than
- # useless and saying "disagrees" is a false alarm, so it declines to rule.
- # V8 has ONE CpuProfiler. If the trace carried the profiler categories, the
- # "standalone" profile IS the in-trace profile and comparing them is
- # comparing a thing with itself, which always agrees and therefore proves
- # nothing. Detected structurally rather than trusted to the caller.
+ # A comparison built on a handful of JS samples cannot distinguish the two parsers from sampling
+ # noise, so it declines to rule. And V8 has ONE CpuProfiler: if the trace carried the profiler
+ # categories, the 'standalone' profile IS the in-trace profile and comparing them always agrees.
+ # Detected structurally rather than trusted to the caller.
if (
len(standalone.samples) == len(in_trace.samples)
and len(standalone.nodes) == len(in_trace.nodes)
@@ -221,30 +216,17 @@ def compare_with_trace_profile(
}
-# ═══════════════════════════════════════════════════════════════════════════
# Harness adapter (INTERFACES.md section 3)
-# ═══════════════════════════════════════════════════════════════════════════
-#
-# MEASURED, NOT ASSUMED, and it changed this design: V8 HAS ONE CPU PROFILER.
-# Starting `Profiler.start` while a trace with `disabled-by-default-v8.cpu_profiler`
-# is already running returns THE SAME SAMPLES. On a real capture both routes
-# reported identical counts, 2169 samples over 17 nodes, because the inspector
-# profiler and the tracing profiler share one `CpuProfiler` on the isolate.
-#
-# Two consequences, both load-bearing:
-#
-# 1. `compare_with_trace_profile` must refuse to compare a profile with
-# itself. A cross-check that can never fail is worse than no cross-check,
-# because it reads as corroboration.
-# 2. This instrument is only meaningful at instrument level 1, where the trace
-# carries NO profiler categories. There it is a genuine capability: stacks
-# without the ProfileChunk volume that dominates an L2 trace buffer. A real
-# L1 capture confirmed the two clocks agree, with 1748 of 1748 standalone
-# samples landing inside the trace's own `RunTask` span, so its samples can
-# still be joined to classified task windows.
-#
-# At level 2 and above it stands down and says so, rather than quietly
-# duplicating what `tracing` already has.
+# MEASURED, NOT ASSUMED, and it changed this design: V8 HAS ONE CPU PROFILER. Starting
+# `Profiler.start` while a trace with `disabled-by-default-v8.cpu_profiler` is running returns THE
+# SAME SAMPLES: on a real capture both routes reported 2169 samples over 17 nodes, because the
+# inspector and tracing profilers share one `CpuProfiler` on the isolate.
+# Two consequences: `compare_with_trace_profile` must refuse to compare a profile with itself,
+# since a cross-check that can never fail reads as corroboration; and this instrument is only
+# meaningful at level 1, where the trace carries NO profiler categories and it gives stacks
+# without the ProfileChunk volume that dominates an L2 buffer (a real L1 capture had 1748 of 1748
+# standalone samples inside the trace's own `RunTask` span). At level 2 and above it stands down
+# and says so.
import time # noqa: E402
diff --git a/tests/studio/studiobench/instruments/frames.js b/tests/studio/studiobench/instruments/frames.js
index 013586c2c00..0f3a7c3e88b 100644
--- a/tests/studio/studiobench/instruments/frames.js
+++ b/tests/studio/studiobench/instruments/frames.js
@@ -1,35 +1,26 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
// The frame recorder. Installed as an init script before any app code runs.
-//
-// SALVAGED from playwright_reasoning_pane.py's RECORDER_INIT, with the two things that version
-// got right kept exactly and the one thing it got wrong fixed.
-//
+// SALVAGED from playwright_reasoning_pane.py's RECORDER_INIT, keeping the two things that version
+// got right and fixing the one it got wrong.
// KEPT: ONE self-rescheduling rAF loop as the frame counter, and requestAnimationFrame is NOT
// wrapped. A wrapper that increments per callback counts the page's frame once for the loop and
-// once more for every rAF the app happened to schedule in that frame, so the reported frame rate
-// RISES with how busy the app is. The first version of that file reported 888 fps on a 60Hz page
-// for exactly this reason, which is not a small error, it is the metric inverted.
-//
-// KEPT: blocked time from a 1ms setTimeout, not a MessageChannel ping-pong. The MessageChannel
-// version ticks about 150,000 times a second and halves Firefox's frame rate before any app code
-// runs. This ticks about 150 times a second and costs nothing on any engine. Blocked time is the
-// column that MOVES WHEN FPS DOES NOT: a page with 65% idle still paints every frame on time, so
-// fps stays pinned at 60 while the work per chunk quietly triples.
-//
-// FIXED: the clamp is calibrated during an ENFORCED IDLE WINDOW that the driver opens, not from
-// the first 60 ticks of whatever the page was doing. `setTimeout(fn, 1)` has a floor of about 4ms
-// by spec, the real floor differs by engine and build, and blocked time is a SUBTRACTION against
-// it -- so a wrong clamp invents block on one engine and hides it on another. Calibrating from
-// the first 60 ticks means calibrating while the app is booting, or worse, on a rung where 31,637
-// elements are already standing and the "idle" floor is really the app's own steady-state load.
-// That produces a clamp far above 4ms, which then subtracts real block out of every window and
-// reports a page pinned at 100% busy as 0.2% busy.
-//
-// And when the calibrated clamp comes out ABOVE 10ms, the answer is not a number. A 10ms floor
-// under a 1ms timer means the machine could not answer an idle timer promptly, so nothing was
-// idle and there is no floor to subtract. busy_pct is then null with a reason, never 0.2%.
+// once more for every rAF the app scheduled, so the reported frame rate RISES with how busy the
+// app is: the first version reported 888 fps on a 60Hz page, which is the metric inverted.
+// KEPT: blocked time from a 1ms setTimeout, not a MessageChannel ping-pong, which ticks about
+// 150,000 times a second and halves Firefox's frame rate before any app code runs. This ticks
+// about 150 times a second. Blocked time is the column that MOVES WHEN FPS DOES NOT: a page with
+// 65% idle still paints every frame on time, so fps stays pinned at 60 while the work per chunk
+// triples.
+// FIXED: the clamp is calibrated during an ENFORCED IDLE WINDOW the driver opens, not from the
+// first 60 ticks of whatever the page was doing. `setTimeout(fn, 1)` has a ~4ms spec floor that
+// differs by engine and build, and blocked time is a SUBTRACTION against it, so a wrong clamp
+// invents block on one engine and hides it on another. Calibrating from the first 60 ticks means
+// calibrating while the app is booting, or on a rung where the 'idle' floor is really the app's
+// steady-state load, which reports a page pinned at 100% busy as 0.2% busy.
+// And when the calibrated clamp comes out ABOVE 10ms the answer is not a number: the machine
+// could not answer an idle timer promptly, so nothing was idle and there is no floor to subtract.
+// busy_pct is then null with a reason.
(() => {
if (window.__sb && window.__sb.frames) return;
@@ -39,8 +30,8 @@
const MAX_CLAMP_MS = 10.0;
const CALIBRATION_TICKS = 60;
- // A window long enough to exceed this is minutes of 60 Hz, which no slot in the scene is. The
- // cap exists so a pathological window cannot balloon the payload, not as a routine path.
+ // A window long enough to exceed this is minutes of 60 Hz, which no slot in the scene is. The cap
+ // stops a pathological window ballooning the payload; it is not a routine path.
const GAPS_CAP = 50000;
const R = {
@@ -50,9 +41,9 @@
lagTicks: 0,
lagSumMs: 0,
blockedMs: 0,
- // The same blocked time again, NEVER reset. `blockedMs` is drained by the window reader, and
- // a settle watch has to read blocked time on its own cadence while that reader is running.
- // Two readers draining one accumulator would each see a fraction of the block.
+ // The same blocked time again, NEVER reset. `blockedMs` is drained by the window reader, and a
+ // settle watch has to read blocked time on its own cadence while that reader runs; two readers
+ // draining one accumulator would each see a fraction of the block.
blockedTotalMs: 0,
clampMs: null,
clampReason: "not calibrated",
@@ -64,8 +55,8 @@
),
longTasks: 0,
longTaskMs: 0,
- // Every rAF the APP schedules, counted separately from the loop's own. Not a frame rate; it
- // is how the tri-clock check tells "the page is idle" from "the loop is starved".
+ // Every rAF the APP schedules, counted separately from the loop's own. Not a frame rate: it is
+ // how the tri-clock check tells 'the page is idle' from 'the loop is starved'.
appRafs: 0,
};
@@ -79,8 +70,8 @@
};
nativeRaf(frame);
- // Count the app's own rAF traffic without pumping it. A pass-through wrapper is safe here
- // BECAUSE it is not the frame counter: the loop above is.
+ // Count the app's own rAF traffic without pumping it. A pass-through wrapper is safe here BECAUSE
+ // it is not the frame counter.
window.requestAnimationFrame = function (cb) {
R.appRafs += 1;
return nativeRaf(cb);
@@ -122,8 +113,8 @@
sorted.length === 0 ? null : sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * q))];
window.__sb.frames = {
- // Opened by the driver during an ENFORCED IDLE WINDOW: nothing streaming, no action running,
- // the page at rest. Returns what it measured so the driver can record it rather than trust it.
+ // Opened by the driver during an ENFORCED IDLE WINDOW: nothing streaming, no action running, the
+ // page at rest. Returns what it measured so the driver can record it rather than trust it.
beginCalibration() {
R.calibrating = true;
R.calibration = [];
@@ -141,9 +132,9 @@
}
const median = samples[Math.floor(samples.length / 2)];
if (median > MAX_CLAMP_MS) {
- // NOT a clamp. A 1ms timer that takes longer than 10ms to come back on an idle page means
- // the page was not idle, so there is no floor here to subtract and every blocked-time
- // figure derived from it would be a subtraction against the app's own steady load.
+ // NOT a clamp. A 1ms timer taking longer than 10ms on an idle page means the page was not idle,
+ // so there is no floor to subtract and every blocked-time figure would be a subtraction against
+ // the app's own steady load.
R.clampMs = null;
R.clampReason =
"the calibrated timer clamp came out at " +
@@ -177,9 +168,8 @@
return performance.now();
},
- // Drain the window. `elapsedMs` is the DRIVER's measure of the window, passed in rather than
- // computed here, so fps is per real elapsed time even when the page could not run its own
- // clock reads promptly.
+ // Drain the window. `elapsedMs` is the DRIVER's measure, passed in rather than computed here, so
+ // fps is per real elapsed time even when the page could not run its own clock reads promptly.
read(elapsedMs) {
const gaps = R.frameGaps.slice().sort((a, b) => a - b);
let over33 = 0;
@@ -191,21 +181,18 @@
app_rafs: R.appRafs,
fps: elapsed === null ? null : Math.round((R.frames / (elapsed / 1000)) * 10) / 10,
frames_over_33: over33,
- // As a SHARE of the frames observed, because the denominator is not fixed: headless
- // Chromium has no vsync and runs the loop as fast as it can, so a raw count is not
- // comparable across engines or loads.
+ // As a SHARE of the frames observed, because the denominator is not fixed: headless Chromium has
+ // no vsync and runs the loop as fast as it can, so a raw count is not comparable across engines
+ // or loads.
frames_over_33_pct:
gaps.length === 0 ? null : Math.round((over33 / gaps.length) * 1000) / 10,
p50_frame_ms: quantile(gaps, 0.5),
p95_frame_ms: quantile(gaps, 0.95),
- // The RAW deltas, not only the summary. time_in_jank_pct and jank_index are defined over
- // the whole distribution (a share of wall time, and a sum of squared over-budget time),
- // and neither can be recovered from percentiles. Without this the scoring layer has to
- // either skip two of its six metrics or invent them from p95, which is the kind of
- // plausible-but-unfounded number this tool exists to refuse.
- // `gaps` is sorted ASCENDING, so a head slice would drop exactly the janky frames and
- // score a stuttering window as clean. Over the cap this emits null and says why: the
- // scoring layer then reads "failed", not a number built from the fastest frames.
+ // The RAW deltas, not only the summary: time_in_jank_pct and jank_index are defined over the
+ // whole distribution and neither can be recovered from percentiles, so without this the scoring
+ // layer would skip two of its six metrics or invent them from p95. `gaps` is sorted ASCENDING,
+ // so a head slice would drop exactly the janky frames; over the cap this emits null and says
+ // why, and the scoring layer reads 'failed' rather than a number built from the fastest frames.
frame_gaps_ms:
gaps.length > GAPS_CAP ? null : gaps.map((g) => Math.round(g * 10) / 10),
frame_gaps_truncated: gaps.length > GAPS_CAP,
@@ -218,8 +205,8 @@
clamp_reason: R.clampReason,
long_tasks: R.longTaskSupported ? R.longTasks : null,
long_task_ms: R.longTaskSupported ? Math.round(R.longTaskMs) : null,
- // The point of the flag: without it an engine with no Long Tasks API reports zero jank in
- // exactly the same shape as an engine that had none.
+ // The point of the flag: without it an engine with no Long Tasks API reports zero jank in the
+ // same shape as an engine that had none.
long_task_supported: R.longTaskSupported,
};
if (R.clampMs === null) {
@@ -248,8 +235,8 @@
},
};
- // Two rAFs: the second is the frame that has PAINTED the first's work. Every action's timing
- // that is clocked across a paint uses this, so the paint floor is one shared constant.
+ // Two rAFs: the second is the frame that has PAINTED the first's work. Every action timing
+ // clocked across a paint uses this, so the paint floor is one shared constant.
window.__sbNextPaint = () =>
new Promise((resolve) => nativeRaf(() => nativeRaf(() => resolve(performance.now()))));
})();
diff --git a/tests/studio/studiobench/instruments/glass.js b/tests/studio/studiobench/instruments/glass.js
index 7931139e23b..4698fbc1705 100644
--- a/tests/studio/studiobench/instruments/glass.js
+++ b/tests/studio/studiobench/instruments/glass.js
@@ -1,21 +1,15 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
-// The glass: instrumented accessors that turn "the autoscroll observer forces a synchronous
-// layout on every streamed character" from a reading of the source into a counted, timed number.
-//
-// This is the direct instrument for M3. `use-intent-aware-autoscroll.tsx` installs a
-// MutationObserver with `subtree: true, characterData: true` over the whole viewport; its callback
-// synchronously reads `scrollHeight` (stabilize(), :435), writes the inherited custom property
-// `--aui-scroll-stabilizer` on the scroll container (:466), which invalidates style for every
-// descendant, and calls `scrollTo` (:487). All three are invisible to a React Profiler, to
-// markdown timing and to a DOM node census. They are visible HERE.
-//
-// PERTURBING BY CONSTRUCTION, and so off at level 0. Wrapping a hot getter on Element.prototype
-// costs something on every call site in the app, not only the ones under suspicion, and the
-// wrapper's own `performance.now()` pair is a large fraction of a cheap read. Level 1 and above,
-// and the headline numbers come from level 0, where none of this is installed.
-//
+// The glass: instrumented accessors that turn "the autoscroll observer forces a synchronous layout
+// on every streamed character" from a reading of the source into a counted, timed number.
+// The direct instrument for M3. `use-intent-aware-autoscroll.tsx` installs a MutationObserver with
+// `subtree: true, characterData: true` over the whole viewport; its callback synchronously reads
+// `scrollHeight` (:435), writes the inherited custom property `--aui-scroll-stabilizer` on the
+// scroll container (:466), which invalidates style for every descendant, and calls `scrollTo`
+// (:487). All three are invisible to a React Profiler, to markdown timing and to a node census.
+// PERTURBING BY CONSTRUCTION, and so off at level 0: wrapping a hot getter on Element.prototype
+// costs something on every call site in the app, and the wrapper's own `performance.now()` pair is
+// a large fraction of a cheap read. Level 1 and above; the headline numbers come from level 0.
// Split by WHOSE read it is. "Layout was forced 4,000 times" is not attribution; "3,980 of them
// were on the scroll viewport, from inside a MutationObserver callback" is.
@@ -28,8 +22,8 @@
scroll_height_ms: 0,
viewport_reads: 0,
viewport_ms: 0,
- // Reads taken while a MutationObserver callback is on the stack. This is the number that
- // separates "the app reads scrollHeight" from "the app reads scrollHeight per mutation".
+ // Reads taken while a MutationObserver callback is on the stack: the number that separates "the app
+ // reads scrollHeight" from "the app reads scrollHeight per mutation".
reads_in_observer: 0,
reads_in_observer_ms: 0,
scroll_top_writes: 0,
diff --git a/tests/studio/studiobench/instruments/heap.py b/tests/studio/studiobench/instruments/heap.py
index 28c82dd6506..1c479f7825e 100644
--- a/tests/studio/studiobench/instruments/heap.py
+++ b/tests/studio/studiobench/instruments/heap.py
@@ -28,12 +28,12 @@ from typing import Any, Iterable
from ..analysis import CellFailure
-# 4096 bytes between samples. Small enough to resolve a per-sibling allocation
-# at a few hundred siblings, large enough not to perturb the allocation path.
+# 4096 bytes between samples. Small enough to resolve a per-sibling allocation at a few hundred
+# siblings, large enough not to perturb the allocation path.
DEFAULT_SAMPLING_INTERVAL = 4096
-# `includeObjectsCollectedByMajorGC` / `MinorGC` landed in V8 10.8, which
-# shipped in Chrome 108. Below that the parameter is accepted and ignored.
+# `includeObjectsCollectedByMajorGC` / `MinorGC` landed in V8 10.8, which shipped in Chrome 108.
+# Below that the parameter is accepted and ignored.
MIN_CHROME_FOR_GC_FLAGS = 108
@@ -253,18 +253,13 @@ def survival_ratio(
}
-# ═══════════════════════════════════════════════════════════════════════════
# Harness adapter (INTERFACES.md section 3)
-# ═══════════════════════════════════════════════════════════════════════════
-#
-# Level 3. One sampling session per window, started with
-# `includeObjectsCollectedByMajorGC` so that transient garbage is visible; the
-# hypothesis is per-render churn, and a survivors-only profile of that is empty.
-#
-# The survivors arm is NOT taken here. It needs a second run of the identical
-# workload with the flag off, which is an ablation arm and belongs to Layer 3.
-# What this instrument emits is the allocation side plus the site breakdown, so
-# `analysis.heap.survival_ratio` can be applied across two arms afterwards.
+# Level 3. One sampling session per window, started with `includeObjectsCollectedByMajorGC` so
+# transient garbage is visible: the hypothesis is per-render churn, and a survivors-only profile
+# of that is empty.
+# The survivors arm is NOT taken here. It needs a second run of the identical workload with the
+# flag off, which is an ablation arm belonging to Layer 3. This emits the allocation side plus the
+# site breakdown, so `analysis.heap.survival_ratio` can be applied across two arms afterwards.
import time # noqa: E402
@@ -311,9 +306,8 @@ class HeapInstrument:
try:
self.profiler.start()
except CellFailure as exc:
- # The version gate. Refusing is correct: an older browser ignores
- # the flag silently and hands back survivors only, which for a
- # transient-allocation hypothesis reads as "no allocation here".
+ # The version gate. Refusing is correct: an older browser ignores the flag silently and hands back
+ # survivors only, which for a transient-allocation hypothesis reads as "no allocation here".
self._reason = f"{exc.gate}: {exc.detail}"
self.profiler = None
except Exception as exc: # noqa: BLE001
@@ -360,9 +354,8 @@ class HeapInstrument:
return payload
def end_cell(self, cell: Any) -> dict | None:
- # A prose key is OMITTED when there is nothing to say, never set to
- # None. `None` is reserved for a QUANTITY that could not be measured,
- # and overloading it for "no comment" would make the two indistinguishable.
+ # A prose key is OMITTED when there is nothing to say, never set to None. `None` is reserved for a
+ # QUANTITY that could not be measured.
out = merge(
measured("overhead_ms", round(self._overhead_ms, 3)),
measured("windows_sampled", self._windows),
diff --git a/tests/studio/studiobench/instruments/input.js b/tests/studio/studiobench/instruments/input.js
index 02096b60cc4..5b8352dfce4 100644
--- a/tests/studio/studiobench/instruments/input.js
+++ b/tests/studio/studiobench/instruments/input.js
@@ -1,37 +1,27 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
// Keystroke-to-paint, measured from the page side of a REAL key event.
-//
// WHY THIS IS NOT THE SALVAGED KEYSTROKE_JS. The old harness typed by calling the native value
-// setter on the textarea and dispatching a synthetic `input` event. That reaches React's
-// controlled input, so it measured something -- but it enters the pipeline AFTER hit testing,
-// after the browser's own event routing, and it carries no `latencyInfo`, which is the trace
-// field that says when the OS delivered the keypress. A synthetic event therefore cannot show
-// input queueing delay at all: it is dispatched from a task that is by definition already running,
-// so the queue is empty by construction and the number reads clean exactly when a user would be
-// waiting longest.
-//
+// setter and dispatching a synthetic `input` event. That reaches React's controlled input, but
+// enters the pipeline AFTER hit testing and event routing and carries no `latencyInfo`, so it
+// cannot show input queueing delay at all: it is dispatched from a task already running, so the
+// queue is empty by construction and the number reads clean exactly when a user would be waiting
+// longest.
// So the driver types with `page.keyboard`, which goes in through CDP as a real input event, and
-// this file's job is only to observe from the page: mark when the key arrived, and mark the first
-// paint after the character landed in the value. The subtraction is done here because a
-// driver-side clock would include the CDP round trip, which is not something a user experiences.
-//
-// THE CLOCK STARTS AT THE KEYDOWN, NOT AT THE INPUT HANDLER, and that is the whole point of using
-// a real key event. `input` is dispatched as the default action of `keydown`, so any handler that
-// blocks the main thread on the way in has ALREADY finished by the time `input` fires: a start
-// taken here reads `performance.now()` after the queue drained and subtracts the wait out of the
-// number. Measured directly against the harness's own 400 ms injected keydown stall, an
-// input-anchored clock moved keystroke p95 by -14.8 ms while the user waited 400 ms, and the
-// integrity gate in `instruments/selfcheck.py` -- which requires that stall to move p95 by at
-// least 350 ms -- can never pass on it.
-//
+// this file only observes from the page: mark when the key arrived, and mark the first paint
+// after the character landed in the value. The subtraction is done here because a driver-side
+// clock would include the CDP round trip.
+// THE CLOCK STARTS AT THE KEYDOWN, NOT AT THE INPUT HANDLER. `input` is dispatched as the default
+// action of `keydown`, so a handler that blocks the main thread on the way in has ALREADY
+// finished when `input` fires and a start taken there subtracts the wait out of the number.
+// Against the harness's own 400 ms injected keydown stall, an input-anchored clock moved keystroke
+// p95 by -14.8 ms, and the integrity gate in `instruments/selfcheck.py`, which requires 350 ms of
+// movement, can never pass on it.
// A trusted event's `timeStamp` is a `DOMHighResTimeStamp` on the same origin as
-// `performance.now()`, set when the occurrence the event signals happened rather than when it was
-// dispatched, so `keydown.timeStamp` is the hardware arrival time and carries the queueing delay.
-// Verified on all three engines: with the 400 ms stall armed, `performance.now()` inside the
-// `input` handler is ~400 ms past `keydown.timeStamp` and ~0 ms past the INPUT event's own
-// timeStamp, which is why the anchor is the keydown and not this event.
+// `performance.now()`, set when the occurrence happened rather than when it was dispatched, so
+// `keydown.timeStamp` is the hardware arrival time and carries the queueing delay. Verified on all
+// three engines: with the stall armed, `performance.now()` inside the `input` handler is ~400 ms
+// past `keydown.timeStamp` and ~0 ms past the input event's own timeStamp.
(() => {
if (window.__sb && window.__sb.input) return;
@@ -46,9 +36,9 @@
dropped: 0,
keyAt: null,
unanchored: 0,
- // Every `input` event this instrument saw, whether it became a sample or was coalesced behind
- // a paint that had not finished. Without it there is no denominator: `samples` alone cannot
- // distinguish "the page painted every keystroke" from "most of them never reached here".
+ // Every `input` event this instrument saw, whether it became a sample or was coalesced behind an
+ // unfinished paint. Without it there is no denominator: `samples` alone cannot distinguish 'the
+ // page painted every keystroke' from 'most never reached here'.
seen: 0,
};
@@ -57,18 +47,18 @@
? window.__sbNextPaint()
: new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(() => r(performance.now()))));
- // The key that produced the character being measured. The LAST unconsumed keydown, because a
- // key that produced no character (an arrow, a modifier) must not anchor the next one that did.
+ // The key that produced the character being measured: the LAST unconsumed keydown, because a key
+ // that produced no character must not anchor the next one that did.
const onKeyDown = (ev) => {
if (!S.armed || (S.target && ev.target !== S.target)) return;
S.keyAt = ev.timeStamp;
};
- // Consume the anchor, and refuse an implausible one rather than quoting it. A synthetic event
- // constructed by a page script carries the time of its CONSTRUCTOR, an engine that does not put
- // key events on the performance timeline could report 0, and a composition commit produces an
- // `input` with no keydown at all. Each of those falls back to this handler's own clock, which is
- // the old behaviour, and is counted so the report can say how many samples were not anchored.
+ // Consume the anchor, and refuse an implausible one rather than quoting it. A page-constructed
+ // synthetic event carries its CONSTRUCTOR's time, an engine that does not put key events on the
+ // performance timeline could report 0, and a composition commit produces an `input` with no
+ // keydown. Each falls back to this handler's own clock, the old behaviour, and is counted so the
+ // report can say how many samples were unanchored.
const anchor = (now) => {
const at = S.keyAt;
S.keyAt = null;
@@ -82,8 +72,8 @@
const onInput = (ev) => {
if (!S.armed || ev.target !== S.target) return;
S.seen += 1;
- // One in flight at a time. A burst of characters typed faster than the page can paint would
- // otherwise attribute one paint to several keystrokes and report each of them as fast.
+ // One in flight at a time: a burst typed faster than the page can paint would otherwise attribute
+ // one paint to several keystrokes and report each as fast.
if (S.pending !== null) {
S.dropped += 1;
S.keyAt = null;
@@ -99,8 +89,8 @@
at_ms: Math.round(at * 10) / 10,
// Keystroke to paint: from the key arriving to the frame that shows it.
latency_ms: Math.round((paintedAt - started) * 10) / 10,
- // The two halves, kept separately so a regression can be attributed rather than guessed:
- // how long the key waited to be handled, and how long the page then took to paint it.
+ // The two halves, kept separately so a regression can be attributed: how long the key waited to be
+ // handled, and how long the page then took to paint it.
input_delay_ms: keyAt === null ? null : Math.round((at - keyAt) * 10) / 10,
paint_ms: Math.round((paintedAt - at) * 10) / 10,
anchored_on: keyAt === null ? "input" : "keydown",
@@ -111,8 +101,8 @@
};
window.__sb.input = {
- // `selector` is resolved here rather than passed as a handle, so the driver can re-arm across
- // a page that re-rendered its composer without holding a stale node.
+ // `selector` is resolved here rather than passed as a handle, so the driver can re-arm across a
+ // page that re-rendered its composer without holding a stale node.
arm(selector) {
const el = document.querySelector(selector);
if (!el) return { armed: false, reason: "no element matched " + selector };
@@ -127,28 +117,24 @@
S.seen = 0;
S.armed = true;
el.addEventListener("input", onInput, true);
- // On the WINDOW, in capture, so the anchor is taken however the app routes the key and even
- // if a handler on the way in stops propagation. Idempotent: re-arming across a re-rendered
- // composer must not leave two of these behind, each overwriting the other's anchor.
+ // On the WINDOW, in capture, so the anchor is taken however the app routes the key and even if a
+ // handler stops propagation. Idempotent: re-arming across a re-rendered composer must not leave
+ // two behind, each overwriting the other's anchor.
window.removeEventListener("keydown", onKeyDown, true);
window.addEventListener("keydown", onKeyDown, true);
return { armed: true, baseline_length: S.baseline.length };
},
- // IS ANYTHING STILL IN FLIGHT? The driver polls this instead of waiting a fixed interval.
- //
- // A fixed wait loses whichever keystroke had not painted when it expired, and the keystroke
- // that has not painted yet is the SLOWEST one -- so the metric dropped precisely the sample it
- // exists to catch, and a build that made typing worse read faster. A bigger constant has the
- // same defect on a slower machine or a heavier rung; the only wait that does not is one that
- // ends when the work does.
+ // IS ANYTHING STILL IN FLIGHT? The driver polls this instead of waiting a fixed interval, which
+ // would lose whichever keystroke had not painted when it expired: the SLOWEST one, so the metric
+ // would drop precisely the sample it exists to catch and a build that made typing worse would read
+ // faster. A bigger constant has the same defect on a slower machine.
settled() {
return { pending: S.pending !== null, samples: S.samples.length, seen: S.seen };
},
- // Drain. `expected` is how many characters the driver actually sent, so the report can say
- // "27 of 30 keystrokes produced a measurement" instead of quoting a median over an unknown
- // denominator.
+ // Drain. `expected` is how many characters the driver actually sent, so the report can say '27 of
+ // 30 keystrokes produced a measurement' instead of quoting a median over an unknown denominator.
collect(expected) {
const samples = S.samples.slice();
const latencies = samples.map((s) => s.latency_ms).sort((a, b) => a - b);
@@ -161,8 +147,8 @@
.sort((a, b) => a - b);
const unanchored = S.unanchored;
const seen = S.seen;
- // A sample still in flight AT THIS MOMENT is one the collect is about to lose. Reported so
- // the driver can fail the reading rather than publish a percentile over what survived.
+ // A sample still in flight AT THIS MOMENT is one the collect is about to lose. Reported so the
+ // driver can fail the reading rather than publish a percentile over what survived.
const pendingNow = S.pending !== null;
S.samples = [];
S.unanchored = 0;
@@ -171,25 +157,25 @@
samples: samples.length,
samples_attempted: true,
expected: expected === undefined ? null : expected,
- // The denominator. `inputs_seen` is every keystroke that reached this instrument;
- // `samples + coalesced` must account for all of them, and `pending_at_collect` says
- // whether one was thrown away by the drain itself.
+ // The denominator. `inputs_seen` is every keystroke that reached this instrument; `samples +
+ // coalesced` must account for all of them, and `pending_at_collect` says whether one was thrown
+ // away by the drain.
inputs_seen: seen,
pending_at_collect: pendingNow,
- // How many samples could not be anchored on their own key event and fell back to the
- // input handler's clock. A number quoted from those understates the wait by the queueing
- // delay, so it is reported rather than blended in silently.
+ // How many samples could not be anchored on their own key event and fell back to the input
+ // handler's clock. A number quoted from those understates the wait by the queueing delay, so it
+ // is reported rather than blended in.
unanchored: unanchored,
input_delay_p95_ms:
delays.length === 0 ? null : delays[Math.min(delays.length - 1, Math.floor(delays.length * 0.95))],
- // Not the same question as `samples`. A dropped sample is a keystroke that arrived while
- // a previous one had not painted yet, which is itself the symptom.
+ // Not the same question as `samples`: a dropped sample is a keystroke that arrived while a
+ // previous one had not painted, which is itself the symptom.
coalesced: S.dropped,
p50_ms: at(0.5),
p95_ms: at(0.95),
max_ms: latencies.length === 0 ? null : latencies[latencies.length - 1],
- // The FIRST sample is systematically a cold outlier and is reported separately rather
- // than dropped, because on a jammed page it is also the largest real number in the set.
+ // The FIRST sample is systematically a cold outlier and is reported separately rather than
+ // dropped, because on a jammed page it is also the largest real number in the set.
first_ms: samples.length === 0 ? null : samples[0].latency_ms,
// Proof the characters reached the controlled component and not only the DOM node.
text_length: observedText === null ? null : observedText.length,
diff --git a/tests/studio/studiobench/instruments/layoutcost.js b/tests/studio/studiobench/instruments/layoutcost.js
index 8612e5a7f33..271db52cd90 100644
--- a/tests/studio/studiobench/instruments/layoutcost.js
+++ b/tests/studio/studiobench/instruments/layoutcost.js
@@ -2,96 +2,46 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
/*
- * layoutcost.js -- counts the DOM operations suspected of forcing synchronous layout
- * during streaming, and times them.
+ * layoutcost.js -- counts and times the DOM operations suspected of forcing synchronous layout
+ * during streaming.
*
- * WHAT IT COUNTS AND WHY THESE FIVE
+ * Under investigation: use-intent-aware-autoscroll.tsx, whose MutationObserver on the thread
+ * viewport reads `scrollHeight`, writes `--aui-scroll-stabilizer` and calls `scrollTo` on every
+ * delivery -- per streamed character, at a cost proportional to the whole thread. So the five
+ * instrumented operations are exactly the ones that shape is made of: scrollHeight reads (the
+ * forced-layout trigger), scrollTop writes, scrollTo calls, MutationObserver callbacks AND
+ * records per callback (one callback with 400 records and 400 callbacks cost wildly differently
+ * and look identical in a callback-only profile), and custom-property writes with
+ * `--aui-scroll-stabilizer` counted separately. Timings are here because a count cannot tell
+ * 4,000 cheap reads from 4,000 that each walk a 300-message thread.
+ * Configured childList + subtree + characterData + an attributeFilter.
*
- * The mechanism under investigation is
- * studio/frontend/src/components/assistant-ui/use-intent-aware-autoscroll.tsx. It installs a
- * MutationObserver on the thread viewport (`.aui-thread-viewport.aui-stream-viewport`) with
- * `{childList, subtree, characterData, attributes, attributeFilter:["class","hidden",
- * "aria-hidden","aria-expanded","data-state"]}`. Every delivery of that observer runs
- * `el.scrollHeight` (a read that forces style plus layout when the DOM is dirty, and the DOM is
- * always dirty here because the mutation is what woke the observer), then
- * `el.style.setProperty("--aui-scroll-stabilizer", "<n>px")`, then
- * `el.scrollTo({top, behavior:"instant"})`. During token streaming the characterData mutations
- * arrive per streamed character, and each one costs something proportional to the size of the
- * whole thread, not to the size of the change. That is the shape of the hypothesis, so the
- * instrument records exactly the five operations that shape is made of:
+ * SELF COST. Wrapping a getter to time it makes it slower, so the distortion is measured rather
+ * than assumed: `selfCostEstimate()` times N wrapped reads against N through the ORIGINAL
+ * descriptor on a detached clean element, giving the per-call wrapper overhead; and the Python
+ * driver runs the same cell with and without injection and compares frame statistics, which
+ * catches cache effects and lost inlining that no microbenchmark sees. If the two runs disagree
+ * about the app, the counts stay usable and the timings do not.
+ * Reported as `overheadMsPerCall`.
*
- * scrollHeight READS getter on Element.prototype. The forced-layout trigger.
- * scrollTop WRITES setter on Element.prototype. Cheap to issue, expensive later.
- * scrollTo CALLS method on Element.prototype. The pin.
- * MutationObserver callback invocations, and the number of MutationRecords handed to
- * each one. Records per callback is the number that separates "the
- * observer fires once with 400 records" from "it fires 400 times",
- * which are wildly different costs and look identical in a profile
- * that only counts callbacks.
- * setProperty("--..") custom property writes, with `--aui-scroll-stabilizer` counted on
- * its own so the autoscroll path is separable from every other CSS
- * variable the app writes.
+ * `clockGranularityMs` exists because a clean read can be faster than a clamped
+ * `performance.now()` can resolve: maxMs 0 means "below the clock", not "free".
+ * A genuine 0.003 ms read is indistinguishable from zero.
*
- * Counts alone would be enough to test the "per character" claim. The timings are here because
- * a count cannot distinguish 4,000 cheap reads from 4,000 reads that each walk a 300-message
- * thread, and the whole question is which of those is happening.
+ * OFF BY DEFAULT because it perturbs the measurement; the driver injects it only for the deep
+ * tier, where the question has narrowed to "which operation". `window.__sbLayoutCostDisabled`
+ * is a secondary escape hatch for bisecting the instrument itself.
*
- * THE SELF-COST PROBLEM, STATED PLAINLY
+ * It does not measure layout time, only how often the app asks for something that can force it;
+ * attribution comes from the trace. `window.scrollY`, `getBoundingClientRect`, `offsetHeight`
+ * and `getComputedStyle` also force layout and are deliberately NOT wrapped: they are not in the
+ * path under investigation and every wrapper makes the run less like the app.
*
- * Wrapping the `scrollHeight` getter to time it makes `scrollHeight` slower. This instrument
- * changes the thing it measures. There is no version of this technique that does not, so the
- * honest move is to measure the distortion instead of hoping it is small:
- *
- * 1. `selfCostEstimate()` times N wrapped reads and N reads through the ORIGINAL descriptor,
- * which this file keeps a reference to for exactly this purpose, both on a detached and
- * clean element so no real layout work is included. The difference is the per-call
- * overhead of the wrapper as installed, including its own bookkeeping. The deep tier can
- * subtract `overheadMsPerCall * scrollHeightReads` from the total, or at minimum print it
- * next to the total so a reader can see whether the instrument is a rounding error or a
- * third of the number.
- * 2. The Python driver runs the SAME benchmark cell twice, once with this instrument injected
- * and once without, and compares the frame statistics of the two. That is the real check.
- * Step 1 measures what the wrapper costs per call in isolation; step 2 measures what the
- * whole instrument costs the app in situ, including cache effects and lost inlining that
- * no microbenchmark can see. If the two runs disagree about the app, the counts from this
- * file are still usable and its timings are not, and that has to be discovered rather than
- * assumed.
- *
- * `clockGranularityMs`, also reported by `selfCostEstimate()`, exists because a single
- * `scrollHeight` read on a clean DOM can be faster than `performance.now()` can resolve. In a
- * browser without cross-origin isolation the timer is clamped, so a genuine 0.003 ms read is
- * indistinguishable from zero. A `maxMs` of 0 therefore means "below the clock", not "free",
- * and the granularity number is what lets the report say so.
- *
- * WHY IT IS OFF BY DEFAULT
- *
- * See above: it perturbs the measurement. The default benchmark run must be as close to the
- * shipping app as the harness can manage, so the driver injects this file only for the deep
- * tier, where the question has already narrowed to "which operation" and a known, quantified
- * distortion is worth paying for. Injection is the opt-in. Setting
- * `window.__sbLayoutCostDisabled = true` before this script runs is a secondary escape hatch
- * for bisecting the instrument itself.
- *
- * WHAT IT DOES NOT MEASURE
- *
- * It does not measure layout time. It measures how often the app asks for something that can
- * force layout, and how long the asking took from JS. Actual layout attribution comes from the
- * trace, and the point of this file is to tell the trace where to look.
- *
- * Reads through `window.scrollY`, `document.documentElement.scrollHeight` in the getter's own
- * frame, `getBoundingClientRect`, `offsetHeight` and `getComputedStyle` are NOT wrapped. They
- * force layout too. They are not in the code path under investigation, and every wrapper added
- * here makes the run less like the app.
- *
- * ZERO DISCIPLINE
- *
- * A count of 0 from this file is a real observation: the operation did not happen. A count of 0
- * because the patch could not be installed is not, and the two must never print the same. So
- * `snapshot()` carries `attempted` per instrumented family, `unavailable` lists the names whose
- * original descriptor was missing or non-configurable, and a WebKit build that refuses the
- * patch reads as "not attempted" rather than as a quiet zero. The Python side turns those into
- * Measure objects. Failing to install is not an error and never throws: a partial instrument
- * that is honest about which half ran is more useful than a page that fails to boot.
+ * ZERO DISCIPLINE. A 0 because the operation did not happen and a 0 because the patch could not
+ * be installed must never print the same, so `snapshot()` carries `attempted` per family and
+ * `unavailable` lists names whose descriptor was missing or non-configurable. Failing to install
+ * is not an error and never throws.
+ * A WebKit build that refuses the patch reads as not attempted.
*/
(function () {
@@ -103,9 +53,7 @@
var W = window;
- // Idempotence. add_init_script runs per document, and a page that creates an iframe or that
- // the driver reloads must not stack wrappers: a doubly wrapped getter would double every
- // count and square nothing useful.
+ // Idempotence: add_init_script runs per document; double wrapping would double every count.
if (W.__sbLayoutCostInstalled) {
return;
}
@@ -116,8 +64,7 @@
var MAX_KEYS = 24;
var OTHER_KEY = "__other__";
- // Capture these before the app can touch them. An app that replaces performance.now (some
- // test doubles do) would otherwise silently redefine every timing in this file.
+ // Capture before the app can replace performance.now.
var perf = W.performance;
var rawNow =
perf && typeof perf.now === "function" ? perf.now.bind(perf) : null;
@@ -138,9 +85,6 @@
}
}
- // ---------------------------------------------------------------------------------------
- // State
- // ---------------------------------------------------------------------------------------
function freshTiming() {
return { totalMs: 0, maxMs: 0, samples: 0 };
@@ -199,8 +143,7 @@
unavailable: [],
selfCost: null,
clockGranularityMs: null,
- // Sink for the self-cost loops. A read whose result is thrown away can in principle be
- // optimised out; assigning it somewhere observable keeps both loops honest.
+ // Sink so the discarded read cannot be optimised out.
sink: 0
};
@@ -228,18 +171,10 @@
}
}
- // ---------------------------------------------------------------------------------------
- // Per-target breakdown
- //
- // Keyed by a short, stable descriptor rather than by element identity. Holding elements in a
- // Map would pin detached DOM alive for the whole run, which on a streaming thread means
- // holding every message ever rendered. The key is deliberately coarse: the question is "which
- // KIND of element is being read", and a 300-message thread has maybe six kinds.
- //
- // The cap matters. A page that reads scrollHeight on thousands of distinct elements would
- // otherwise grow an unbounded object inside the hot path, so key 25 and beyond all land in
- // one bucket and `breakdownCapped` says the truncation happened.
- // ---------------------------------------------------------------------------------------
+ // Keyed by a coarse stable descriptor, not element identity, so detached DOM is not pinned alive.
+ // Coarse because the question is which KIND of element is read, and a 300-message thread has
+ // about six kinds.
+ // Cap keys so a page reading thousands of distinct elements cannot grow an unbounded object.
function keyFor(el) {
try {
@@ -260,8 +195,7 @@
if (list.length > 0) {
cls = String(list[0] || "");
}
- // contains(), not matches(): a class test cannot invoke the selector engine and
- // cannot be tricked into anything expensive, and this runs inside the getter.
+ // contains(), not matches(): no selector engine inside the getter.
isViewport = !!list.contains && list.contains(VIEWPORT_CLASS);
}
} catch (e) {
@@ -300,18 +234,13 @@
};
state.breakdown[key] = b;
if (!isOther) {
- // The overflow bucket is not charged to the budget, so `breakdownKeyCount` is the number
- // of REAL keys and the object holds at most MAX_KEYS + 1 entries. Counting the bucket
- // would make a capped run report 25 keys and a reader would reasonably conclude the cap
- // does not work.
+ // The overflow bucket is not charged to the budget, so breakdownKeyCount counts REAL keys.
+ // At most MAX_KEYS + 1 entries.
state.breakdownKeyCount += 1;
}
return b;
}
- // ---------------------------------------------------------------------------------------
- // Element.prototype patches
- // ---------------------------------------------------------------------------------------
var ElementProto =
typeof W.Element === "function" && W.Element.prototype
@@ -360,8 +289,7 @@
return origGet.call(this);
}
var t0 = now();
- // Outside try/catch on purpose. If the native getter throws, the app must see the
- // same exception it would have seen without this file.
+ // Outside try/catch: a throwing native getter must reach the app unchanged.
var value = origGet.call(this);
try {
var dt = now() - t0;
@@ -401,18 +329,14 @@
Object.defineProperty(ElementProto, "scrollTop", {
configurable: true,
enumerable: d.enumerable,
- // The getter is left exactly as it was. Reading scrollTop also forces layout, but it is
- // not the operation under investigation and wrapping it would add cost to a path the
- // hypothesis does not name.
+ // scrollTop's getter is left alone: not the operation under investigation.
get: d.get,
set: function (v) {
if (!state.active) {
origSet.call(this, v);
return;
}
- // Writes are counted, not timed. Issuing a scroll write is cheap; the cost lands in
- // the next layout, where a timer wrapped around the setter would not see it. A number
- // that looks like a duration but is not one is worse than no number.
+ // Writes are counted, not timed; the cost lands in the next layout.
try {
state.counters.scrollTopWrites += 1;
bucketFor(this).scrollTopWrites += 1;
@@ -477,9 +401,6 @@
}
}
- // ---------------------------------------------------------------------------------------
- // CSSStyleDeclaration.prototype.setProperty
- // ---------------------------------------------------------------------------------------
function installSetProperty() {
var proto =
@@ -516,8 +437,7 @@
value: function (name) {
if (state.active) {
try {
- // Only custom properties. Ordinary style writes are far more numerous and are not
- // part of the mechanism being tested; counting them would bury the signal.
+ // Only custom properties: ordinary style writes would bury the signal.
if (typeof name === "string" && name.charCodeAt(0) === 45 && name.charCodeAt(1) === 45) {
state.counters.customPropSets += 1;
if (name === STABILIZER_PROP) {
@@ -538,22 +458,10 @@
}
}
- // ---------------------------------------------------------------------------------------
- // MutationObserver
- //
- // A subclass rather than a Proxy or a plain function: `instanceof MutationObserver` keeps
- // working, `observe` / `disconnect` / `takeRecords` stay native (they are inherited, and
- // `observe` is overridden only to read its arguments before delegating), and the callback is
- // the single thing that changes.
- //
- // The split into viewportObserver and other exists because the app runs several observers and
- // an aggregate count cannot tell the autoscroll one from React's own. Two independent
- // discriminators are recorded per observer: the observe() target carrying the
- // `.aui-stream-viewport` class, and an attributeFilter containing "aria-expanded", which no
- // other observer in this app requests. An observer counts as the viewport observer if EITHER
- // fires, because the classes are renamed more often than the filter is, and both booleans are
- // reported separately so a reader can see when they disagree instead of trusting the merge.
- // ---------------------------------------------------------------------------------------
+ // A subclass, not a Proxy: instanceof and the native methods keep working.
+ // Two independent discriminators (viewport class, aria-expanded filter) separate the autoscroll
+ // observer from React's, reported separately so disagreement is visible.
+ // The split key is `viewportObserver`; no other observer requests that attributeFilter.
function installMutationObserver() {
var Native = W.MutationObserver;
@@ -577,7 +485,7 @@
Wrapped = class extends Native {
constructor(callback) {
if (typeof callback !== "function") {
- // Let the native constructor produce its own TypeError, unchanged.
+ // Let the native constructor produce its own TypeError.
super(callback);
return;
}
@@ -596,8 +504,7 @@
try {
return callback.call(this, records, observer);
} finally {
- // finally, not a trailing statement: a callback that throws still consumed the
- // time, and dropping the sample would make a broken build look fast.
+ // finally: a throwing callback still consumed the time.
try {
var dt = now() - t0;
state.counters.moCallbacks += 1;
@@ -655,8 +562,7 @@
} catch (e) {
/* classification is optional, observing is not */
}
- // apply(arguments), not (target, init): observe() with a missing options argument has
- // its own spec behaviour and must keep it.
+ // apply(arguments), not (target, init): observe() with no options has its own spec behaviour.
return super.observe.apply(this, arguments);
}
};
@@ -684,9 +590,6 @@
}
}
- // ---------------------------------------------------------------------------------------
- // Self cost
- // ---------------------------------------------------------------------------------------
function measureClockGranularityMs() {
var best = null;
@@ -726,9 +629,7 @@
state.breakdown = saved.breakdown;
state.breakdownKeyCount = saved.breakdownKeyCount;
state.breakdownCapped = saved.breakdownCapped;
- // The live objects were replaced, so the convenience aliases on the public API have to be
- // rebound or a caller reading __sbLayoutCost.counters directly would watch a detached copy
- // that never increments again.
+ // Rebind the aliases or a caller reading __sbLayoutCost.counters watches a detached copy.
api.counters = state.counters;
api.timings = state.timings;
api.mo = state.mo;
@@ -759,22 +660,16 @@
var origGet = originals.scrollHeightDesc.get;
var result;
- // The probe runs through the live wrapper, so it increments the real counters. Saving and
- // restoring them keeps the estimate from contaminating the measurement it exists to
- // correct, while still timing the wrapper exactly as the app sees it, bookkeeping included.
+ // Save/restore counters so the probe does not contaminate the measurement it corrects.
var saved = copyRawState();
try {
var probe = doc.createElement("div");
probe.className = "sb-selfcost-probe";
- // Detached and never inserted. An attached element would make both loops pay for real
- // layout, which is the app's cost and not the instrument's, and would swamp the
- // difference this function is trying to resolve. The number produced here is the cost of
- // the WRAPPER, which is the only part this file is responsible for.
+ // Detached and never inserted: this measures the WRAPPER, not the app's layout.
var i;
var sink = 0;
- // Warm both paths first. The first few hundred calls run in the interpreter tier, and
- // whichever loop goes first would otherwise be charged for the JIT.
+ // Warm both paths so neither loop is charged for the JIT.
for (i = 0; i < n; i++) {
sink += probe.scrollHeight;
}
@@ -841,14 +736,9 @@
return copyPlain(result);
}
- // ---------------------------------------------------------------------------------------
- // Public surface
- // ---------------------------------------------------------------------------------------
function copyPlain(v) {
- // Hand written rather than JSON round tripping: JSON turns a non finite number into null
- // and silently drops undefined, and a timing that reads null because of the serialiser
- // would be indistinguishable from a timing that was never taken.
+ // Hand written: JSON turns non-finite into null and drops undefined.
if (v === null || typeof v !== "object") {
return v;
}
@@ -878,7 +768,7 @@
installedAt: api.installedAt,
resetAt: api.resetAt,
snapshotAt: nowStamp(),
- // Per family, so a count of zero can be told apart from a patch that never landed.
+ // Per family, so zero counts differ from a patch that never landed.
attempted: copyPlain(state.attempted),
unavailable: copyPlain(state.unavailable),
counters: copyPlain(state.counters),
@@ -918,9 +808,7 @@
}
function uninstall() {
- // Restores the exact descriptors that were captured, so a page can be handed back to a
- // measurement that must not carry the instrument's overhead. Counters are left alone: the
- // driver usually snapshots after uninstalling.
+ // Restores the captured descriptors; counters are left alone.
state.active = false;
try {
if (originals.scrollHeightDesc && ElementProto) {
@@ -986,8 +874,7 @@
W.__sbLayoutCost = api;
if (W.__sbLayoutCostDisabled === true) {
- // Injected but told to stand down. Everything reads as not attempted, which is the correct
- // answer and not a row of zeros.
+ // Injected but told to stand down: everything reads as not attempted, not a row of zeros.
return;
}
diff --git a/tests/studio/studiobench/instruments/pagejs.py b/tests/studio/studiobench/instruments/pagejs.py
index ee6551d81d0..e568367a20f 100644
--- a/tests/studio/studiobench/instruments/pagejs.py
+++ b/tests/studio/studiobench/instruments/pagejs.py
@@ -57,9 +57,9 @@ class _PageInstrument(Instrument):
self.unavailable = f"could not install {self.script_name}: {exc}"
def start_cell(self, cell: Cell) -> None:
- # Re-read the page every cell. A crashed renderer is recovered by opening a NEW page, so
- # an instrument that cached it in attach() would spend the rest of the run evaluating
- # against a closed one.
+ # Re-read the page every cell: a crashed renderer is recovered by opening a NEW page, so an
+ # instrument that cached it in attach() would spend the rest of the run evaluating against a closed
+ # one.
self.page = self.ctx.page if self.ctx else None
def _eval(
@@ -117,8 +117,8 @@ class FramesInstrument(_PageInstrument):
pass
ctx.cdp.on("Page.screencastFrame", on_frame)
- # Tiny frames: the point is the COUNT and its timing, and a full-size capture would
- # cost the renderer real encode time inside the window being measured.
+ # Tiny frames: the point is the COUNT and its timing, and a full-size capture would cost the
+ # renderer real encode time inside the window being measured.
ctx.cdp.send(
"Page.startScreencast",
{
@@ -213,27 +213,17 @@ class FramesInstrument(_PageInstrument):
"against and frame counts rest on the page's own report alone"
)
return result
- # THE THIRD CLOCK IS NOT RESOLVED, AND THIS SAYS SO RATHER THAN INVENTING IT.
- #
- # The design called for a tri-clock check that excludes a window the three clocks disagree
- # about. Two of the three are sound here. The 1ms timer has a real expectation -- ticks
- # against elapsed/clamp -- and `timer_clock_ratio` below is a direct measure of how much
- # of its budget the main thread could answer. The compositor is a liveness signal only,
- # for the reason in this method's docstring.
- #
- # The rAF loop has NO sound expectation on a headless engine. There is no display, so the
- # loop runs as fast as it can and the absolute frame rate is not a user-visible quantity
- # at all. Normalising it against the best window seen in the cell was tried and is wrong:
- # an early idle window runs far above 60, so every later window scores about 0.44 against
- # it and `clocks_agree` came out FALSE on 34 of 34 windows of a page that was demonstrably
- # steady at 60 fps with 2% blocked time. A gate that fires on every window of every run is
- # worse than no gate, because it gets switched off and takes the real ones with it.
- #
- # So `clocks_agree` is null WITH A REASON, `timer_clock_ratio` is the load-bearing
- # availability signal, and the frame columns are reported as the page's own account of
- # itself. Restoring the third clock needs a vsync-locked source -- a headed run, or
- # `Page.startScreencast` correlated against `LatencyInfo` in a trace, which is Layer 2's
- # surface, not this one's.
+ # THE THIRD CLOCK IS NOT RESOLVED, AND THIS SAYS SO RATHER THAN INVENTING IT. Two of the three are
+ # sound: the 1ms timer has a real expectation and `timer_clock_ratio` measures how much of its
+ # budget the main thread could answer, and the compositor is a liveness signal only. The rAF loop
+ # has NO sound expectation on a headless engine, since there is no display and it runs as fast as
+ # it can. Normalising against the best window in the cell was tried and is wrong: an early idle
+ # window runs far above 60, so every later window scores about 0.44 and `clocks_agree` came out
+ # FALSE on 34 of 34 windows of a page demonstrably steady at 60 fps with 2% blocked time.
+ # So `clocks_agree` is null WITH A REASON, `timer_clock_ratio` is the load-bearing availability
+ # signal, and the frame columns are the page's own account of itself. Restoring the third clock
+ # needs a vsync-locked source: a headed run, or `Page.startScreencast` correlated against
+ # `LatencyInfo` in a trace, which is Layer 2's surface.
result["timer_clock_ratio"] = round(lag_ticks / expected_ticks, 3)
result["clocks_agree"] = None
result["clocks_reason"] = (
diff --git a/tests/studio/studiobench/instruments/selfcheck.py b/tests/studio/studiobench/instruments/selfcheck.py
index 4a7b5aece84..f5052d68d29 100644
--- a/tests/studio/studiobench/instruments/selfcheck.py
+++ b/tests/studio/studiobench/instruments/selfcheck.py
@@ -55,16 +55,16 @@ from ..scoring.schema import ExcludedCell, Measure
STALL_TOLERANCE_MS = 20.0
INJECTED_STALL_MS = 120.0
-#: Milliseconds of main-thread time burned per SSE chunk when the streaming-cost injection is
-#: armed. Sized against what it has to stand in for: preprocessLaTeX over a 96,000 character reply
-#: is projected at 246 ms across a whole stream, so a per-chunk figure in the low single digits is
-#: the same ORDER as the effect the metric is meant to resolve, not a boulder it cannot miss.
+#: Milliseconds of main-thread time burned per SSE chunk when the streaming-cost injection is armed.
+#: Sized against what it stands in for: preprocessLaTeX over a 96,000 character reply is projected
+#: at 246 ms across a whole stream, so a per-chunk figure in the low single digits is the same ORDER
+#: as the effect the metric must resolve.
INJECTED_STREAM_COST_MS = 3.0
#: The share of injected cost `stream_cost` must read back. Not 1.0: the metric measures the task
#: chain from the chunk to the event loop draining, and a burn queued as a microtask lands inside
-#: that chain but the chain also contains the app's own work, so recovery is bounded below by
-#: attribution and above by nothing. Under-recovery is the failure that matters, because a metric
-#: that recovers a quarter of a known cost will under-report an unknown one by the same factor.
+#: that chain but so does the app's own work. Under-recovery is the failure that matters, because a
+#: metric that recovers a quarter of a known cost will under-report an unknown one by the same
+#: factor.
MIN_STREAM_COST_RECOVERY = 0.70
INJECTED_INPUT_DELAY_MS = 400.0
MIN_INPUT_P95_SHIFT_MS = 350.0
@@ -143,8 +143,9 @@ class SelfCheckReport:
return {"ok": self.ok, "gates": [gate.to_json() for gate in self.gates]}
-# ---------------------------------------------------------------------------------------
# pure evaluators
+
+
# ---------------------------------------------------------------------------------------
@@ -446,13 +447,12 @@ def evaluate_tri_clock(
)
-# ---------------------------------------------------------------------------------------
# browser-side snippets and drivers
+
+#: Burns a known amount of main-thread time once, on the next frame. A busy wait, not a sleep: a
+#: sleep is recovered through a different scheduler path than a blocked main thread.
# ---------------------------------------------------------------------------------------
-#: Burns a known amount of main-thread time once, on the next frame. A busy wait, not a sleep:
-#: a sleep is recovered through a different scheduler path than a blocked main thread, which is
-#: the thing being calibrated.
STALL_INJECT_JS = """
(stallMs) => {
return new Promise((resolve) => {
@@ -465,20 +465,17 @@ STALL_INJECT_JS = """
}
"""
-#: Burns a known amount of main-thread time PER SSE CHUNK, inside the task chain that chunk
-#: starts. This is the streaming analogue of STALL_INJECT_JS: a stall injected once tests whether
-#: the frame recorder is watching the right window, and this tests whether the streaming-cost
-#: accumulator integrates a cost spread thinly across a whole stream, which is the shape of every
-#: effect it was built for and the shape a single-action metric cannot see.
-#:
-#: THE BURN IS QUEUED AS A MICROTASK, not run inline in the decode wrapper, and that is what makes
-#: the check independent of wrapper order. `add_init_script` runs scripts in the order they were
-#: added, and the instrument's own TextDecoder wrapper is installed by `Instrument.attach` after
-#: the scripts assembled in __main__. Whichever wrapper ends up outermost, a microtask queued
-#: during the decode runs after the current task's synchronous code and before the MessageChannel
-#: macrotask that closes the measured chain -- so the burn is inside the chain either way. Burning
-#: inline would land it before the accumulator timestamps the chunk under one ordering and after
-#: it under the other, and the check would silently measure nothing.
+#: Burns a known amount of main-thread time PER SSE CHUNK, inside the task chain that chunk starts.
+#: The streaming analogue of STALL_INJECT_JS: this tests whether the streaming-cost accumulator
+#: integrates a cost spread thinly across a whole stream, which is the shape of every effect it was
+#: built for.
+#: THE BURN IS QUEUED AS A MICROTASK, not run inline in the decode wrapper, which makes the check
+#: independent of wrapper order. `add_init_script` runs scripts in the order added and the
+#: instrument's own TextDecoder wrapper is installed by `Instrument.attach` after the scripts
+#: assembled in __main__; whichever ends up outermost, a microtask queued during the decode runs
+#: after the current task's synchronous code and before the MessageChannel macrotask that closes
+#: the chain. Burning inline would land it before the accumulator timestamps the chunk under one
+#: ordering and after it under the other.
STREAM_COST_INJECT_JS = """
(() => {
if (window.__sbStreamCostInject) { return; }
@@ -582,7 +579,7 @@ INPUT_DELAY_INIT_JS = """
})();
"""
-#: Reads `supportedEntryTypes` directly. Never a try/catch around observe().
+#:Reads `supportedEntryTypes` directly. Never a try/catch around observe().
LONGTASK_SUPPORT_JS = """
() => {
try {
diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py
index 9ce7949c651..11f943bdf5f 100644
--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py
+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py
@@ -44,9 +44,9 @@ from studiobench.scene.actions import keystroke # noqa: E402
INPUT_JS = Path(__file__).resolve().parents[1] / "input.js"
-#: A DOM small enough to run the instrument and explicit enough to hold a paint open. Paints
-#: resolve only when this driver says so, which is what makes "still in flight at the drain" a
-#: state the test can create rather than race for.
+#: A DOM small enough to run the instrument and explicit enough to hold a paint open. Paints resolve
+#: only when this driver says so, which makes "still in flight at the drain" a state the test can
+#: create rather than race for.
DRIVER = """
import fs from "node:fs";
let now = 0;
diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py
index 6fb98499b99..2bc41366d1a 100644
--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py
+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py
@@ -72,12 +72,11 @@ def _typed(page, *, delay_armed: bool) -> dict:
def test_an_injected_keydown_stall_moves_keystroke_p95():
playwright = pytest.importorskip("playwright.sync_api", reason = "playwright is not installed")
- # IMPORTORSKIP IS NOT ENOUGH IN THIS SUITE. tests/studio/
- # test_heavy_thread_measurement_integrity.py puts a stub `playwright.sync_api` into
- # `sys.modules` at collection time and, as its own docstring says, it stays there for the rest
- # of the session. So on the CPU job the import above SUCCEEDS against the stub and every name
- # it hands back raises RuntimeError when called. The stub is a bare ModuleType with no
- # `__file__`, which the real package always has, so that is what distinguishes them.
+ # IMPORTORSKIP IS NOT ENOUGH IN THIS SUITE. tests/studio/test_heavy_thread_measurement_integrity.py
+ # puts a stub `playwright.sync_api` into `sys.modules` at collection time and it stays there for
+ # the rest of the session, so on the CPU job the import above SUCCEEDS against the stub and every
+ # name raises RuntimeError when called. The stub is a bare ModuleType with no `__file__`, which the
+ # real package always has.
if getattr(playwright, "__file__", None) is None:
pytest.skip("playwright.sync_api is the CPU-job stub, so there is no browser to drive")
@@ -91,9 +90,9 @@ def test_an_injected_keydown_stall_moves_keystroke_p95():
context.add_init_script(input_delay_init_script())
context.add_init_script(resources.read_text("instruments/input.js"))
page = context.new_page()
- # `goto`, never `set_content`: `document.open()` removes every listener registered on
- # the window, so a page built that way silently discards the injected stall and the
- # test would pass against a broken instrument.
+ # `goto`, never `set_content`: `document.open()` removes every listener registered on the window, so
+ # a page built that way silently discards the injected stall and the test would pass against a
+ # broken instrument.
page.goto(URL)
quiet = _typed(page, delay_armed = False)
diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py
index 936bd0e4aef..dbe533f6777 100644
--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py
+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py
@@ -31,7 +31,7 @@ from studiobench.instruments.streamcost import StreamCostInstrument # noqa: E40
STREAMCOST_JS = Path(__file__).resolve().parents[1] / "streamcost.js"
-#: `IDLE_GAP_MS` in the instrument. A stall longer than this is the case that used to vanish.
+#:`IDLE_GAP_MS` in the instrument. A stall longer than this is the case that used to vanish.
IDLE_GAP_MS = 1500
HARNESS_JS = r"""
@@ -66,8 +66,7 @@ setTimeout(() => {
#: The same file, driven through the ordering that loses a burst: a window that closes while the
#: chain the last chunk started has not reached its macrotask yet. `readWhilePending` decides which
-#: of the two orderings the harness produces, so the pinned case and its control differ by nothing
-#: else.
+#: ordering the harness produces, so the pinned case and its control differ by nothing else.
PENDING_HARNESS_JS = r"""
const fs = require("fs");
const src = fs.readFileSync(process.argv[2], "utf8");
@@ -111,9 +110,9 @@ if (readWhilePending) {
#: The same file again, driven through ONE `decode()` of a whole batched read. `payload` decides
#: what that read carries: a burst of the pacer's own frames larger than the decoder's scan bound,
-#: the same burst small enough to sit under it, or a blob of the size that bound exists to keep out.
-#: The burn after the decode stands in for the SSE parse, the delta accumulation and the render --
-#: the task chain `delta_task_ms` is supposed to be charging.
+#: the same burst small enough to sit under it, or a blob of the size that bound exists to keep
+#: out. The burn after the decode stands in for the SSE parse, the delta accumulation and the
+#: render.
BATCH_HARNESS_JS = r"""
const fs = require("fs");
const src = fs.readFileSync(process.argv[2], "utf8");
@@ -238,7 +237,8 @@ def one_decoded_batch(mode: str, burn_ms: float) -> dict:
#: The task chain one burst starts, in the harness above. Large enough that losing it is
-#: unmistakable in the assertions and small enough that node's timers stay honest.
+#: unmistakable and small enough that node's timers stay honest.
+# This is what `delta_task_ms` charges.
BURST_CHAIN_MS = 40.0
@@ -287,8 +287,7 @@ def test_a_burst_whose_chain_has_already_closed_is_charged_once():
assert out["second"]["delta_task_ms"] < BURST_CHAIN_MS * 0.1, out
-#: `MAX_SSE_CHUNK_CHARS` in the instrument. A single decode above it is the case that used to be
-#: discarded whole.
+#:`MAX_SSE_CHUNK_CHARS` in the instrument. A single decode above it is the case that used to be discarded whole.
MAX_SSE_CHUNK_CHARS = 65536
@@ -405,8 +404,8 @@ class _FakeStreamCostPage:
than driving node keeps the test about the DRIVER's ordering, which is where the defect lives.
"""
- #: What one boundary scan costs. `querySelectorAll` collects its matches up front over the
- #: whole document, so this is the one part of the instrument that grows with the rung.
+ #: What one boundary scan costs. `querySelectorAll` collects its matches up front over the whole
+ #: document, so this is the one part of the instrument that grows with the rung.
SCAN_MS = 3.9
def __init__(self) -> None:
@@ -471,8 +470,8 @@ def test_the_close_side_drain_does_not_disturb_the_window_s_own_reading():
assert out["streaming_observed"] is True
assert out["reply_chars_delta"] == 1_000
- # The window's own overhead figure is what `read()` returned; the close scan is reported
- # beside it rather than folded into it.
+ # The window's own overhead figure is what `read()` returned; the close scan is reported beside it
+ # rather than folded into it.
assert out["overhead_ms"] == pytest.approx(_FakeStreamCostPage.SCAN_MS, abs = 0.05)
assert out["close_scan_overhead_ms"] == pytest.approx(_FakeStreamCostPage.SCAN_MS, abs = 0.05)
diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py
index 5e445774807..cea83f6df43 100644
--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py
+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py
@@ -37,12 +37,12 @@ if str(_STUDIO_TESTS) not in sys.path:
_STREAMCOST_JS = _STUDIO_TESTS / "studiobench" / "instruments" / "streamcost.js"
-#: The two documents. 40,000 elements is the size the 289.6 ms figure was measured against; 4,000
-#: is roughly what a window of six messages leaves standing at the same rung.
+#: The two documents. 40,000 elements is the size the 289.6 ms figure was measured against; 4,000 is
+#: roughly what a window of six messages leaves standing at the same rung.
FULL_ELEMENTS = 40_000
WINDOWED_ELEMENTS = 4_000
-#: How many reads to take. The quantity is a few milliseconds, so one reading is noise.
+#:How many reads to take. The quantity is a few milliseconds, so one reading is noise.
READS = 40
#: What counts as "no longer biased". The old reading's residual is milliseconds per call; the new
@@ -125,8 +125,8 @@ def browser():
def _page(browser, elements: int):
page = browser.new_page(viewport = {"width": 900, "height": 600})
page.set_content("<!doctype html><meta charset=utf-8><body></body>")
- # BEFORE the document exists, exactly as the real harness installs it via add_init_script:
- # the hook has to be on TextDecoder.prototype before any decode happens.
+ # BEFORE the document exists, exactly as the real harness installs it via add_init_script: the hook
+ # has to be on TextDecoder.prototype before any decode happens.
page.add_script_tag(content = _STREAMCOST_JS.read_text(encoding = "utf-8"))
got = page.evaluate(BUILD_JS, elements)
page.evaluate(FEED_JS, 200)
@@ -134,8 +134,7 @@ def _page(browser, elements: int):
def _per_call_ms(page, which: str) -> float:
- # Median of several passes. A single pass on a shared machine picks up whatever else is
- # running, and this box is running other people's sweeps.
+ # Median of several passes: a single pass on a shared machine picks up whatever else is running.
return statistics.median(page.evaluate(TIME_JS, [which, READS]) for _ in range(5))
@@ -165,8 +164,8 @@ def test_the_wire_denominator_is_the_same_price_on_a_windowed_document(browser,
f"RESIDUAL {wire_residual:+.4f} ms/call"
)
- # The old reading really is cheaper on the smaller document. If this ever stops being true the
- # test below is proving nothing, so it is asserted rather than assumed.
+ # The old reading really is cheaper on the smaller document. If this ever stops being true the test
+ # below is proving nothing, so it is asserted rather than assumed.
assert dom_residual > 0, (
"the O(document) read was not measurably cheaper on the windowed document, so this "
f"machine cannot demonstrate the bias at all (full {dom_full}, windowed {dom_win})"
@@ -195,8 +194,8 @@ def test_the_wire_count_is_identical_on_both_documents_for_identical_traffic(bro
assert a["wire_chars"] == b["wire_chars"] > 0
assert a["wire_frames"] == b["wire_frames"] == 200
assert a["wire_parse_failures"] == b["wire_parse_failures"] == 0
- # The DOM reading, by contrast, is a different number on the two documents -- which is exactly
- # why it could not be the denominator.
+ # The DOM reading, by contrast, is a different number on the two documents, which is exactly why it
+ # could not be the denominator.
assert a["wire_chars"] == 200 * len("token 0 ") or a["wire_chars"] > 0
diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py
index 0db00ed4bb5..d3d5a8c0f7a 100644
--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py
+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py
@@ -124,9 +124,9 @@ def test_a_clean_stream_is_scoreable_and_counts_every_character(page):
"failures": 0,
"pending_chars": 0,
"carried_flushes": got["carried_flushes"],
- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the
- # id is which decoder the two numbers above are about, and the exact integer
- # depends on how many decoders the page has built before this assertion.
+ # Not pinned to a value: the id says which decoder the two numbers above are about, and the integer
+ # depends on how many decoders the page has built.
+ # Same footing as `carried_flushes` beside it.
"decoder_id": got["decoder_id"],
}, got
@@ -206,18 +206,15 @@ def test_a_clean_window_stays_scoreable(page):
assert out["reply_chars_delta"] == len("all good")
-# ── a frame split INSIDE the "data:" prefix ─────────────────────────
-#
-# The hook starts buffering when a chunk contains a complete `data:`. The socket does not respect
-# that boundary: it can cut a frame anywhere, including between the "da" and the "ta:". Neither
-# half then contains the marker and neither completes a buffered frame, so both were discarded --
-# and the frame's characters left the denominator WITHOUT incrementing `wireParseFailures`, so the
-# window-integrity check above could not see it either. Later frames make the window look
-# scoreable while the count underneath it is short, which inflates every cost-per-character
-# derived from it, at exactly the moment the instrument exists to measure: chunks arrive ragged
-# when the renderer is jammed.
+# A frame split INSIDE the "data:" prefix. The hook starts buffering when a chunk contains a
+# complete `data:`, but the socket can cut a frame between the "da" and the "ta:": neither half
+# contains the marker or completes a buffered frame, so both were discarded and the frame's
+# characters left the denominator WITHOUT incrementing `wireParseFailures`. Later frames then make
+# the window look scoreable while the count underneath is short, inflating every
+# cost-per-character at exactly the moment the instrument exists to measure.
+# ── a frame split INSIDE the "data:" prefix ─────────────────────────
def _halves(text: str, at: int) -> tuple:
return text[:at], text[at:]
@@ -236,9 +233,8 @@ def test_the_counter_survives_a_split_inside_the_data_prefix(page):
"failures": 0,
"pending_chars": 0,
"carried_flushes": got["carried_flushes"],
- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the
- # id is which decoder the two numbers above are about, and the exact integer
- # depends on how many decoders the page has built before this assertion.
+ # Not pinned to a value: the id says which decoder the two numbers above are about.
+ # Same footing as `carried_flushes` beside it.
"decoder_id": got["decoder_id"],
}, got
@@ -277,9 +273,8 @@ def test_unrelated_text_ending_in_a_marker_letter_does_not_corrupt_the_next_fram
"failures": 0,
"pending_chars": 0,
"carried_flushes": got["carried_flushes"],
- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the
- # id is which decoder the two numbers above are about, and the exact integer
- # depends on how many decoders the page has built before this assertion.
+ # Not pinned to a value: the id says which decoder the two numbers above are about.
+ # Same footing as `carried_flushes` beside it.
"decoder_id": got["decoder_id"],
}, got
@@ -438,9 +433,8 @@ def test_an_aborted_frame_does_not_follow_the_stream_that_replaces_it(page):
"failures": 0,
"pending_chars": 0,
"carried_flushes": got["carried_flushes"],
- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the
- # id is which decoder the two numbers above are about, and the exact integer
- # depends on how many decoders the page has built before this assertion.
+ # Not pinned to a value: the id says which decoder the two numbers above are about.
+ # Same footing as `carried_flushes` beside it.
"decoder_id": got["decoder_id"],
}, got
@@ -534,8 +528,8 @@ def test_an_abort_does_not_cost_the_next_response_its_reading_when_that_one_is_s
_feed(page, tail)
out = inst.close(_Window())
assert out["reply_chars_delta"] == len("a clean reply"), out
- # The new response DID carry a frame across a decode boundary, so the counter really moved:
- # this window is accepted despite that, not because nothing was counted.
+ # The new response DID carry a frame across a decode boundary, so the counter really moved: this
+ # window is accepted despite that, not because nothing was counted.
assert out["wire_carried_frames_counted_in_window"] == 0, out
assert out["reply_chars_scoreable"] is True, out
@@ -595,15 +589,14 @@ def test_a_frame_that_really_did_straddle_the_open_still_refuses_its_window(page
assert out["reply_chars_scoreable"] is False, out
-# ── the tail of a split frame is stream traffic on the numerator too ─────────
-#
-# `noteSse` was gated on the marker while the character counter was gated on `looksSse ||
-# pending`, so the two halves of one frame were attributed to two different things: the tail's
-# characters went into the denominator and its render work went nowhere. The bias is downward on
-# `stream_delta_cost_ms_per_kchar` and it grows with fragmentation, which is the regime the
-# instrument exists to measure.
+# The tail of a split frame is stream traffic on the numerator too. `noteSse` was gated on the
+# marker while the character counter was gated on `looksSse || pending`, so the two halves of one
+# frame went to two different things: the tail's characters into the denominator and its render
+# work nowhere. The bias is downward on `stream_delta_cost_ms_per_kchar` and grows with
+# fragmentation, which is the regime the instrument exists to measure.
+# ── the tail of a split frame is stream traffic on the numerator too ─────────
def test_the_tail_of_a_split_frame_is_counted_as_stream_traffic(page):
"""THE DEFECT, at the quantity the numerator is built from.
diff --git a/tests/studio/studiobench/instruments/streamcost.js b/tests/studio/studiobench/instruments/streamcost.js
index 408db129bd5..92df11c06eb 100644
--- a/tests/studio/studiobench/instruments/streamcost.js
+++ b/tests/studio/studiobench/instruments/streamcost.js
@@ -1,108 +1,50 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
-// The streaming-phase cost accumulator. Installed as an init script before any app code runs.
-//
-// WHAT IS MISSING FROM THE HARNESS THAT THIS FILLS. `time_in_jank_pct`, `jank_index` and
-// `max_frame_ms` do cover the stream -- `_frame_measures` pools every window whose kind is not
-// `idle`, and the streaming windows are in that pool. What they cannot do is SEPARATE it. One
-// 57.3 s film collapses eighteen action windows and the streaming stretch into a single number,
-// and the action windows dominate it: measured on a 100K null control, `reasoning_toggle` alone
+// The streaming-phase cost accumulator, installed as an init script before any app code runs.
+// WHAT THE HARNESS CANNOT DO: `time_in_jank_pct`, `jank_index` and `max_frame_ms` cover the
+// stream but cannot SEPARATE it. One 57.3 s film collapses eighteen action windows and the
+// streaming stretch into one number; on a 100K null control `reasoning_toggle` alone
// contributes 2,865 ms of blocked time at 99.3% busy with a 1,866 ms worst frame, while the
-// streaming stretch next to it runs at 3.6% busy. A change to the streaming path moves the second
-// and is scored against the first.
-//
-// The window KIND cannot be used to make that separation, which is the trap this file exists past.
-// `SceneRunner._gap_window` opens EVERY inter-slot gap as `kind = "stream"`, so on the standard
-// film eighteen windows are labelled `stream:` and only the first four contain any streaming at
-// all; `stream:drain` at the end of the cell is 7 ms of nothing, the stream having finished
-// forty seconds earlier. The streaming phase is therefore detected here, from the SSE traffic
-// itself, and not read off a label.
-//
-// THREE THINGS ARE ACCUMULATED, and they are different quantities on purpose:
-//
-// deltaTaskMs The main-thread time of the task chains that SSE chunks start. Measured from
-// the first decode of a burst to the moment the event loop next reaches a
-// macrotask, so it spans the SSE parse, the delta accumulation, the cumulative
-// re-parse and React's own render and commit. This is the TARGETED numerator: it
-// excludes the background churn -- highlighting, GC, the app's own timers -- that
-// a whole-window blocked-time figure charges to the stream.
-//
-// blockedMs Blocked time, on exactly the discipline frames.js uses (a 1 ms timer, gap minus
-// the calibrated clamp), but ACCUMULATED ONLY WHILE THE STREAM IS RUNNING. This
-// is the BROAD numerator, and it is the honest one: it catches stream-driven cost
-// that lands outside the delta's own task chain, which is most of the async
-// highlighting work.
-//
-// streamingMs How much of the window the stream was actually running, so a window that is
-// half stream and half idle is not read as a whole window of streaming.
-//
-// WHY THE END OF A TASK CHAIN IS TIMED WITH MessageChannel AND NOT setTimeout. A nested
-// `setTimeout(fn, 0)` is clamped to 4 ms once the nesting level passes five, and a chunk arriving
-// inside a fetch reader continuation inherits that nesting. That clamp would add a systematic
-// ~4 ms to every chunk -- about 720 ms over a 13 s stream -- which is larger than most effects
-// this is meant to resolve. A MessageChannel message is not clamped and runs as a macrotask
-// immediately after the current task and its microtasks, which is exactly the boundary wanted.
-//
-// This is NOT the MessageChannel pattern frames.js bans. That one is a ping-pong LOOP that
-// re-posts itself and ticks about 150,000 times a second, halving the frame rate before any app
-// code runs. This posts once per SSE burst, about fourteen times a second at field cadence, and
-// only while a stream is in flight.
-//
-// AND IT DOES NOT TOUCH requestAnimationFrame. frames.js owns the one rAF loop that counts
-// frames. Nothing here schedules, wraps or counts a rAF, so the 888-fps inversion cannot recur
-// through this file.
+// streaming stretch next to it runs at 3.6% busy.
+// The window KIND cannot separate them either: `SceneRunner._gap_window` opens every
+// inter-slot gap as `kind = "stream"`, so eighteen windows are labelled `stream:` and only
+// four carry streaming. The phase is detected here from the SSE traffic itself.
+// THREE ACCUMULATORS, different quantities on purpose. `deltaTaskMs`: main-thread time of the
+// task chains SSE chunks start, first decode of a burst to the next macrotask (the TARGETED
+// numerator). `blockedMs`: frames.js blocked time, but only while the stream runs (the BROAD
+// numerator, catching async highlighting off the chain). `streamingMs`: how much of the window
+// the stream was actually running.
+// Task-chain end is timed with MessageChannel, not setTimeout: a nested `setTimeout(fn, 0)` is
+// clamped to 4 ms past nesting level five, ~720 ms over a 13 s stream. Not the ping-pong loop
+// frames.js bans (one post per SSE burst, ~14/s, only in flight) and it never touches rAF, so
+// the 888-fps inversion cannot recur here.
(() => {
if (window.__sb && window.__sb.streamcost) return;
window.__sb = window.__sb || {};
- // How long after the last SSE chunk the stream is still considered in flight. Field cadence is
- // 24 characters every 73 ms and the pacer is deficit-scheduled, so it bursts after a jam rather
- // than sleeping a gap per chunk; a jammed renderer can therefore go quiet for a while and then
- // receive a burst. 1500 ms is about twenty cadence gaps, long enough not to chop a jam in half
- // and short enough that the forty seconds of post-stream film are never counted as streaming.
+ // How long after the last SSE chunk the stream still counts as in flight. The
+ // deficit-scheduled pacer bursts after a jam rather than sleeping per chunk, so 1500 ms is
+ // about twenty cadence gaps: long enough not to chop a jam, short enough that the post-stream
+ // film is never counted as streaming.
+ // Field cadence is 24 characters every 73 ms.
const IDLE_GAP_MS = 1500;
- // How much of a decoded chunk is scanned for the relay's framing. The substring search below is
- // the per-event work this file declares as O(1), and an unbounded scan over a bundle, a blob or
- // a paste would make it O(the page's whole TextDecoder traffic) instead.
- //
- // IT BOUNDS THE SCAN AND NOT THE PAYLOAD, and it used to bound the payload: the guard read
- // `out.length <= MAX_SSE_CHUNK_CHARS` and dropped anything longer, on the premise that a decode
- // this large is not relay traffic. That premise is false, and it is false in exactly the case
- // this instrument exists for. A read does not carry one cadence gap of the stream, it carries
- // everything the browser buffered since the last one, so the size of a read is the arrival rate
- // times the length of the stall in front of it. Measured against real chromium reading the real
- // pacer through the app's own `getReader()` loop, the largest read of a stream is 32.5
- // characters per millisecond of stall at fast cadence, dead linear over 500 to 3,000 ms: a
- // 2,000 ms stall lands a 65,000 character read and a 3,000 ms stall a 97,500 character one,
- // which the old guard discarded whole. That payload is well-formed SSE -- 470 `data:` frames of
- // the pacer's own framing -- and it is the single largest task chain of the stream, so the guard
- // took the worst burst out of `sseChunks`, `lastSseAt` and the `deltaTaskMs` numerator at the
- // one moment they matter most. Field cadence arrives at 2.97 characters per millisecond and
- // would need a 22 s stall, so this is reachable on `--cadence fast` and not on the default.
- //
- // HOW MUCH IT COST, and it is not a rounding error. The size of the loss is whatever the app
- // spends per streamed character, because what is dropped is one chain over a third of the reply.
- // Sweeping that rate against the real pacer, with the read loop otherwise identical:
- //
- // 0.0 ms per 1,000 characters 15.7 vs 15.4 ms -1.3%, which is the noise floor
- // 0.5 ms per 1,000 characters 55.7 vs 104.7 ms 46.8% of the numerator gone
- // 2.0 ms per 1,000 characters 154.0 vs 350.5 ms 56.1%
- // 5.0 ms per 1,000 characters 349.1 vs 835.5 ms 58.2%
- //
- // A harness that does no work per character loses nothing measurable, which is why this hid: it
- // needs an app that actually renders what it streams. It also gets WORSE as the app gets slower,
- // because a slower reader falls further behind and the batch it is then handed is bigger -- the
- // same shape as the timer attribution above, a build getting worse reading cheaper.
- //
- // Scanning a bounded head rather than rejecting keeps both of the things the cap was for. The
- // work stays bounded by a constant that does not grow with the payload or with the rung, and a
- // bundle still has to put `data:` in its first 65,536 characters to be mistaken for a stream --
- // while a real burst, which starts a frame every 279 characters, is found immediately. Measured
- // in v8: 0.04 us on an SSE batch and 0.68 us on a 2 MB blob with no frame in it, against 0.70 us
- // for the same scan left unbounded.
+ // How much of a decoded chunk is scanned for the relay's framing: the substring search below
+ // is the per-event work this file declares O(1), and an unbounded scan would make it O(the
+ // page's whole TextDecoder traffic).
+ // IT BOUNDS THE SCAN, NOT THE PAYLOAD. The guard used to drop any decode longer than the cap,
+ // on the false premise that a large decode is not relay traffic: a read carries everything
+ // buffered since the last one, and at fast cadence the largest read is 32.5 characters per ms
+ // of stall, so a 3,000 ms stall lands a well-formed 97,500 character read of 470 `data:`
+ // frames, which the old guard discarded whole. Swept against the real pacer that cost 46.8% to
+ // 58.2% of the numerator, and nothing at 0 ms per 1,000 characters: it hid on a harness that
+ // does no per-character work and got worse as the app got slower. Scanning a bounded head
+ // keeps both original goals: constant-bounded work, and a bundle must still put `data:` in its
+ // first 65,536 characters.
+ // Field cadence arrives at 2.97 characters per millisecond.
+ // Measured in v8: 0.04 us on an SSE batch and 0.68 us on a 2 MB blob with no frame in it,
+ // against 0.70 us for the same scan left unbounded.
const MAX_SSE_CHUNK_CHARS = 65536;
const S = {
@@ -112,62 +54,38 @@
blockedMs: 0,
streamingMs: 0,
lastSseAt: 0,
- // Wall time this file spent inside its own hooks, so the overhead it declares is measured
- // rather than asserted. An instrument that guesses its own cost cannot be checked against
- // the overhead_growth_with_length gate.
+ // Wall time spent inside this file's own hooks, so its declared overhead is measured rather
+ // than asserted; an instrument that guesses its own cost cannot be checked against the
+ // overhead_growth_with_length gate.
overheadMs: 0,
decodeCalls: 0,
everStreamed: false,
- // CUMULATIVE characters of assistant text this page has been SENT, counted off the wire.
- // Never reset by `reset()`: the denominator a window wants is the growth across it, which is
- // the difference of two readings of this, and resetting it per window would make every
- // reading zero.
+ // CUMULATIVE characters of assistant text this page has been SENT, counted off the wire. Never
+ // reset by `reset()`: a window wants the growth, which is the difference of two readings.
wireChars: 0,
wireFrames: 0,
wireParseFailures: 0,
};
- // The incremental SSE buffer. A decode() call is a slice of the socket, not an SSE frame: one
- // call can carry three frames and half of a fourth. Whatever is left after the last blank line
- // stays here until the rest of it arrives.
- //
- // PER DECODER, NOT PER PAGE. A `TextDecoder` belongs to ONE response: the app builds a new one
- // beside its own `buffer` for every streaming request, so no legitimate frame ever spans two of
- // them. A single page-wide buffer therefore did not model reassembly, it modelled the socket,
- // and `stop_generation` exists to cut a socket mid-frame. The abandoned JSON tail then had no
- // `\n\n` to close it, so the next response's first chunk was glued behind it, the merged part
- // failed `startsWith("data:")` and was skipped by `continue` WITHOUT counting a parse failure --
- // the silently short denominator this file's own comments call the unacceptable outcome. Worse,
- // the residue never cleared, so `pending_chars` stayed above zero and `reply_chars_scoreable`
- // refused EVERY window from the abort onwards, and once non-empty it pulled unrelated decoder
- // traffic in through the `pending.length > 0` branch below.
- //
- // Keyed weakly, so a finished response's buffer is collected with its decoder rather than
- // retained. Within one response the same decoder is reused for every `decode(chunk, {stream:
- // true})` call, so a genuine split still reassembles exactly as before.
+ // The incremental SSE buffer: a decode() call is a slice of the socket, so one call can carry
+ // three frames and half of a fourth. PER DECODER, NOT PER PAGE: a `TextDecoder` belongs to one
+ // response, and a page-wide buffer modelled the socket instead of reassembly, so a
+ // `stop_generation` cutting a socket mid-frame glued the next response's first chunk behind an
+ // unclosed JSON tail, skipped without counting a parse failure while `pending_chars` stayed
+ // above zero and refused every later window. Keyed weakly, so the buffer dies with its
+ // decoder.
+ // It failed `startsWith("data:")`.
const DECODER_STATE = new WeakMap();
- //: How many times a frame that was ALREADY BUFFERED when a decode call began was completed and
- //: counted. Cumulative and never reset, for the same reason `wireChars` is not: a window wants
- //: the growth across it. This is what turns "a buffer was pending when the window opened" into
- //: "characters that arrived before this window were counted inside it", which is the thing that
- //: actually makes the denominator wrong. A buffer left behind by an ABORTED response never
- //: completes, so it never increments this and never costs the next response its reading.
- //:
- //: PER DECODER, alongside `pending`, and it was page-global on `S` until it was not. `close`
- //: pairs the buffer pending AT THE OPEN with the flushes carried ACROSS the window, and those
- //: were read at two scopes: the buffer belonged to one decoder, the counter to all of them.
- //: `stop_generation` leaves the old decoder holding half a frame and `send_turn` follows it in
- //: all three shipped schedules, so if that NEW response was itself split -- ordinary, and
- //: likeliest exactly when the renderer is jammed -- its own split moved the shared counter and
- //: the window was refused for a stale buffer that never flushed. Measured before this change: a
- //: two-chunk reply after an abort delivered all 13 of its characters inside the window and was
- //: refused, while the same traffic without the abort was accepted. The loss is biased against
- //: the most expensive windows, in the flattering direction.
+ //: How many times a frame already buffered when a decode call began was completed and counted.
+ //: Cumulative and never reset, like `wireChars`: this turns "a buffer was pending at the open"
+ //: into "characters that arrived before this window were counted inside it". An aborted
+ //: response's buffer never completes, so it never increments this. PER DECODER, like `pending`:
+ //: page-global, `close` paired one decoder's buffer with flushes counted across all of them, so
+ //: a split after a `stop_generation` refused the window for a stale buffer that never flushed.
const CARRIED_BY_ID = new Map();
- //: A bounded history, on the same discipline as `MAX_PENDING_CHARS`. `close` asks about the
- //: decoder pending when the window OPENED, which by then may be neither active nor alive, so
- //: the count cannot live only in the `WeakMap`. Keyed by an integer and holding no reference to
- //: any `TextDecoder`, so it cannot defeat the weak scoping above, and trimmed so a long film
- //: cannot grow it without limit.
+ //: A bounded history, on the same discipline as `MAX_PENDING_CHARS`: `close` asks about the
+ //: decoder pending when the window OPENED, which may be neither active nor alive, so the count
+ //: cannot live only in the `WeakMap`. Keyed by an integer holding no reference to any
+ //: `TextDecoder`, and trimmed so a long film cannot grow it without limit.
const MAX_DECODER_HISTORY = 64;
let DECODER_SEQ = 0;
const noteCarried = (st) => {
@@ -177,9 +95,8 @@
CARRIED_BY_ID.delete(CARRIED_BY_ID.keys().next().value);
}
};
- //: The carried count of a NAMED decoder, or of whichever one is active when no name is given.
- //: A decoder that never carried a frame is absent from the map and answers 0, which is the same
- //: answer it would have given had it still been alive.
+ //: The carried count of a NAMED decoder, or of whichever is active when no name is given. A
+ //: decoder that never carried a frame is absent and answers 0, as it would have while alive.
const carriedFor = (id) => {
if (typeof id !== "number") return active.carriedFlushes;
return CARRIED_BY_ID.get(id) || 0;
@@ -198,23 +115,17 @@
}
return st;
};
- //: The decoder that most recently delivered a chunk, which is the stream a window is measuring.
- //: `wireIntegrity` reports THIS buffer: half a frame of the response being measured is a short
- //: denominator and must refuse the window, while half a frame of a response that was aborted
- //: three slots ago says nothing about it.
- //: Built by `newState()` rather than by hand: a sentinel missing any field a real decoder state
- //: carries reads `undefined` on a window that opens before the first stream, and `undefined` is
- //: not zero. Its id comes from the same sequence, so `carriedFor` answers about it like any other.
+ //: The decoder that most recently delivered a chunk, i.e. the stream a window is measuring.
+ //: `wireIntegrity` reports THIS buffer: half a frame of the measured response is a short
+ //: denominator, while half a frame of a response aborted three slots ago says nothing. Built by
+ //: `newState()` so a window opening before the first stream reads zero, not undefined.
let active = newState();
- //: The decoder holding a speculative marker fragment, if any, and there is only ever one worth
- //: holding. A fragment is at most four characters and lives on the decoder that produced it, but
- //: it has to be REPORTED whoever holds it: a decoder whose first chunk is "dat" has not been
- //: identified as the stream yet, and leaving its fragment out of `wireIntegrity` would be the
- //: instrument saying nothing was outstanding while a frame was. Erring the other way costs at
- //: most four characters of unrelated traffic marking a window unscoreable, which is the
- //: direction this file has always chosen. It is dropped as soon as some OTHER decoder is
- //: identified as the stream, so an unrelated chunk that happened to end in "data" cannot leave a
- //: permanent residue behind it.
+ //: The decoder holding a speculative marker fragment, if any. A fragment is at most four
+ //: characters and lives on the decoder that produced it, but must be reported whoever holds it:
+ //: a decoder whose first chunk is "dat" is not identified as the stream yet, and omitting its
+ //: fragment would claim nothing was outstanding while a frame was. Erring the other way costs
+ //: at most four characters marking a window unscoreable. Dropped once another decoder is the
+ //: stream.
let markerHold = null;
const setMarkerTail = (st, frag) => {
st.markerTail = frag;
@@ -224,26 +135,18 @@
const heldMarkerChars = () =>
active.markerTail.length +
(markerHold && markerHold !== active ? markerHold.markerTail.length : 0);
- // A frame is a few hundred bytes. If this ever grows past a sane bound the stream is not what
- // we think it is, and dropping the buffer is better than growing it without limit inside a hook
- // that runs fourteen times a second.
+ // A frame is a few hundred bytes; past a sane bound the stream is not what we think it is, and
+ // dropping the buffer beats growing it without limit in a hook that runs 14 times a second.
const MAX_PENDING_CHARS = 262144;
- // ── the frame marker, and the halves of it a split can leave ──────────────────────────────────
- //
- // The socket can cut a frame ANYWHERE, including inside these five characters: one decode()
- // returns "da" and the next "ta: {...}\n\n". Neither chunk contains the marker, so the detector
- // below saw two unrelated chunks, discarded both, and lost the frame WITHOUT counting a parse
- // failure -- which left the wire character count short with nothing anywhere to say so, and
- // `reply_chars_scoreable` reporting the window as sound. The denominator going quietly short
- // inflates every cost-per-character above it, and it goes short exactly when the renderer is
- // jammed and chunks arrive ragged, which is the moment the instrument exists to measure.
+ // The socket can cut a frame anywhere, including inside these five characters: one decode()
+ // returns "da" and the next "ta: {...}\n\n". Neither contains the marker, so the detector
+ // discarded both and lost the frame without counting a parse failure, leaving the denominator
+ // short exactly when the renderer is jammed and chunks arrive ragged.
const SSE_MARKER = "data:";
- // The fragment of the marker the last chunk MIGHT have ended on. At most four characters ("d",
- // "da", "dat", "data"), so this cannot become the memory hazard MAX_PENDING_CHARS guards
- // `pending` against: buffering whole unrelated TextDecoder chunks on the chance that one of them
- // is an SSE frame would be a worse bug than the one being fixed.
- // Lives in the per-decoder state above, for the same reason `pending` does.
+ // The fragment of the marker the last chunk might have ended on: at most four characters, so
+ // it cannot become the memory hazard MAX_PENDING_CHARS guards `pending` against. Lives in the
+ // per-decoder state, for the same reason `pending` does.
//: The longest tail of `s` that is a PROPER PREFIX of the marker, or "".
const partialMarkerTail = (s) => {
@@ -253,10 +156,9 @@
return "";
};
- //: Does `s` CONTINUE the marker that `frag` started? This is what keeps a speculative fragment
- //: harmless. Unrelated traffic ending in "d" would otherwise be glued onto the front of the next
- //: chunk, and a real frame arriving there would become "ddata: {...}", which does not start with
- //: the marker and would be skipped in silence -- the same defect one step to the left.
+ //: Does `s` CONTINUE the marker `frag` started? Without this, unrelated traffic ending in "d"
+ //: would be glued onto the next chunk and a real frame would become "ddata: {...}", skipped in
+ //: silence: the same defect one step to the left.
const continuesMarker = (frag, s) => {
const rest = SSE_MARKER.slice(frag.length);
const n = Math.min(rest.length, s.length);
@@ -264,34 +166,20 @@
};
const now = () => performance.now();
- // Takes the instant to judge, rather than reading the clock itself, so a caller that has to
- // attribute a whole INTERVAL can ask about the instant the interval began. See the timer below.
+ // Takes the instant to judge rather than reading the clock, so a caller attributing a whole
+ // INTERVAL can ask about the instant it began. See the timer below.
const streamingAt = (t) => S.lastSseAt > 0 && t - S.lastSseAt < IDLE_GAP_MS;
- // ── the task-chain timer ──────────────────────────────────────────────────────────────────
- //
- // One pending measurement at a time. A burst of chunks delivered in one task must be charged
- // once, from the first chunk to the loop draining, not once per chunk: the pacer sends the
- // whole shortfall in a single burst when it has fallen behind, and charging per chunk would
- // multiply one task chain by the number of chunks that started it.
+ // One pending measurement at a time: a burst delivered in one task must be charged once, from
+ // the first chunk to the loop draining. The pacer sends the whole shortfall in one burst when
+ // behind, and charging per chunk would multiply one chain by the chunks that started it.
let chainStart = null;
// Close whatever chain is open and charge it to the accumulator it was opened against. Called
- // from the MessageChannel callback -- the ordinary end of a chain -- and from `read()`, which
- // is the case that used to lose it.
- //
- // WHY read() HAS TO DO THIS. `read()` arrives on its own task, from the driver, at a window
- // boundary; the message closing the last burst's chain is a task too, and the two are on
- // different task queues, so a window can close in the gap between a burst's decode and its
- // chain's macrotask. The snapshot then takes `deltaTaskMs` without that burst in it and
- // `reset()` zeroes the accumulator, so the callback charges the burst to a fresh one -- which
- // the tail `read(0)` in `StreamCostInstrument.close` discards, and which the next `open()`
- // resets in any case. The burst's characters are counted in the denominator and its targeted
- // cost is silently gone from the numerator. Reproduced against real chromium under a synthetic
- // stream of known per-burst cost: 0 to 2 of 200 bursts lost per run, one per window boundary at
- // most, and always downward.
- //
- // The stale message is harmless: it finds `chainStart === null` and returns. A burst that
- // starts after this posts a message of its own.
+ // from the MessageChannel callback and from `read()`, the case that used to lose it: `read()`
+ // arrives on its own task at a window boundary, on a different queue, so a window could close
+ // between a burst's decode and its chain's macrotask, counting the characters and losing the
+ // cost. Reproduced in chromium: 0 to 2 of 200 bursts lost per run, always downward. A stale
+ // message is harmless, finding `chainStart === null`.
const closeChain = () => {
if (chainStart === null) return;
S.deltaTaskMs += now() - chainStart;
@@ -300,32 +188,16 @@
const chan = new MessageChannel();
chan.port1.onmessage = closeChain;
- // ── the wire-side character counter ───────────────────────────────────────────────────────
- //
- // THIS REPLACES AN O(DOCUMENT) READ THAT BIASED THE COMPARISON, and the bias only became
- // visible once an arm existed that changes the size of the document.
- //
- // The denominator used to be read from the DOM: `querySelectorAll('[data-role="assistant"]')`,
- // last element, `textContent.length`, at both ends of every window. That is O(the whole
- // document) regardless of how few elements match, and it measured 3.9 ms per call against
- // 42,000 elements -- 38.8 ms per cell at 10K and 289.6 ms at 100K. The file's own note said it
- // "is identical on both arms of an A/B and cancels in a paired ratio". That was true of every
- // arm this project had ever run, and it is FALSE for a virtualised one: an arm whose entire
- // purpose is to put a tenth of the elements in the document pays a tenth of this cost, so the
- // instrument hands the treatment a saving it did not earn, in the direction that flatters
- // exactly the hypothesis under test. Nothing measured on such an arm could be quoted while that
- // read was in the paired path.
- //
- // Counting off the wire removes it rather than balancing it. Both arms are fed by the SAME
- // pacer -- that is a design invariant of runtime/ab.py, not a coincidence -- so the bytes are
- // identical by construction and this counter is identical by construction. It is O(the chunk),
- // about fourteen chunks a second, and independent of the thread's size, the rung and the arm.
- //
- // It is also a BETTER denominator than the one it replaces. The DOM read measured the last
- // assistant message, so a `send_turn` mid-film made the reading shrink and the window's growth
- // unmeasurable ("it is a different message"). Characters delivered in the window is the
- // quantity the cost per character actually wants, and it does not care how many messages they
- // were spread over.
+ // THIS REPLACES AN O(DOCUMENT) READ THAT BIASED THE COMPARISON. The denominator used to be a
+ // `querySelectorAll('[data-role="assistant"]')` last-element textContent read at both ends of
+ // every window: 38.8 ms per cell at 10K and 289.6 ms at 100K. "It cancels in a paired ratio"
+ // is FALSE for a virtualised arm, which pays a tenth of the cost and is handed a saving it did
+ // not earn, flattering the hypothesis under test. Counting off the wire removes it: both arms
+ // are fed by the SAME pacer (a runtime/ab.py invariant), so the counter is identical by
+ // construction, O(the chunk), and independent of thread size, rung and arm. It is also the
+ // better denominator: the DOM read shrank on a mid-film `send_turn`.
+ // About 3.9 ms per call against 42,000 elements.
+ // `textContent.length` over the whole document.
const countDeltaChars = (st, text, carriedMarker) => {
const carried = st.pending.length > 0 || Boolean(carriedMarker);
st.pending += text;
@@ -334,11 +206,11 @@
st.pending = "";
return;
}
- // Frames are separated by a blank line. Anything after the last one is incomplete.
+ // Frames are separated by a blank line; anything after the last one is incomplete.
const parts = st.pending.split("\n\n");
st.pending = parts.pop();
- // Something that was in the buffer before this call just became a counted frame. A window
- // whose OPEN saw a non-empty buffer is only wrong if this happens inside it.
+ // Something buffered before this call just became a counted frame. A window whose OPEN saw a
+ // non-empty buffer is only wrong if this happens inside it.
if (carried && parts.length > 0) noteCarried(st);
for (const part of parts) {
const line = part.trim();
@@ -350,16 +222,16 @@
const choices = frame && frame.choices;
if (!choices || !choices.length) continue;
const delta = choices[0].delta || {};
- // Both fields, and both are counted. `_gguf_chat_delta_line` emits reasoning as
- // `reasoning_content` WITH `content: ""` beside it, so summing them is not double
- // counting; it is the two halves of one turn.
+ // Both fields, and both counted: `_gguf_chat_delta_line` emits reasoning as
+ // `reasoning_content` with `content: ""` beside it, so summing them is the two halves of one
+ // turn, not double counting.
const content = typeof delta.content === "string" ? delta.content.length : 0;
const reasoning =
typeof delta.reasoning_content === "string" ? delta.reasoning_content.length : 0;
S.wireChars += content + reasoning;
S.wireFrames += 1;
} catch (err) {
- // COUNTED, NOT SWALLOWED. A parse failure means the denominator is short by an unknown
+ // COUNTED, NOT SWALLOWED: a parse failure means the denominator is short by an unknown
// amount, and a silently short denominator inflates every cost-per-character above it.
S.wireParseFailures += 1;
}
@@ -377,94 +249,65 @@
}
};
- // ── the SSE detector ──────────────────────────────────────────────────────────────────────
- //
// The app reads the relay's response through its own TextDecoder, so decode() is the first
- // main-thread code that sees a chunk. Wrapping it costs one call per chunk and is O(1) in the
- // thread's size. This hook is the flat part of the instrument; the reply-length read below is
- // the part that is not, and its measured cost is recorded there rather than claimed away here.
+ // main-thread code to see a chunk: one call per chunk, O(1) in thread size. This hook is the
+ // flat part of the instrument; the reply-length read below is not, and records its own cost.
const nativeDecode = TextDecoder.prototype.decode;
TextDecoder.prototype.decode = function (input, options) {
const out = nativeDecode.call(this, input, options);
const t = now();
S.decodeCalls += 1;
if (typeof out === "string" && out.length > 0) {
- // Reassembly state belongs to THIS decoder. Whether this decoder is also the stream being
- // MEASURED is not known yet -- that is decided below, once the chunk has been looked at --
- // and promoting it here handed `active` to any decoder in the page. An unrelated one has an
- // empty buffer, so `wireIntegrity` then reported nothing outstanding while an SSE decoder
- // held half a frame: the window closing there published a denominator short by that frame
- // with a clean bill of health, and the window the suffix landed in was handed the whole
- // frame and accepted it. A response that was aborted mid-frame keeps its half frame to
- // itself.
+ // Reassembly state belongs to THIS decoder. Whether it is also the stream being MEASURED is
+ // decided below, once the chunk has been looked at: promoting here handed `active` to any
+ // decoder in the page, so `wireIntegrity` reported nothing outstanding while an SSE decoder
+ // held half a frame.
const st = stateFor(this);
- // The fragment the previous chunk ended on, but only if THIS chunk continues it. A split
+ // The fragment the previous chunk ended on, but only if THIS chunk continues it: a split
// inside the marker is repaired here rather than in the buffer, so a fragment that turns out
- // to be ordinary text ending in "d" is dropped instead of corrupting the frame behind it.
- // `markerTail` is only ever set when `pending` is empty, so the two can never both hold a
- // half of the same frame.
+ // to be ordinary text ending in "d" is dropped. `markerTail` is only set when `pending` is
+ // empty, so the two can never hold halves of the same frame.
const carriedMarker = Boolean(st.markerTail && continuesMarker(st.markerTail, out));
const chunk = carriedMarker ? st.markerTail + out : out;
setMarkerTail(st, "");
- // THE BOUND IS ON THE SCAN, NOT ON THE PAYLOAD. A chunk at or under the cap is its own head,
- // so nothing is allocated on the ordinary path and the ordinary path is unchanged. Over the
- // cap, v8 slices a string by reference rather than by copy, so the head costs no walk of the
- // payload either. Both things the cap was for survive: the search stays bounded by a constant
- // that grows with neither the payload nor the rung, and a bundle, a blob or a paste still has
- // to put the marker in its first MAX_SSE_CHUNK_CHARS characters to be mistaken for a stream.
- // What does NOT survive is the old rejection: a read carries everything the browser buffered
- // since the last one, so a stall past about two seconds at fast cadence hands the app one
- // well-formed 97,500 character batch, and dropping it took the largest burst of the stream
- // out of `sseChunks`, `lastSseAt` and the `deltaTaskMs` numerator at the moment a stall makes
- // it largest.
+ // THE BOUND IS ON THE SCAN, NOT THE PAYLOAD. A chunk at or under the cap is its own head, so
+ // the ordinary path allocates nothing, and over the cap v8 slices by reference. Both original
+ // goals survive: a constant-bounded search, and a bundle must still put the marker in its
+ // first MAX_SSE_CHUNK_CHARS characters. What does not survive is the old rejection, which
+ // dropped a well-formed 97,500 character batch at the moment a stall made it largest.
const head = chunk.length <= MAX_SSE_CHUNK_CHARS ? chunk : chunk.slice(0, MAX_SSE_CHUNK_CHARS);
const looksSse = head.indexOf(SSE_MARKER) >= 0;
- // A CONTINUATION IS STREAM TRAFFIC, AND IT STARTS A TASK CHAIN LIKE ANY OTHER CHUNK.
- // `noteSse` was gated on the marker alone while the counter below was gated on
- // `looksSse || pending`, so the tail of a frame the socket cut inside its JSON body was
- // COUNTED in the denominator and charged to nothing. The three quantities it skipped are
- // the three the batched-read note above names: `sseChunks`, `lastSseAt` and the
- // `deltaTaskMs` numerator. When the suffix lands on a later task the first half rendered
- // nothing and its chain has already closed, so the render the suffix does start is charged
- // to no window at all, and `stream_delta_cost_ms_per_kchar` is biased DOWNWARD exactly as
- // fragmentation rises -- the mirror of the denominator defect the comment below describes,
- // and in the flattering direction. A stale `lastSseAt` also lets `replyChars` decide the
- // stream has gone idle and return `null` for a window still carrying it.
+ // A CONTINUATION IS STREAM TRAFFIC AND STARTS A TASK CHAIN LIKE ANY OTHER CHUNK. `noteSse`
+ // was gated on the marker alone while the counter was gated on `looksSse || pending`, so the
+ // tail of a frame cut inside its JSON body was counted in the denominator and charged to
+ // nothing, biasing `stream_delta_cost_ms_per_kchar` DOWNWARD as fragmentation rises. A stale
+ // `lastSseAt` also lets `replyChars` call the stream idle for a window still carrying it.
const continuesFrame = st.pending.length > 0;
if (looksSse || continuesFrame) noteSse();
- // `looksSse || pending` and not just `looksSse`. THE SECOND HALF OF A SPLIT FRAME CONTAINS
- // NO "data:" -- it is the tail of a JSON body and a blank line -- so gating the counter on
- // that marker dropped the whole frame whenever the socket cut one in two. Found by
- // test_the_counter_survives_a_frame_split_across_two_decode_calls, which is precisely the
- // condition the instrument exists to measure: chunks arrive ragged when the renderer is
- // jammed, so the denominator would have gone quietly short exactly where the numerator went
- // up, and the cost per character would have been overstated at the worst moment.
- //
- // Once a partial frame is held, every subsequent chunk is fed until it completes. Unrelated
- // TextDecoder traffic can therefore land in the buffer; it cannot be counted, because it
- // will not parse as a frame, and it is bounded by MAX_PENDING_CHARS and reported through
- // wire_parse_failures rather than absorbed.
- //
- // AND THE CHUNK THAT IS NEITHER is kept only as far as it could be the START of a marker.
- // That is the third case, the one a complete-marker test cannot see: "da" carries no marker
- // and completes no buffered frame, and discarding it loses the frame that arrives next.
+ // `looksSse || pending`, not just `looksSse`: THE SECOND HALF OF A SPLIT FRAME CONTAINS NO
+ // "data:", so gating the counter on the marker dropped the whole frame whenever the socket cut
+ // one in two. Chunks arrive ragged when the renderer is jammed, so the denominator went short
+ // exactly where the numerator went up. Once a partial frame is held every chunk is fed until
+ // it completes; unrelated traffic is bounded by MAX_PENDING_CHARS and reported through
+ // wire_parse_failures. A chunk that is neither is kept only as far as it could START a marker:
+ // "da" carries no marker and completes no frame, and discarding it loses the next frame.
+ // Found by test_the_counter_survives_a_frame_split_across_two_decode_calls.
// The whole chunk is fed to the counter and only the SCAN was bounded above: the denominator
// is characters delivered, so counting a batched read's head would understate it by exactly
// the amount a stall made it large.
- // AND ONLY NOW IS THIS DECODER THE ONE A WINDOW IS MEASURING: it either carries the relay's
- // framing or is completing a frame of its own. A decoder that is neither cannot take
- // `active` away from one that is.
+ // ONLY NOW IS THIS DECODER THE ONE A WINDOW IS MEASURING: it either carries the relay's
+ // framing or is completing a frame of its own, and a decoder that is neither cannot take
+ // `active` from one that is.
if (looksSse || continuesFrame) {
- // A fragment held by a DIFFERENT decoder cannot be part of this stream, and this one is
- // the stream. Dropped rather than carried, so unrelated traffic that ended in "data"
- // cannot report an outstanding frame for the rest of the cell.
+ // A fragment held by a DIFFERENT decoder cannot be part of this stream. Dropped rather than
+ // carried, so unrelated traffic ending in "data" cannot report an outstanding frame.
if (markerHold && markerHold !== st) setMarkerTail(markerHold, "");
active = st;
countDeltaChars(st, chunk, carriedMarker);
} else {
- // A speculative fragment is kept on the decoder that produced it, and it is reported
- // through `wireIntegrity` only if that decoder is the active stream. Unrelated traffic
- // ending in "d" is therefore held without ever claiming to be an outstanding frame.
+ // A speculative fragment stays on the decoder that produced it and is reported through
+ // `wireIntegrity` only if that decoder is the active stream, so unrelated traffic ending in
+ // "d" never claims an outstanding frame.
setMarkerTail(st, partialMarkerTail(chunk));
}
}
@@ -472,26 +315,18 @@
return out;
};
- // ── blocked time while streaming ──────────────────────────────────────────────────────────
- //
// The same 1 ms timer discipline as frames.js, and deliberately the same calibrated clamp:
- // blocked time is a SUBTRACTION against an idle floor, and two instruments subtracting two
- // different floors would report two different amounts of block for one page. The clamp is read
- // from frames.js rather than calibrated again here, so if frames.js could not establish one
- // this reports null with frames.js's own reason.
+ // blocked time is a subtraction against an idle floor, and two instruments subtracting
+ // different floors would report two amounts of block for one page. The clamp is read from
+ // frames.js, so if none could be established this reports null with frames.js's reason.
let lastTick = now();
const tick = () => {
const t = now();
const gap = t - lastTick;
- // Attributed by the state at the START of the interval, never at its end. A timer callback
- // cannot run while the main thread is blocked -- it is queued when the timer expires and
- // waits for the stack to empty -- so a stall is only ever observed once it is already over.
- // Reading `streaming()` here would therefore ask whether the stream is in flight NOW, after
- // the stall, and a stall longer than IDLE_GAP_MS answers no: the whole interval, which IS
- // the stall, would be dropped from both accumulators. That discards precisely the worst
- // stream-induced stalls, and it discards them at exactly the point where a regression grows
- // past 1.5 s, so a build getting worse would read cheaper. A negative difference (a chunk
- // that arrived DURING the interval) is below the threshold too, which is the overlap case.
+ // Attributed by the state at the START of the interval, never at its end: a timer callback
+ // cannot run while the main thread is blocked, so a stall is only observed once over, and
+ // reading `streaming()` here would drop the whole interval for any stall longer than
+ // IDLE_GAP_MS, discarding the worst stream-induced stalls so a worsening build reads cheaper.
const wasStreaming = streamingAt(lastTick);
lastTick = t;
if (wasStreaming) {
@@ -504,25 +339,13 @@
};
setTimeout(tick, 1);
- // Characters of the reply currently being streamed. The LAST assistant message only, not the
- // whole thread: at the 100K rung the thread holds about 190,000 assistant characters and
- // reading all of them costs real time on every window boundary, while the quantity wanted is
- // the growth of the one reply the pacer is feeding. Reading the last message alone is O(reply)
- // rather than O(thread), so this stays flat as the rung climbs.
- //
- // It is also the only reading that survives the end of the film. `thread_reopen` and
- // `delete_message` rebuild and then cut the thread, so a whole-thread character count jumps and
- // then falls for reasons that have nothing to do with streaming.
- // MEASURED, and it is not free. `querySelectorAll` is O(the whole DOM), not O(the reply), so
- // this is the one part of the instrument whose cost grows with the rung. Called at both ends of
- // every window it totalled 38.8 ms per cell at 10K and 289.6 ms at 100K -- about 3.9 ms per
- // call against 42,000 elements.
- //
- // Most of that was spent on windows with no stream in them. The standard film opens about
- // thirty-seven windows and only eight carry streaming, so the read is skipped once the stream
- // has been finished for longer than the idle gap. `null` is returned rather than a stale count,
- // and a window that then turns out to have carried traffic reports its growth as unmeasurable
- // with a reason instead of inventing a delta from a reading that was never taken.
+ // Characters of the reply currently streaming: the LAST assistant message only, since at 100K
+ // the thread holds about 190,000 assistant characters while the quantity wanted is the growth
+ // of the one reply the pacer is feeding. O(reply) rather than O(thread), and the only reading
+ // that survives `thread_reopen` and `delete_message`. Not free: `querySelectorAll` is O(the
+ // whole DOM), 289.6 ms per cell at 100K, and most of it was spent on windows with no stream,
+ // so the read is skipped once the stream has been idle past the gap; `null` is returned rather
+ // than a stale count, and a window that did carry traffic reports unmeasurable with a reason.
const replyChars = (force) => {
if (!force && S.lastSseAt > 0 && now() - S.lastSseAt >= IDLE_GAP_MS) return null;
const all = document.querySelectorAll('[data-role="assistant"]');
@@ -533,12 +356,12 @@
window.__sb.streamcost = {
// Drain the window. `elapsedMs` is the DRIVER's measure, passed in for the same reason
- // frames.js takes it: the page cannot be trusted to read its own clock promptly in the very
- // windows this is measuring.
+ // frames.js takes it: the page cannot be trusted to read its own clock promptly in exactly
+ // the windows this is measuring.
read(elapsedMs) {
const t = now();
- // BEFORE the snapshot, because a chain still in flight belongs to the window that started
- // it and `reset()` below is about to throw it away. See `closeChain`.
+ // BEFORE the snapshot, because a chain still in flight belongs to the window that started it
+ // and `reset()` is about to throw it away. See `closeChain`.
closeChain();
const f = window.__sb.frames;
const clampInfo = f && f.clamp ? f.clamp() : { clampMs: null, reason: "frames.js absent" };
@@ -547,8 +370,8 @@
sse_chunks_attempted: true,
sse_bursts: S.sseBursts,
decode_calls: S.decodeCalls,
- // Never a bare zero: a window with no streaming in it says so with a flag, and the
- // scoring layer skips it rather than folding a zero cost into the numerator.
+ // Never a bare zero: a window with no streaming says so with a flag, and the scoring layer
+ // skips it rather than folding a zero cost into the numerator.
streaming_observed: S.sseChunks > 0,
streaming_ms: Math.round(S.streamingMs * 10) / 10,
streaming_ms_attempted: true,
@@ -558,9 +381,8 @@
clamp_ms: clampInfo.clampMs === null ? null : clampInfo.clampMs,
};
if (clampInfo.clampMs === null || clampInfo.clampMs === undefined) {
- // Blocked time is a subtraction against the idle floor. Without a floor there is no
- // subtraction to make, and reporting the raw lag instead would be a different quantity
- // wearing this one's name.
+ // Blocked time is a subtraction against the idle floor; without a floor there is no
+ // subtraction to make, and the raw lag would be a different quantity wearing this one's name.
out.stream_blocked_ms = null;
out.stream_blocked_ms_reason =
"no timer clamp was established, so there is no idle floor to subtract: " +
@@ -576,34 +398,25 @@
return out;
},
- // THE DENOMINATOR, read off the wire. O(1): it returns a counter the decode hook maintains.
- // Called at window open and window close, so the window's growth is the difference.
- //
- // Cumulative since page load and monotonic, so unlike the DOM reading it replaces there is no
- // "the reply shrank, so it is a different message" case to handle: characters delivered only
- // ever go up.
- // The two things that can make `wireChars` short by an unknown amount, read at the same O(1)
- // cost as the counter itself so a window boundary can capture both ends.
- // `forId` names the decoder to answer about, which `close` passes back from what `open`
- // sampled. WHOSE BUFFER AND WHOSE FLUSH HAVE TO BE THE SAME DECODER, and reading the active
- // one at both ends does not make them so: the decoder pending at the open can complete its
- // carried frame inside the window and be replaced as active before the close, and comparing
- // the two ends' ids would call that a mismatch and discard the very carry it looks for.
- // Naming the decoder answers regardless of who is active now, so a window that really did
- // count characters delivered before it opened is still refused.
+ // THE DENOMINATOR, read off the wire. O(1), called at window open and close so the growth is
+ // the difference. Cumulative since page load and monotonic, so unlike the DOM reading it
+ // replaces there is no "the reply shrank, so it is a different message" case.
+ // The two things that can make `wireChars` short by an unknown amount, at the counter's O(1)
+ // cost. `forId` names the decoder to answer about, because the buffer and the flush must
+ // belong to the SAME decoder: the one pending at the open can complete its carried frame
+ // inside the window and be replaced as active before the close, so comparing the two ends'
+ // ids would discard the very carry it looks for.
wireIntegrity(forId) {
// The marker fragment counts as buffered, because it is: the frame it begins has not been
- // counted yet, so a window closing on it has a denominator that is short by that frame. The
- // cost of being honest here is that a stray one to four characters of unrelated traffic can
- // mark a window unscoreable, which is the direction to err in -- an unscoreable window is
- // "we could not tell", and a silently short denominator is "it was fine".
+ // counted, so a window closing on it is short by that frame. The cost is that one to four
+ // characters of unrelated traffic can mark a window unscoreable, the right way to err.
return {
failures: S.wireParseFailures,
pending_chars: active.pending.length + heldMarkerChars(),
// WHICH decoder the two numbers above are about, so the close can ask about the same one.
decoder_id: active.id,
- // Read as a DELTA across the window by `StreamCostInstrument.close`, so a buffer that was
- // pending at the open refuses the window only when ITS OWN frame was completed inside it.
+ // Read as a DELTA across the window by `StreamCostInstrument.close`, so a buffer pending at
+ // the open refuses the window only when its own frame was completed inside it.
carried_flushes: carriedFor(forId),
};
},
@@ -611,14 +424,10 @@
return S.wireChars;
},
- // The OLD reading, kept as a cross-check and NEVER called inside a measured window. See
- // `end_cell` in streamcost.py: it runs once per cell, after the film, where its cost is
- // charged to nothing.
- //
- // Worth keeping rather than deleting, because the two numbers answer different questions and
- // a disagreement between them is a finding: the wire count is what the app was SENT and the
- // DOM count is what it RENDERED. On a windowed arm they are expected to disagree, by exactly
- // the messages that are not mounted.
+ // The OLD reading, kept as a cross-check and never called inside a measured window (see
+ // `end_cell` in streamcost.py, once per cell after the film). Worth keeping because the two
+ // answer different questions, what the app was SENT versus what it RENDERED, and on a windowed
+ // arm they should disagree by exactly the unmounted messages.
replyCharsDom(force) {
const t = now();
const n = replyChars(Boolean(force));
@@ -646,7 +455,7 @@
},
// For the selftest: force the detector into the streaming state without a real stream, so a
- // synthetic injection can be measured on exactly the accumulators a real stream uses.
+ // synthetic injection is measured on the accumulators a real stream uses.
__markStreaming() {
noteSse();
},
diff --git a/tests/studio/studiobench/instruments/streamcost.py b/tests/studio/studiobench/instruments/streamcost.py
index 09518632a65..c2db6423920 100644
--- a/tests/studio/studiobench/instruments/streamcost.py
+++ b/tests/studio/studiobench/instruments/streamcost.py
@@ -66,34 +66,30 @@ class StreamCostInstrument(_PageInstrument):
self._overhead_ms = 0.0
def start_cell(self, cell: Cell) -> None:
- # PER CELL, like every other instrument that declares overhead (heap.py, tracing.py,
- # coverage.py all zero theirs here). One instrument instance serves the whole session, so
- # an accumulator carried across cells reports cell k's overhead as the sum of cells 1..k.
- # The rung ladder is run in ascending order, so that sum climbs with the rung and
- # `overhead_growth_with_length` -- the gate whose entire job is to catch an instrument
- # whose cost tracks the treatment -- would read manufactured growth off a flat instrument.
+ # PER CELL, like every other instrument that declares overhead. One instrument instance serves the
+ # whole session, so an accumulator carried across cells reports cell k's overhead as the sum of
+ # cells 1..k; the rung ladder runs ascending, so `overhead_growth_with_length`, the gate whose job
+ # is to catch an instrument whose cost tracks the treatment, would read manufactured growth off a
+ # flat instrument.
super().start_cell(cell)
self._overhead_ms = 0.0
self._chars_open = None
def open(self, window: Window) -> None:
- # Drain first, so the window starts from zero even if the previous close did not run
- # (an instrument that raised is disabled for the rest of the cell, and the accumulator
- # would otherwise carry that cell's remaining traffic into this window).
+ # Drain first, so the window starts from zero even if the previous close did not run (an
+ # instrument that raised is disabled for the rest of the cell, and the accumulator would carry
+ # that cell's remaining traffic into this window).
self._eval("() => window.__sb.streamcost && window.__sb.streamcost.reset()")
- # O(1), off the wire. This used to be a `querySelectorAll` over the whole document at
- # both ends of every window; see the long note in streamcost.js for why that had to go.
+ # O(1), off the wire. This used to be a `querySelectorAll` over the whole document at both ends of
+ # every window; see the note in streamcost.js.
self._chars_open = self._eval(
"() => window.__sb.streamcost && window.__sb.streamcost.replyChars()"
)
- # THE DENOMINATOR'S INTEGRITY, SAMPLED AT BOTH ENDS. A frame that cannot be parsed --
- # unrelated `TextDecoder` traffic appended while `pending` holds a split SSE frame, say --
- # increments a diagnostic and leaves `wireChars` short by an amount nobody can recover.
- # Counting the failures was not enough on its own: nothing consulted the count, so the
- # affected `reply_chars_delta` was still accepted as a denominator and every cost-per-
- # character derived from it came out inflated by an unknown factor. A delta spanning a new
- # failure, or ending with an unterminated frame still buffered, is now marked unscoreable
- # at the window that contains it.
+ # THE DENOMINATOR'S INTEGRITY, SAMPLED AT BOTH ENDS. A frame that cannot be parsed increments a
+ # diagnostic and leaves `wireChars` short by an unrecoverable amount. Counting failures was not
+ # enough: nothing consulted the count, so the affected `reply_chars_delta` was still accepted and
+ # every cost-per-character came out inflated. A delta spanning a new failure, or ending with an
+ # unterminated frame still buffered, is now marked unscoreable.
self._integrity_open = (
self._eval("() => window.__sb.streamcost && window.__sb.streamcost.wireIntegrity()")
or {}
@@ -109,26 +105,22 @@ class StreamCostInstrument(_PageInstrument):
"unavailable": self.unavailable or "the page did not answer",
"stream_cost_attempted": False,
}
- # No `force` argument any more. The old DOM read was skipped on windows long past the
- # stream because it was expensive; this one is a counter read, so it is taken on every
- # window unconditionally and the "the open-read was skipped, so the pair is half-taken"
- # case cannot arise.
+ # No `force` argument any more. The old DOM read was skipped on windows long past the stream
+ # because it was expensive; this is a counter read, taken unconditionally, so the half-taken pair
+ # cannot arise.
chars_close = self._eval("() => window.__sb.streamcost.replyChars()")
out["stream_cost_attempted"] = True
out["reply_chars_open"] = self._chars_open
out["reply_chars_close"] = chars_close
- # WHERE THE DENOMINATOR CAME FROM, in the payload rather than in this file. A reader
- # comparing a run recorded before this change with one recorded after is comparing two
- # different quantities -- the growth of the last mounted message against the characters
- # delivered to the page -- and nothing else in the row would tell them so.
+ # WHERE THE DENOMINATOR CAME FROM, in the payload rather than in this file: a reader comparing
+ # runs recorded before and after this change is comparing the growth of the last mounted message
+ # against the characters delivered to the page.
out["reply_chars_source"] = "sse_wire"
- # ABOUT THE DECODER THAT WAS PENDING AT THE OPEN, named rather than assumed. The refusal
- # below pairs a buffer with a flush, and the two were read at different scopes: the buffer
- # belonged to one decoder, the flush counter was page-wide. Since `send_turn` follows
- # `stop_generation` in every shipped schedule, an abort's orphaned half frame was routinely
- # paired with a carried flush produced by the NEW response's own split, refusing a window
- # that had delivered every one of its characters. Naming the open's decoder makes the pair
- # a statement about one response.
+ # ABOUT THE DECODER THAT WAS PENDING AT THE OPEN, named rather than assumed. The refusal below
+ # pairs a buffer with a flush, and the two were read at different scopes: the buffer belonged to
+ # one decoder, the flush counter was page-wide. Since `send_turn` follows `stop_generation` in
+ # every shipped schedule, an abort's orphaned half frame was routinely paired with a carried
+ # flush from the NEW response's split, refusing a window that had delivered every character.
integrity = (
self._eval(
"(id) => window.__sb.streamcost.wireIntegrity(id)",
@@ -138,25 +130,19 @@ class StreamCostInstrument(_PageInstrument):
)
failures = (integrity.get("failures") or 0) - (self._integrity_open.get("failures") or 0)
residual = integrity.get("pending_chars") or 0
- # AND THE OTHER END OF THE SAME SPLIT. `pending` is not cleared by `reset()` -- it cannot
- # be, it holds half a frame -- so a frame the socket cut across a window boundary is still
- # buffered when the NEXT window opens. Its suffix arrives inside that window, the parser
- # adds the WHOLE frame's characters there and empties the buffer, and the close reading
- # then sees no failures and no residual and calls the window scoreable. Part of its
- # denominator was delivered before it opened. The window that closed on the split is
- # already refused by `residual`; this refuses its partner, and the two together are what
- # make `reply_chars_delta` mean "characters delivered IN this window" rather than
- # "characters counted in this window".
- #
- # AND IT IS THE FLUSH, NOT THE BUFFER, THAT MAKES THE DELTA WRONG. `open()` samples the
- # integrity before the action has created the response it is about to measure, so the
- # buffer it sees can belong to a response that is already over: `stop_generation` exists to
- # cut a socket mid-frame and `send_turn` follows it in every shipped schedule. That half
- # frame is never completed and never counted anywhere, so it takes nothing out of this
- # window's delta -- refusing on its presence alone threw away the one stream-cost reading
- # the send_turn window has. `carried_flushes` counts the completions of frames that were
- # already buffered when the decode began, so the pair "a buffer was pending at the open"
- # and "a carried frame was counted inside the window" is what the refusal now needs.
+ # AND THE OTHER END OF THE SAME SPLIT. `pending` is not cleared by `reset()` (it holds half a
+ # frame) so a frame cut across a window boundary is still buffered when the NEXT window opens;
+ # its suffix arrives inside that window, the parser adds the WHOLE frame's characters there, and
+ # the close sees no failures and calls the window scoreable although part of its denominator was
+ # delivered before it opened. The window that closed on the split is already refused by
+ # `residual`; this refuses its partner, and together they make `reply_chars_delta` mean
+ # 'delivered IN this window'.
+ # AND IT IS THE FLUSH, NOT THE BUFFER, THAT MAKES THE DELTA WRONG. `open()` samples integrity
+ # before the action has created the response, so the buffer it sees can belong to a response
+ # that is already over: `stop_generation` cuts a socket mid-frame and `send_turn` follows it.
+ # That half frame is never completed or counted, so refusing on its presence alone threw away
+ # the one stream-cost reading the send_turn window has. `carried_flushes` counts completions of
+ # frames already buffered when the decode began, so the refusal needs both halves of the pair.
pending_at_open = self._integrity_open.get("pending_chars") or 0
carried = (integrity.get("carried_flushes") or 0) - (
self._integrity_open.get("carried_flushes") or 0
@@ -192,13 +178,10 @@ class StreamCostInstrument(_PageInstrument):
"than the idle gap when the window opened"
)
elif chars_close < self._chars_open:
- # NOW UNREACHABLE, AND KEPT ANYWAY. The wire counter is cumulative and monotonic, so
- # it cannot go backwards; if it ever does, the instrument has been reset underneath
- # itself and the delta is meaningless. Under the old DOM reading this branch fired
- # legitimately and often: `send_turn` starts a new assistant message and
- # `thread_reopen` rebuilds the thread, so the LAST message was routinely shorter at
- # the close of a window than at its open, and those windows lost their denominator
- # for a reason that had nothing to do with streaming.
+ # NOW UNREACHABLE, AND KEPT ANYWAY: the wire counter is cumulative and monotonic, so going
+ # backwards means the instrument was reset underneath itself and the delta is meaningless. Under
+ # the old DOM reading this fired legitimately and often, because `send_turn` starts a new
+ # assistant message and `thread_reopen` rebuilds the thread.
out["reply_chars_delta"] = None
out["reply_chars_delta_reason"] = (
f"the wire character counter went backwards, from {self._chars_open} to "
@@ -211,18 +194,13 @@ class StreamCostInstrument(_PageInstrument):
self._overhead_ms += float(out.get("overhead_ms") or 0.0)
- # THE CLOSE-SIDE SCAN IS NOT FREE AND WAS NOT COUNTED. `read()` snapshots `overhead_ms`
- # into its result and then calls `reset()`, so the `replyChars()` above -- which at close
- # is FORCED whenever the window carried traffic, and is the `querySelectorAll` that is
- # O(the whole DOM) rather than O(the reply) -- accumulated into a page-side total that
- # nothing ever read: the next `open()` begins by resetting it. Exactly half of every
- # window's boundary scans were therefore missing from the one number whose job is to make
- # the level 0 claim checkable from the payload rather than from a docstring, and the half
- # that was missing is the rung-dependent half. Drained here, after the scan, rather than
- # by reordering the pair: the close read needs `force`, and `force` is
- # `streaming_observed`, which only `read()` can answer. Nothing else is lost by the second
- # drain -- every other accumulator it clears was already snapshotted above, and `open()`
- # clears them again before the next window.
+ # THE CLOSE-SIDE SCAN IS NOT FREE AND WAS NOT COUNTED. `read()` snapshots `overhead_ms` and then
+ # calls `reset()`, so the `replyChars()` above (forced at close whenever the window carried
+ # traffic, and O(the whole DOM)) accumulated into a page-side total nothing ever read, since the
+ # next `open()` resets it. Half of every window's boundary scans were missing from the one number
+ # that makes the level 0 claim checkable, and the missing half is the rung-dependent one. Drained
+ # here rather than by reordering the pair, because the close read needs `force`, which only
+ # `read()` can answer.
tail = self._eval("() => window.__sb.streamcost.read(0)")
close_scan_ms = float(tail.get("overhead_ms") or 0.0) if isinstance(tail, dict) else 0.0
self._overhead_ms += close_scan_ms
@@ -233,16 +211,12 @@ class StreamCostInstrument(_PageInstrument):
return out
def end_cell(self, cell: Cell) -> Optional[dict]:
- # Declared even though level 0 is not obliged to: the whole argument for calling this
- # level 0 is that its cost does not grow with the rung, and a claim like that should be
- # checkable from the payload rather than from this docstring.
- #
- # THE ONE DOM READ, and it happens HERE, which is after the last window of the cell has
- # closed and before the next cell's first one opens. Its cost is charged to no window and
- # therefore to no arm. It exists because the wire count and the DOM count answer different
- # questions -- what the app was sent, and what it rendered -- and their disagreement is
- # the cheapest available check that a windowed arm is dropping text rather than merely
- # not mounting it.
+ # Declared even though level 0 is not obliged to: the whole argument for calling this level 0 is
+ # that its cost does not grow with the rung, and that should be checkable from the payload.
+ # THE ONE DOM READ, after the cell's last window closed and before the next cell's first opens, so
+ # its cost is charged to no window and no arm. It exists because the wire count and the DOM count
+ # answer different questions, what the app was sent and what it rendered, and their disagreement
+ # is the cheapest check that a windowed arm is dropping text rather than merely not mounting it.
wire = self._eval("() => window.__sb.streamcost.wireStats()") or {}
dom_chars = self._eval("() => window.__sb.streamcost.replyCharsDom(true)")
return {
diff --git a/tests/studio/studiobench/instruments/tracing.py b/tests/studio/studiobench/instruments/tracing.py
index 6c7b5a3e99a..23717693e7d 100644
--- a/tests/studio/studiobench/instruments/tracing.py
+++ b/tests/studio/studiobench/instruments/tracing.py
@@ -42,19 +42,18 @@ from typing import Any, Callable, Sequence
from ..analysis import CellFailure
+
+# L0 is the only level headline numbers may come from: nothing is attached beyond the renderer's own metrics counters.
# --------------------------------------------------------------------- ladder
-# L0 is the only level headline numbers may come from. Nothing is attached to
-# the renderer beyond the metrics counters it already maintains for itself.
L0 = "L0"
-# L1 adds the timeline trace: task boundaries, frames, layout, user timing. No
-# CPU profiler, so no stacks, so no naming, but the cheapest level that gives a
-# real task tree.
+# L1 adds the timeline trace (task boundaries, frames, layout, user timing). No CPU profiler,
+# so no stacks and no naming, but the cheapest level giving a real task tree.
L1 = "L1"
# L2 adds the V8 CPU profiler. This is the level that NAMES A FRAME.
L2 = "L2"
-# L3 adds precise coverage and heap sampling. Every timing from L3 is discarded
-# by construction; only integers cross the boundary.
+# L3 adds precise coverage and heap sampling. Every timing from L3 is discarded by
+# construction; only integers cross the boundary.
L3 = "L3"
LEVELS = (L0, L1, L2, L3)
@@ -64,32 +63,29 @@ _TIMELINE_CATEGORIES: tuple[str, ...] = (
"disabled-by-default-devtools.timeline",
"disabled-by-default-devtools.timeline.frame",
"blink.user_timing",
- # `toplevel` carries `ThreadControllerImpl::RunTask` with `src_file` /
- # `src_func`, which is the C++ that POSTED each task. `RunTask` itself has
- # empty args and says nothing about origin.
+ # `toplevel` carries `ThreadControllerImpl::RunTask` with `src_file` / `src_func`, the C++ that
+ # POSTED each task; `RunTask` itself says nothing about origin.
"toplevel",
"toplevel.flow",
- # `scheduler` is the one that matters most and it is easy to miss. It does
- # not add events; it adds TYPED ARGS to the `toplevel` slice:
- # `args.renderer_main_thread_task_execution.task_type` and
- # `args.sequence_manager_task.queue_name`. Those turn task origin from an
- # inference into a read value: a React scheduler callback arrives labelled
- # `TASK_TYPE_*POSTED_MESSAGE` on `FRAME_PAUSABLE_TQ`, and a timer arrives
- # labelled `TASK_TYPE_JAVASCRIPT_TIMER_*`. Without this category both are
- # just "a mojo pipe task" and have to be told apart by guesswork.
+ # `scheduler` matters most and is easy to miss: it adds no events, only TYPED ARGS to the
+ # `toplevel` slice (task_type, queue_name), which turn task origin from an inference into a
+ # read value. A React scheduler callback arrives labelled `TASK_TYPE_*POSTED_MESSAGE` on
+ # `FRAME_PAUSABLE_TQ`, a timer as `TASK_TYPE_JAVASCRIPT_TIMER_*`.
+ # Spelled `args.renderer_main_thread_task_execution.task_type` and
+ # `args.sequence_manager_task.queue_name`.
"scheduler",
- # `sequence_manager` adds the DoWork / SelectNextTask / DoIdleWork scoping
- # slices. It does NOT carry queue names, contrary to a natural reading.
+ # `sequence_manager` adds the DoWork / SelectNextTask / DoIdleWork scoping slices. It does NOT
+ # carry queue names, contrary to a natural reading.
"sequence_manager",
"latencyInfo",
"benchmark",
"input",
)
-# `disabled-by-default-devtools.timeline.stack` attaches call-site information
-# to timeline events. `disabled-by-default-v8.cpu_profiler` is the one that
-# produces `Profile` / `ProfileChunk`, and without it every microsecond stays a
-# bucket with no stack.
+# `...timeline.stack` attaches call-site information to timeline events;
+# `disabled-by-default-v8.cpu_profiler` is what produces `Profile` / `ProfileChunk`, and
+# without it every microsecond stays a bucket with no stack.
+# In full, `disabled-by-default-devtools.timeline.stack`.
_STACK_CATEGORIES: tuple[str, ...] = (
"disabled-by-default-devtools.timeline.stack",
"disabled-by-default-v8.cpu_profiler",
@@ -97,19 +93,15 @@ _STACK_CATEGORIES: tuple[str, ...] = (
"disabled-by-default-v8.gc",
)
-# There is NO knob for the tracing CPU profiler's sampling interval.
-# `v8/src/profiler/tracing-cpu-profiler.cc` hard-codes 100 us when the
-# `disabled-by-default-v8.cpu_profiler` category is enabled, and the
-# `disabled-by-default-v8.cpu_profiler.hires` category that used to lower it is
-# no longer registered in V8's category list at all: DevTools still sends it,
-# and it does nothing. Effective spacing measured on a real capture is ~150 us
-# once sampler overhead and idle are included.
-#
-# The consequence is structural and cannot be tuned away: a set of 40 us
-# scheduler tasks yields at most one sample each, so leaf rankings over short
-# task windows are underpowered. That is handled honestly by
-# `analysis.cpuprofile.self_time_in_windows`, which reports `underpowered`
-# rather than by a category flag that does not work.
+# There is NO knob for the tracing CPU profiler's sampling interval:
+# `v8/src/profiler/tracing-cpu-profiler.cc` hard-codes 100 us, and the `...cpu_profiler.hires`
+# category that used to lower it is no longer registered in V8's category list, so DevTools
+# still sends it and it does nothing. Measured spacing on a real capture is ~150 us.
+# That category is `disabled-by-default-v8.cpu_profiler.hires`.
+# The consequence is structural: a set of 40 us scheduler tasks yields at most one sample each,
+# so leaf rankings over short task windows are underpowered.
+# `analysis.cpuprofile.self_time_in_windows` reports `underpowered` rather than relying on a
+# category flag that does not work.
TRACING_PROFILER_INTERVAL_US = 100
CATEGORIES_BY_LEVEL: dict[str, tuple[str, ...]] = {
@@ -119,19 +111,18 @@ CATEGORIES_BY_LEVEL: dict[str, tuple[str, ...]] = {
L3: _TIMELINE_CATEGORIES + _STACK_CATEGORIES,
}
-# The default perfetto buffer is 200 MB. A long rung at L2 produces a lot of
-# ProfileChunks, so the buffer is set explicitly and large: an overflow costs a
-# whole cell, and memory is cheaper than a re-run.
+# The default perfetto buffer is 200 MB. A long rung at L2 produces many ProfileChunks, so the
+# buffer is set explicitly and large: an overflow costs a whole cell, and memory is cheaper
+# than a re-run.
DEFAULT_BUFFER_KB = 640 * 1024
-# `bufferUsageReportingInterval` is clamped to a 250 ms floor in
-# `tracing_handler.cc` (`kMinimumReportingInterval`), so asking for less is
-# silently ignored. 500 ms is what the DevTools frontend itself uses.
+# `bufferUsageReportingInterval` is clamped to a 250 ms floor in `tracing_handler.cc`, so asking
+# for less is silently ignored. 500 ms is what the DevTools frontend uses.
+# `kMinimumReportingInterval`.
BUFFER_POLL_MS = 500
-# A trace this close to full did not lose data but was about to, and the next
-# run at the next rung will. Surfaced as a warning so a rung ladder does not
-# walk off a cliff between two cells.
+# A trace this close to full did not lose data but was about to, and the next rung will.
+# Surfaced as a warning so a rung ladder does not walk off a cliff.
BUFFER_WARN_FRACTION = 0.80
@@ -237,8 +228,7 @@ class TraceCapture:
self.buffer_kb = int(buffer_kb)
self.record_mode = record_mode
cats = list(CATEGORIES_BY_LEVEL[level]) + [c for c in extra_categories if c]
- # Order-stable de-duplication so the recorded category list in the
- # report is reproducible between runs.
+ # Order-stable de-duplication so the recorded category list is reproducible between runs.
self.categories: tuple[str, ...] = tuple(dict.fromkeys(cats))
self._wait = wait or (lambda ms: time.sleep(ms / 1000.0))
self._usage: list[float] = []
@@ -252,8 +242,8 @@ class TraceCapture:
def _subscribe(self) -> None:
if self._subscribed:
return
- # Playwright delivers the event's `params` object as the single
- # positional argument, and it is `None` for a param-less event.
+ # Playwright delivers the event's `params` object as the single positional argument, and it is
+ # `None` for a param-less event.
self.cdp.on("Tracing.bufferUsage", self._on_buffer_usage)
self.cdp.on("Tracing.tracingComplete", self._on_complete)
self._subscribed = True
@@ -268,9 +258,8 @@ class TraceCapture:
def start(self) -> None:
if self.level == L0:
- # L0 attaches nothing. Making this a no-op rather than an error is
- # what lets one code path run every level, including the level whose
- # entire definition is "do not instrument".
+ # L0 attaches nothing. A no-op rather than an error is what lets one code path run every level,
+ # including the one whose definition is 'do not instrument'.
self._running = True
self._t_start = time.perf_counter()
return
@@ -340,8 +329,8 @@ class TraceCapture:
f"Tracing.tracingComplete did not arrive within {timeout_s:.0f}s",
)
self._wait(50)
- # `dataLossOccurred` can arrive first on a very small trace; give the
- # stream handle a moment before deciding there is not one.
+ # `dataLossOccurred` can arrive first on a very small trace; give the stream handle a moment
+ # before deciding there is not one.
grace = time.time() + 5.0
while "stream" not in self._complete and time.time() < grace:
self._wait(50)
@@ -405,8 +394,8 @@ class TraceCapture:
try:
self.cdp.send("IO.close", {"handle": handle})
except Exception:
- # The stream is temp storage in the browser process; failing to
- # close it leaks a file, it does not invalidate the trace we hold.
+ # The stream is temp storage in the browser process; failing to close it leaks a file, it does
+ # not invalidate the trace we hold.
pass
blob = "".join(parts)
raw = base64.b64decode(blob) if binary else blob.encode("utf-8")
@@ -415,9 +404,6 @@ class TraceCapture:
return raw.decode("utf-8", errors = "strict"), chunks
-# ------------------------------------------------------------------- overheads
-
-
@dataclass
class OverheadLedger:
"""Per-rung measured cost of instrumentation, and the disqualification gate.
@@ -438,11 +424,10 @@ class OverheadLedger:
run. Same rule, same tolerance, two entry points.
"""
- # rung label -> level -> observed cost of the identical cell (ms, or any
- # single consistent scalar such as median frame time)
+ # rung label -> level -> observed cost of the identical cell (ms, or any single consistent
+ # scalar such as median frame time)
cells: dict[str, dict[str, float]] = field(default_factory = dict)
- # A level whose overhead ratio rises by more than this across the ladder is
- # disqualified from exponent claims.
+ # A level whose overhead ratio rises by more than this across the ladder is disqualified from exponent claims.
growth_tolerance: float = 0.15
def record(self, rung: str, level: str, cost: float) -> None:
@@ -612,29 +597,17 @@ def load_trace_json(result: TraceResult) -> dict[str, Any]:
return json.loads(result.text)
-# ═══════════════════════════════════════════════════════════════════════════
# Harness adapter (INTERFACES.md section 3)
-# ═══════════════════════════════════════════════════════════════════════════
-#
-# ONE TRACE PER WINDOW, not one per cell. The alternative was a cell-long trace
-# plus `performance.mark` bracketing to locate each window inside it, and it was
-# rejected for three reasons, in increasing order of importance:
-#
-# 1. A mark is a `Runtime.evaluate` round trip, which is renderer-visible work
-# inside the measured interval. Small, but correlated with nothing and
-# still avoidable.
-# 2. A cell-long trace at L2 accumulates ProfileChunks for the whole cell, so
-# the buffer-overflow risk grows with cell length. Per-window traces keep
-# each buffer bounded by one window.
-# 3. Per-window numbers have to exist at `close()` time, because the harness
-# emits `window.row()` right after. A trace drained at `end_cell` can only
-# backfill the cell row, so every window row would carry nulls.
-#
-# `Tracing.end` and the drain happen inside `close()`, which the harness calls
-# AFTER it has stamped `t_close_ms`. The window duration is therefore already
-# fixed and the drain cannot inflate it. The `Tracing.start` in `open()` does
-# land inside the window, so it is timed and reported as `overhead_ms` rather
-# than waved away.
+# ONE TRACE PER WINDOW, not one per cell. The alternative, a cell-long trace plus
+# `performance.mark` bracketing, was rejected because a mark is a `Runtime.evaluate` round trip
+# inside the measured interval; because a cell-long L2 trace accumulates ProfileChunks so
+# overflow risk grows with cell length; and above all because per-window numbers must exist at
+# `close()` time, since the harness emits `window.row()` right after and a trace drained at
+# `end_cell` would leave every window row null.
+# `Tracing.end` and the drain happen inside `close()`, which the harness calls AFTER stamping
+# `t_close_ms`, so the window duration is already fixed and the drain cannot inflate it. The
+# `Tracing.start` in `open()` does land inside the window, so it is timed and reported as
+# `overhead_ms`.
from ..analysis import assert_no_bare_zero, measured, merge, unmeasured # noqa: E402
from . import register_instrument # noqa: E402
@@ -664,26 +637,23 @@ class TracingInstrument:
self.ctx = ctx
def start_cell(self, cell: Any) -> None:
- # `ctx.page` and `ctx.cdp` may be REPLACED between cells when a crashed
- # renderer is recovered, so they are re-read here and never cached in
- # `attach`. INTERFACES.md section 7 is explicit about this.
+ # `ctx.page` and `ctx.cdp` may be REPLACED between cells when a crashed renderer is recovered,
+ # so they are re-read here and never cached in `attach`. INTERFACES.md section 7.
self.cell = cell
self.cdp = getattr(self.ctx, "cdp", None)
self._overhead_ms = 0.0
self._windows = 0
self._failed_windows = []
lvl = int(getattr(cell, "instrument_level", 1) or 1)
- # The instrument ladder is expressed HERE. L0 never reaches this method
- # because the registry filters on `level`, and L3 uses the same category
- # set as L2: L3 adds coverage and heap sampling, which are other
- # instruments, not more categories.
+ # The instrument ladder is expressed HERE. L0 never reaches this method because the registry
+ # filters on `level`, and L3 uses the same category set as L2 (its coverage and heap sampling
+ # are other instruments, not more categories).
self.trace_level = L1 if lvl <= 1 else (L2 if lvl == 2 else L3)
if self.cdp is not None:
try:
enable_metrics(self.cdp)
except Exception:
- # Metrics are a cross-check, not the measurement. Losing them
- # costs the cross-check and nothing else.
+ # Metrics are a cross-check, not the measurement. Losing them costs the cross-check and nothing else.
pass
def open(self, window: Any) -> None:
@@ -702,18 +672,14 @@ class TracingInstrument:
self.capture = None
self.metrics = None
else:
- # THE METRICS PROBE MAY NOT TAKE THE TRACE WITH IT. `start_cell` says what losing the
- # probe is meant to cost -- "the cross-check and nothing else" -- and dropping a
- # STARTED capture here cost the whole run instead. Tracing is per-browser: with
- # `self.capture` cleared, `close()` returns early and never sends `Tracing.end`,
- # `detach()` has nothing to stop, and the next window's `Tracing.start` fails with
- # "Tracing has already been started (possibly in another tab)" -- so every remaining
- # window reports `tracing did not start`, while the abandoned session keeps recording
- # into the browser underneath the numbers this tool is taking. One failed
- # `Performance.getMetrics` (the domain is enabled under a bare `except` in
- # `start_cell`, so it can be off) was enough. `close()` and `_analyse` already guard
- # every use of `self.metrics` with `is not None`, so the cross-check is the only
- # thing that goes.
+ # THE METRICS PROBE MAY NOT TAKE THE TRACE WITH IT. Dropping a STARTED capture here cost the
+ # whole run: tracing is per-browser, so with `self.capture` cleared `close()` returns early and
+ # never sends `Tracing.end`, `detach()` has nothing to stop, and the next window's
+ # `Tracing.start` fails with 'Tracing has already been started', so every remaining window
+ # reports `tracing did not start` while the abandoned session keeps recording underneath. One
+ # failed `Performance.getMetrics` was enough. `close()` and `_analyse` already guard every use
+ # of `self.metrics`, so only the cross-check goes.
+ # `start_cell` says what losing the metrics probe costs, and it can be off.
try:
self.metrics = MetricsWindow(self.cdp)
self.metrics.open()
@@ -755,10 +721,9 @@ class TracingInstrument:
return payload
def end_cell(self, cell: Any) -> dict | None:
- # `overhead_ms` is required from every instrument at level >= 1
- # (INTERFACES.md section 3) and it is what the report layer's
- # `overhead_growth_with_length` gate consumes. It is a MEASURED wall
- # cost of this instrument's own calls, not an estimate from a table.
+ # `overhead_ms` is required from every instrument at level >= 1 (INTERFACES.md section 3) and
+ # feeds the report layer's `overhead_growth_with_length` gate. It is a MEASURED wall cost of
+ # this instrument's own calls, not an estimate from a table.
out = merge(
measured("overhead_ms", round(self._overhead_ms, 3)),
measured("windows_traced", self._windows),
@@ -823,9 +788,8 @@ class TracingInstrument:
},
)
- # Cross-check against the renderer's own accounting. A disagreement means
- # the trace is missing tasks, so it is reported rather than silently
- # tolerated, but it does not void the window on its own.
+ # Cross-check against the renderer's own accounting. A disagreement means the trace is missing
+ # tasks, so it is reported rather than silently tolerated, but it does not void the window.
try:
if self.metrics is not None:
payload.update(
@@ -845,9 +809,8 @@ class TracingInstrument:
unmeasured("task_duration_crosscheck_drift", f"{type(exc).__name__}: {exc}")
)
- # Named frames need the V8 profiler, which only L2+ turns on. At L1 this
- # is an honest null with a reason, never an empty list that reads as
- # "nothing was hot".
+ # Named frames need the V8 profiler, which only L2+ turns on. At L1 this is an honest null with
+ # a reason, never an empty list that reads as 'nothing was hot'.
if self.trace_level == L1:
payload.update(
unmeasured(
diff --git a/tests/studio/studiobench/pacer.py b/tests/studio/studiobench/pacer.py
index f56cb2b0365..c6ba7b43f2a 100644
--- a/tests/studio/studiobench/pacer.py
+++ b/tests/studio/studiobench/pacer.py
@@ -47,12 +47,11 @@ from dataclasses import dataclass, field
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Optional
-# The field's own arrival rate: 90,262 characters in 276 seconds is 327 characters a second,
-# which at 24 characters a chunk is one chunk every 73ms. It is part of the fixture and getting it
-# wrong is how the earlier harness failed to reproduce anything: at a 2ms gap the renderer is the
-# bottleneck from the first chunk, the run opens at 45 fps instead of 59, and there is no healthy
-# baseline left to degrade FROM. The trace's whole shape is "idle between chunks at the start, not
-# idle at the end".
+# The field's own arrival rate: 90,262 characters in 276 seconds is 327 a second, which at 24
+# characters a chunk is one chunk every 73ms. Getting it wrong is how the earlier harness
+# failed to reproduce anything: at a 2ms gap the renderer is the bottleneck from the first
+# chunk, the run opens at 45 fps instead of 59, and there is no healthy baseline to degrade
+# FROM. The trace's shape is 'idle between chunks at the start, not idle at the end'.
CAD_FIELD = (24, 73)
# For the small rungs, where the field cadence would spend four minutes streaming a reply nobody
# is measuring the cadence of.
@@ -72,12 +71,12 @@ class Script:
chunk_chars: int = CAD_FIELD[0]
gap_ms: int = CAD_FIELD[1]
model: str = "studiobench-pacer"
- # Emitted before anything else, so a driver can prove the stream it is watching is the one it
- # asked for rather than a leftover from the previous cell.
+ # Emitted first, so a driver can prove the stream it is watching is the one it asked for rather
+ # than a leftover from the previous cell.
tag: str = ""
- # Set by the driver to make the pacer stop mid-reply, for the stop-generation action. The
- # pacer keeps the socket OPEN and idle rather than closing it, because a close is a different
- # event from a cancel and the app distinguishes them.
+ # Set by the driver to stop the pacer mid-reply for the stop-generation action. The socket is
+ # kept OPEN and idle rather than closed, because a close is a different event from a cancel and
+ # the app distinguishes them.
hold_after_chars: Optional[int] = None
@@ -101,8 +100,8 @@ class StreamStats:
reasoning_chars_sent: int = 0
content_chars_sent: int = 0
# How often the deficit scheduler had to send more than one chunk to catch up, and the worst
- # shortfall it saw. A run where these stay at zero had no backpressure at all; a run where
- # max_deficit is 40 had the socket blocked for three seconds.
+ # shortfall. Zero here means no backpressure at all; a max_deficit of 40 means the socket was
+ # blocked for three seconds.
bursts: int = 0
max_deficit: int = 0
# Time spent inside a blocking write. This IS the backpressure, measured rather than inferred.
@@ -113,9 +112,8 @@ class StreamStats:
held: bool = False
error: Optional[str] = None
duration_ms: Optional[float] = None
- # The cadence achieved, against the one asked for. A gap that came out at 91ms when 73 was
- # requested is a machine that could not keep up, and it belongs in the report next to the
- # numbers it distorted.
+ # The cadence achieved against the one asked for: a gap of 91ms when 73 was requested is a
+ # machine that could not keep up, and it belongs next to the numbers it distorted.
requested_gap_ms: int = 0
achieved_gap_ms: Optional[float] = None
@@ -143,8 +141,8 @@ class PacerState:
self.stats: list[StreamStats] = []
self.requests_seen = 0
self.model_ids: list[str] = ["studiobench-pacer"]
- # Flipped by the driver to release a held stream (the stop-generation action asserts the
- # app's cancel, so the pacer must be able to sit still while it happens).
+ # Flipped by the driver to release a held stream (the stop-generation action asserts the app's
+ # cancel, so the pacer must sit still while it happens).
self.release = threading.Event()
def set_script(self, script: Script) -> None:
@@ -196,7 +194,7 @@ class _Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "studiobench-pacer"
- # Quiet. A per-request line on stderr at 24 characters a chunk is 4,000 lines a reply.
+ # Quiet: a per-request line on stderr at 24 characters a chunk is 4,000 lines a reply.
def log_message(self, fmt, *args) -> None: # noqa: A003
pass
@@ -305,8 +303,8 @@ class _Handler(BaseHTTPRequestHandler):
)
if not body.get("stream", True):
- # A non-streaming request is a bug in the driver, not a mode: every mechanism this
- # tool measures is on the streaming path. Say so rather than serve it.
+ # A non-streaming request is a bug in the driver, not a mode: every mechanism this tool measures
+ # is on the streaming path.
self._json(
400,
{
@@ -323,19 +321,18 @@ class _Handler(BaseHTTPRequestHandler):
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache, no-transform")
self.send_header("Connection", "keep-alive")
- # CHUNKED, explicitly. On HTTP/1.1 a response with neither Content-Length nor
- # Transfer-Encoding is a keep-alive response of unknown length, and the reader blocks
- # forever waiting for a body that already arrived -- measured here as a client that hung
- # past a 60s timeout having received every byte. Uvicorn, which is what Unsloth's own
- # backend streams through, sends chunked for a StreamingResponse, so this is also the
- # framing the relay and the browser see in production.
+ # CHUNKED, explicitly. On HTTP/1.1 a response with neither Content-Length nor Transfer-Encoding
+ # is a keep-alive response of unknown length, and the reader blocks forever waiting for a body
+ # that already arrived, measured here as a client hanging past a 60s timeout having received
+ # every byte. Uvicorn, which Unsloth's backend streams through, sends chunked for a
+ # StreamingResponse, so this is also the production framing.
self.send_header("Transfer-Encoding", "chunked")
self.send_header("X-Accel-Buffering", "no")
self.end_headers()
# The role chunk first, as every OpenAI-compatible server sends it. The app tolerates its
- # absence, but its presence is what makes the first delta an APPEND rather than a create,
- # which is the path the cumulative buffer takes for every chunk after it.
+ # absence, but its presence makes the first delta an APPEND rather than a create, which is the
+ # path every chunk after it takes.
chunks: list[tuple[str, str]] = []
for piece in _split(script.reasoning, script.chunk_chars):
chunks.append(("reasoning", piece))
@@ -350,8 +347,8 @@ class _Handler(BaseHTTPRequestHandler):
last_activity = time.monotonic()
while sent < len(chunks):
now = time.monotonic()
- # DEFICIT SCHEDULING. How many chunks SHOULD have gone out by now, from wall
- # clock alone; send the shortfall in one burst. Never sleep(gap) per chunk.
+ # DEFICIT SCHEDULING: how many chunks SHOULD have gone out by now, from wall clock alone; send
+ # the shortfall in one burst. Never sleep(gap) per chunk.
should_have_sent = int((now - t0) / gap_s)
deficit = min(should_have_sent, len(chunks)) - sent
if deficit <= 0:
@@ -359,9 +356,8 @@ class _Handler(BaseHTTPRequestHandler):
self._write(b": keep-alive\n\n", stats)
stats.keepalives += 1
last_activity = now
- # Sleep to the next boundary, not a fixed tick: waking late is the SIGNAL, and
- # the burst on the next pass is what a real backend does when a socket that
- # was backed up clears.
+ # Sleep to the next boundary, not a fixed tick: waking late is the SIGNAL, and the burst on the
+ # next pass is what a real backend does when a backed-up socket clears.
target = t0 + (sent + 1) * gap_s
time.sleep(max(0.0, min(target - now, KEEPALIVE_S)))
continue
@@ -372,9 +368,8 @@ class _Handler(BaseHTTPRequestHandler):
for _ in range(deficit):
kind, piece = chunks[sent]
if kind == "reasoning":
- # `content: ""` alongside, which is exactly what the backend's own
- # `_gguf_chat_delta_line` emits. Without it the app takes a different
- # branch in delta accumulation and the cumulative <think> buffer is
+ # `content: ""` alongside, exactly as the backend's own `_gguf_chat_delta_line` emits. Without
+ # it the app takes a different delta-accumulation branch and the cumulative <think> buffer is
# assembled by a path the shipping build never uses.
delta = {"reasoning_content": piece, "content": ""}
stats.reasoning_chars_sent += len(piece)
@@ -399,20 +394,19 @@ class _Handler(BaseHTTPRequestHandler):
script.hold_after_chars is not None
and stats.chars_sent >= script.hold_after_chars
):
- # HOLD, do not close. The stop-generation action needs the app to be visibly
- # mid-stream while it presses stop; closing the socket instead would end the
- # stream on its own and the action would measure a stream that had already
- # finished. The keep-alives make the hold indistinguishable from a model
- # thinking, which is what it is standing in for.
+ # HOLD, do not close: the stop-generation action needs the app visibly mid-stream while it
+ # presses stop, and closing would end the stream on its own so the action would measure a
+ # stream that had already finished. The keep-alives make the hold indistinguishable from a
+ # model thinking.
stats.held = True
while not self.state.release.wait(KEEPALIVE_S):
self._write(b": keep-alive\n\n", stats)
stats.keepalives += 1
break
- # BOTH terminal signals. `streamChatCompletions` throws StreamInterruptedError at EOF
- # unless it saw a finish_reason or a [DONE]; sending one and not the other measures the
- # app's error path and calls it a benchmark.
+ # BOTH terminal signals: `streamChatCompletions` throws StreamInterruptedError at EOF unless it
+ # saw a finish_reason or a [DONE], so sending one and not the other measures the app's error
+ # path and calls it a benchmark.
self._write(_sse(_chunk_frame(request_id, model, {}, finish_reason = "stop")), stats)
prompt_tokens = max(1, len(script.reasoning) // 4)
completion_tokens = max(1, stats.chars_sent // 4)
@@ -643,8 +637,8 @@ def check_planned_streams(streams: list[dict], planned: list[dict]) -> dict:
"checked": bool(planned),
"planned_turns": len(planned),
"turns": turns,
- # The throwaway turns and any retry, kept so a reader can see WHAT else the fixture served
- # without them being mistaken for a planned turn that failed.
+ # The throwaway turns and any retry, kept so a reader can see what else the fixture served
+ # without mistaking it for a planned turn that failed.
"extra": remaining,
"reason": None if ok else "; ".join(t["reason"] for t in failures),
}
@@ -712,8 +706,8 @@ def _selftest() -> int:
stats = pacer.last_stats()
assert stats and stats["completed"], f"stream did not complete: {stats}"
expected = pacer.expected_duration_ms(reasoning, content, "fast")
- # The deficit scheduler's whole promise: duration is set by wall clock, not by how fast
- # the reader is. Generous bound because a loaded CI box is exactly the case it must hold in.
+ # The deficit scheduler's whole promise: duration is set by wall clock, not by how fast the
+ # reader is. Generous bound because a loaded CI box is the case it must hold in.
assert (
elapsed_ms >= expected * 0.75
), f"stream finished in {elapsed_ms:.0f}ms, faster than the {expected:.0f}ms cadence"
@@ -732,22 +726,20 @@ def _selftest() -> int:
f"{stats['bursts']} bursts, max deficit {stats['max_deficit']}"
)
- # PHASE 2: the deficit claim itself. A reader that stalls must NOT slow the stream down;
- # it must come back to a burst and the run must still end on wall clock. This is the
- # difference between measuring a renderer and measuring a server that waits for one.
+ # PHASE 2: the deficit claim itself. A stalled reader must NOT slow the stream down; it must
+ # come back to a burst and the run must still end on wall clock, the difference between
+ # measuring a renderer and measuring a server that waits for one.
pacer.reset()
- # Big enough to exhaust the loopback send buffer, which autotunes to `wmem_max` (4MB on
- # this kernel). A 400KB reply does not block a loopback write no matter how long the
- # reader sleeps, so a smaller body tests nothing.
+ # Big enough to exhaust the loopback send buffer, which autotunes to `wmem_max` (4MB on this
+ # kernel). A 400KB reply does not block a loopback write however long the reader sleeps.
body = "D" * 12_000_000
cad = {"chunk_chars": 12_000, "gap_ms": 1}
pacer.load("", body, tag = "backpressure", **cad)
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- # A small receive buffer is what makes this a test. With the default buffer the kernel
- # absorbs a 17KB reply whole, the pacer's writes never block, it is never late, and there
- # is correctly no deficit to schedule -- which is what the first version of this check
- # measured and wrongly called a failure. Backpressure needs a reader that cannot keep up
- # AND a pipe that fills, which is exactly the case a jammed renderer produces.
+ # A small receive buffer is what makes this a test: with the default buffer the kernel absorbs a
+ # 17KB reply whole, the pacer's writes never block and there is correctly no deficit, which the
+ # first version of this check measured and wrongly called a failure. Backpressure needs a reader
+ # that cannot keep up AND a pipe that fills.
conn.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536)
conn.settimeout(60)
conn.connect((pacer.host, pacer.port))
@@ -767,8 +759,8 @@ def _selftest() -> int:
+ payload
)
started = time.monotonic()
- # Read nothing at all for a stretch several gaps long, then drain. The socket buffer
- # absorbs it, so the pacer keeps its own clock and owes a shortfall when it next looks.
+ # Read nothing for a stretch several gaps long, then drain: the socket buffer absorbs it, so the
+ # pacer keeps its own clock and owes a shortfall when it next looks.
time.sleep(0.4)
conn.setblocking(True)
seen = b""
@@ -787,12 +779,10 @@ def _selftest() -> int:
f"rather than scheduling against wall clock: {s2}"
)
# The honest form of the machine-independence claim. No scheduler can make a stream finish
- # faster than the reader consumes it, so total duration is NOT bounded by the cadence when
- # the reader is the throughput limit. What the deficit scheduler promises is narrower and
- # is exactly what is checked here: the pacer itself adds no delay beyond the cadence, so
- # every millisecond of overrun is accounted for by time blocked in a write. If it were
- # sleeping a gap per chunk instead, the overrun would exceed the blocked time by the
- # stall, since each late chunk would then push the NEXT one late as well.
+ # faster than the reader consumes it, so total duration is NOT bounded by the cadence when the
+ # reader is the throughput limit. What is checked is narrower: the pacer adds no delay beyond
+ # the cadence, so every millisecond of overrun is accounted for by time blocked in a write.
+ # Sleeping a gap per chunk instead, the overrun would exceed the blocked time by the stall.
unblocked_ms = stalled_ms - s2["write_block_ms"]
assert unblocked_ms <= expected2 * 2.5 + 1000, (
f"the stream spent {unblocked_ms:.0f}ms not blocked on a write, against a "
diff --git a/tests/studio/studiobench/report/build.py b/tests/studio/studiobench/report/build.py
index a457dd55f13..1aea467d5ca 100644
--- a/tests/studio/studiobench/report/build.py
+++ b/tests/studio/studiobench/report/build.py
@@ -40,8 +40,8 @@ def _records(path: str | Path) -> list[dict[str, Any]]:
try:
out.append(json.loads(line))
except ValueError:
- # A truncated final line is expected when a run was killed mid-write; the
- # records before it are still good and are still reported.
+ # A truncated final line is expected when a run was killed mid-write; the records before it are
+ # still good and are still reported.
continue
return out
@@ -63,24 +63,19 @@ def _completion_by_rung(records: Sequence[Mapping[str, Any]]) -> dict[int, tuple
if not completed:
reason = f"{failure.get('kind') or 'unknown'}: {failure.get('message') or 'no message'}"
# NOT COMPLETE, RATHER THAN NOT PRESENT. A cell whose thread lost messages, or that stopped
- # following the reply it was measuring, reaches here `completed=True` with a full set of
- # timings, because both gates are advisory where they are emitted. Those timings are
- # CHEAPER than a correct cell's -- fewer rows to render -- and the ladder is ABSOLUTE, with
- # no second arm to contradict them, so the rung was scored against fixed anchors and came
- # out green and fast on a cell whose own self-check had already recorded the loss.
- #
- # It is marked incomplete rather than dropped, which is the distinction this module is
- # built on: `score_rung` gives an incomplete rung 0 and KEEPS ITS WEIGHT, and aggregating
- # over only the rungs that survived is the crash-beats-limp bug wearing a different hat. A
- # build whose thread loses its middle at 100K is not usable at 100K, and lowering `onset`
- # is exactly the honest way to say so. Dropping the cell would delete the most important
- # thing the run has to say.
+ # following the reply, reaches here `completed=True` with a full set of timings, because both
+ # gates are advisory where they are emitted. Those timings are CHEAPER than a correct cell's and
+ # the ladder is ABSOLUTE with no second arm to contradict them, so the rung came out green and
+ # fast on a cell whose own self-check had recorded the loss.
+ # It is marked incomplete rather than dropped: `score_rung` gives an incomplete rung 0 and KEEPS
+ # ITS WEIGHT, and aggregating over only the rungs that survived is the crash-beats-limp bug in a
+ # different hat. A build whose thread loses its middle at 100K is not usable at 100K, and lowering
+ # `onset` is the honest way to say so.
elif str(r.get("cell_id")) in gate_failures:
completed = False
reason = gate_failures[str(r.get("cell_id"))]
- # If a rung was repeated and any rep failed, the rung is not clean. Recording the failure
- # rather than the success is deliberate: the interesting fact about a bimodal rung is
- # that it can fail, not that it can pass.
+ # If a rung was repeated and any rep failed, the rung is not clean. Recording the failure rather
+ # than the success is deliberate: the interesting fact about a bimodal rung is that it can fail.
prev = out.get(int(tokens))
if prev is None or (prev[0] and not completed):
out[int(tokens)] = (completed, reason)
@@ -91,17 +86,15 @@ def score_payload(path: str | Path, declared_rungs: Sequence[int] | None = None)
"""Score one run. `declared_rungs` is the ladder the tier promised, in tokens."""
# BEFORE anything is scored, and against the RAW rows. `--report` reads the same file the run
- # wrote, so a probe run that was allowed to print its own A/B table would be scorable a second
- # time here, hours later, by somebody who was not the one who set the variable. It reads the
- # unfiltered rows deliberately: a probed attempt that was later superseded still means this
- # payload was recorded with the camera in the shot.
+ # wrote, so a probe run allowed to print its own A/B table would be scorable a second time here by
+ # somebody who did not set the variable. It reads the unfiltered rows deliberately: a probed
+ # attempt later superseded still means the payload was recorded with the camera in the shot.
raw = _records(path)
refuse_if_probed(raw, str(path))
- # A CELL THAT WAS RE-RUN IS SCORED ON THE RUN THAT FINISHED IT. `--resume` re-runs the cells
- # that died, under the same `cell_id`, into the same file; scoring both attempts as one cell
- # kept the crash forever, so a rung that had since been re-run successfully still came out
- # INCOMPLETE and zero. The dead attempt is still in the payload and still in EXCLUDED CELLS,
- # which is where a superseded crash belongs -- it is only kept out of the score.
+ # A CELL THAT WAS RE-RUN IS SCORED ON THE RUN THAT FINISHED IT. `--resume` re-runs dead cells under
+ # the same `cell_id` into the same file, and scoring both attempts as one cell kept the crash
+ # forever, so a rung since re-run successfully still came out INCOMPLETE and zero. The dead
+ # attempt is still in the payload and in EXCLUDED CELLS; it is only kept out of the score.
records = latest_attempt_rows(raw)
measures = measures_from_records(records)
completion = _completion_by_rung(records)
@@ -133,10 +126,10 @@ def build_report(
) -> tuple[str, LadderScore, dict[str, Any]]:
"""Return (rendered summary, ladder, assembled payload)."""
- # FIRST, before the payload is assembled. `score_payload` refuses too, but it runs second
- # here, and `assemble_rows` validates the schema on the way past: a probed payload that also
- # trips some unrelated schema complaint would report THAT instead, and the refusal would
- # never be reached. A refusal that any other failure can pre-empt is not a refusal.
+ # FIRST, before the payload is assembled. `score_payload` refuses too but runs second here, and
+ # `assemble_rows` validates the schema on the way past, so a probed payload that also tripped an
+ # unrelated schema complaint would report THAT instead. A refusal that any other failure can
+ # pre-empt is not a refusal.
refuse_if_probed(_records(path), str(path))
payload = assemble_rows(path)
ladder = score_payload(path, declared_rungs)
diff --git a/tests/studio/studiobench/report/overhead.py b/tests/studio/studiobench/report/overhead.py
index 9ef4ba773a1..edec25b22fa 100644
--- a/tests/studio/studiobench/report/overhead.py
+++ b/tests/studio/studiobench/report/overhead.py
@@ -26,13 +26,13 @@ from typing import Any, Mapping
from ..scoring.schema import Measure
-#: Overhead may vary this much across the ladder before the level is disqualified. Generous:
-#: some growth is unavoidable (more DOM means a longer snapshot), and a tight bound would
-#: disqualify every level on every machine, which is the same as having no gate.
+#: Overhead may vary this much across the ladder before the level is disqualified. Generous: some
+#: growth is unavoidable (more DOM means a longer snapshot), and a tight bound would disqualify
+#: every level on every machine.
MAX_OVERHEAD_GROWTH_RATIO = 1.5
-#: Growth below this many milliseconds is not worth acting on however large the ratio looks.
-#: A level whose overhead goes from 0.01 ms to 0.05 ms has a ratio of 5 and does not matter.
+#: Growth below this many milliseconds is not worth acting on however large the ratio looks: a level
+#: whose overhead goes from 0.01 ms to 0.05 ms has a ratio of 5 and does not matter.
MIN_ABSOLUTE_GROWTH_MS = 1.0
diff --git a/tests/studio/studiobench/report/payload.py b/tests/studio/studiobench/report/payload.py
index 2120689033f..4053d0d942e 100644
--- a/tests/studio/studiobench/report/payload.py
+++ b/tests/studio/studiobench/report/payload.py
@@ -205,10 +205,10 @@ def assemble(path: str | Path, *, validate: bool = True) -> dict[str, Any]:
return payload
-#: How the harness layer's `row_type` values map onto the sections of an assembled payload.
-#: Layer 1 writes rows through its `Recorder`; this layer reads them. Keeping the mapping in one
-#: table rather than spread through the renderer means a new row type is one line here and a
-#: visible `unknown_rows` entry until somebody decides where it belongs.
+#: How the harness layer's `row_type` values map onto the sections of an assembled payload. Layer 1
+#: writes rows through its `Recorder` and this layer reads them; keeping the mapping in one table
+#: means a new row type is one line here and a visible `unknown_rows` entry until somebody decides
+#: where it belongs.
ROW_TYPE_SECTIONS: Mapping[str, str] = {
"run_meta": "header",
"gate": "selfcheck",
@@ -217,28 +217,24 @@ ROW_TYPE_SECTIONS: Mapping[str, str] = {
"action": "actions",
"sample": "samples",
"failure": "crashes",
- # Bookkeeping about HOW the A/B was run, not a measurement of the app. It belongs beside the
- # identity fields so a reader can see whether the order was balanced without digging.
+ # Bookkeeping about HOW the A/B was run, not a measurement of the app, so it belongs beside the
+ # identity fields where a reader can see whether the order was balanced.
"ab_plan": "header",
- # The optional surface sweep. Its own section: a surface row is a coverage fact about the UI,
- # not a timing, and folding it into `actions` would put it in front of the scorer.
+ # The optional surface sweep. Its own section: a surface row is a coverage fact about the UI, not a
+ # timing, and folding it into `actions` would put it in front of the scorer.
"surface": "surfaces",
- # The comparability key. Its own section rather than `header`, for two independent reasons.
- #
- # First, `header` is collapsed to its FIRST row when the payload is assembled, so a second row
- # filed there is dropped without a word. Second, the row's `fields` block is identity
- # bookkeeping and not a measurement: `instrument_level: 0` is a true statement about how the
- # run was instrumented, exactly like the `identity` and `config` subtrees, so the section is
- # exempted from the bare-zero ban rather than made to fake a Measure. Left unmapped the row
- # fell into `unknown_rows`, which nothing exempts, and the walker killed every real-path
+ # The comparability key. Its own section rather than `header` for two reasons: `header` is
+ # collapsed to its FIRST row when the payload is assembled, so a second row filed there is dropped
+ # without a word; and the row's `fields` block is identity bookkeeping, not a measurement, so the
+ # section is exempted from the bare-zero ban rather than made to fake a Measure. Left unmapped the
+ # row fell into `unknown_rows`, which nothing exempts, and the walker killed every real-path
# session on `$.unknown_rows[0].fields.instrument_level = 0`.
"comparability": "comparability",
- # The terminal marker for a cell that did not finish. NOT `cells`, which is what the scorer
- # reads, and NOT an exclusion source: the `cell` row it follows is emitted with
- # `completed: false` immediately before it on the same path, and `excluded_from_rows` already
- # turns that into a `rung_incomplete` exclusion. Filing this row as a second exclusion would
- # count one abort twice. It exists so a reader scanning FORWARD can discard the cell's window
- # rows, and that is all it is kept for here.
+ # The terminal marker for a cell that did not finish. NOT `cells`, which is what the scorer reads,
+ # and NOT an exclusion source: the `cell` row it follows is emitted with `completed: false`
+ # immediately before it and `excluded_from_rows` already turns that into a `rung_incomplete`
+ # exclusion, so filing this as a second exclusion would count one abort twice. It exists so a
+ # reader scanning FORWARD can discard the cell's window rows.
"cell_aborted": "aborted_cells",
}
diff --git a/tests/studio/studiobench/report/render.py b/tests/studio/studiobench/report/render.py
index fa8da31048a..e5e4263934d 100644
--- a/tests/studio/studiobench/report/render.py
+++ b/tests/studio/studiobench/report/render.py
@@ -34,7 +34,7 @@ from ..scoring.schema import Measure
from ..scoring.score import LadderScore, RungScore
from .payload import excluded_totals
-#: The three frame numbers that must appear together or not at all.
+#:The three frame numbers that must appear together or not at all.
HEADLINE_FRAME_METRICS = ("time_in_jank_pct", "jank_index", "max_frame_ms")
@@ -141,8 +141,8 @@ def render_rung_table(ladder: LadderScore) -> str:
def render_rung_metrics(rung: RungScore, *, indent: str = " ") -> str:
lines = [f"{indent}{rung.tokens:,} tokens -- score {rung.score:.1f}"]
if not rung.complete:
- # Printing six "not attempted" lines for a rung that never ran buries the one fact that
- # matters, which is why it did not run.
+ # Printing six "not attempted" lines for a rung that never ran buries the one fact that matters,
+ # which is why it did not run.
lines.append(f"{indent} INCOMPLETE: {rung.incomplete_reason}")
lines.append(
f"{indent} scores 0 and keeps its weight; no metric was measured at this rung"
@@ -217,9 +217,9 @@ def render_ab_table(result: AbResult) -> str:
lines.append("")
lines.append("VOID. No numbers are quotable from this comparison.")
lines.append(f" {result.void_reason}")
- # The paragraph under the reason explains the NULL CONTROL, and printing it under an
- # incomplete plan named the wrong cause for the void. A void has more than one cause; the
- # reason line carries which one, and only a null control gets the null-control paragraph.
+ # The paragraph under the reason explains the NULL CONTROL, and printing it under an incomplete
+ # plan named the wrong cause for the void. A void has more than one cause; the reason line carries
+ # which one.
if result.is_null_control:
lines.append(
" The null-treatment control measures whether this machine can currently tell "
@@ -252,10 +252,9 @@ def render_ab_table(result: AbResult) -> str:
if metric.ci_low is not None
else "too few pairs"
)
- # A BOUND IS NOT A MEASUREMENT. An arm under its instrument floor contributes the floor,
- # so the ratio understates the true magnitude and must not be quoted as a point estimate.
- # Marked in the ratio cell rather than footnoted, for the reason the void path gives: the
- # table gets screenshotted and the note does not.
+ # A BOUND IS NOT A MEASUREMENT. An arm under its instrument floor contributes the floor, so the
+ # ratio understates the true magnitude and must not be quoted as a point estimate. Marked in the
+ # ratio cell rather than footnoted, because the table gets screenshotted and the note does not.
ratio_cell = (
f">={metric.ratio_geomean:.3f}"
if metric.bounded and metric.ratio_geomean >= 1.0
diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py b/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py
index eae5980188c..3e55a5e8fd4 100644
--- a/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py
+++ b/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py
@@ -72,7 +72,7 @@ def test_a_fully_resumed_ab_keeps_the_table_the_measured_run_wrote(tmp_path):
_render_ab(paths, SIDES, MEASURED, "c0ffee")
measured = (paths.out / "ab.md").read_text(encoding = "utf-8")
# A real reading, as opposed to an empty table. The fixture is one pair, so the verdict is
- # INCONCLUSIVE rather than a direction; what matters here is that something was measured.
+ # INCONCLUSIVE rather than a direction; what matters is that something was measured.
assert "NO READING" not in measured
assert "keystroke_p95_ms" in measured
diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py b/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py
index 204a34481cb..5830449ebe0 100644
--- a/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py
+++ b/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py
@@ -95,20 +95,17 @@ def test_one_attached_studio_driven_twice_is_a_null_control():
assert is_null_control(sides) is True
-# ── the stream-cost injection needs two origins ─────────────────────────────────────────────
-#
-# The injection is a CONTEXT init script gated on `window.location.origin`, and both arms are
-# driven by one browser context and one page, so the origin is the only thing that can tell them
-# apart. Two arms on one origin therefore BOTH burn the injected cost, the difference between them
-# is zero, and `evaluate_stream_cost_recovery_gate` -- which reads back
-# `(injected_rate - base_rate) * chars` -- reports a recovery of nothing and blames the
-# accumulator. The one flag whose job is to separate "the change did nothing" from "the metric is
-# not watching" would answer the second when the truth is neither.
-#
+# The stream-cost injection needs two origins. The injection is a CONTEXT init script gated on
+# `window.location.origin`, and both arms are driven by one browser context and one page, so the
+# origin is the only thing that can tell them apart. Two arms on one origin BOTH burn the injected
+# cost, the difference is zero, and `evaluate_stream_cost_recovery_gate` reports a recovery of
+# nothing and blames the accumulator, so the one flag whose job is to separate 'the change did
+# nothing' from 'the metric is not watching' answers the second when the truth is neither.
# One attached Unsloth driven twice is not a mistake in general: it is a null control this tool
-# detects on purpose, pinned by the test directly above. It is only fatal with the injection on.
+# detects on purpose, pinned by the test above. It is only fatal with the injection on.
+# ── the stream-cost injection needs two origins ─────────────────────────────────────────────
def _inject_args(attach, attach_b, *extra):
return parse_args(["--attach", attach, "--ab", "main", "--attach-b", attach_b, *extra])
diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py b/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py
index fd3ba853ed2..91c05f7dfe5 100644
--- a/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py
+++ b/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py
@@ -62,18 +62,18 @@ def test_an_action_that_did_not_run_fails(tmp_path):
def test_a_missed_slot_fails_even_though_the_action_ran(tmp_path):
- # `ran` and `slot_missed` are different failures: the second means the film moved on without
- # it, so its timing describes a different session than every other arm's. Default slack is 0,
- # which is the right default for the quiet machine a measurement is taken on.
+ # `ran` and `slot_missed` are different failures: the second means the film moved on without it,
+ # so its timing describes a different session than every other arm's. Default slack is 0, right
+ # for the quiet machine a measurement is taken on.
path = write_payload(tmp_path, [cell([ran("settings", slot_missed = True)])])
assert main(["--assert-liveness", str(path)]) == 1
def test_slack_excuses_a_missed_slot_and_only_up_to_the_number_given(tmp_path):
- # A missed slot is a fact about the MACHINE. The scene is a fixed-duration film on the wall
- # clock and is designed to roll on through one, precisely so a slow machine does not take a
- # different path through a different-length session. On a two-core shared runner, failing on
- # that makes the gate a speed test of the runner rather than a check that the harness works.
+ # A missed slot is a fact about the MACHINE. The scene is a fixed-duration film on the wall clock
+ # designed to roll on through one, precisely so a slow machine does not take a different path
+ # through a different-length session. On a two-core shared runner, failing on that makes the gate
+ # a speed test of the runner.
one = write_payload(tmp_path / "a", [cell([ran("settings", slot_missed = True)])])
assert main(["--assert-liveness", str(one), "--allow-slot-misses", "1"]) == 0
two = write_payload(
@@ -84,8 +84,8 @@ def test_slack_excuses_a_missed_slot_and_only_up_to_the_number_given(tmp_path):
def test_slack_never_excuses_a_scene_problem(tmp_path):
- # The distinction the whole split rests on. An action that was never planned, or whose button
- # was not there, is the harness lying, and no amount of machine slack makes that acceptable.
+ # The distinction the whole split rests on: an action that was never planned, or whose button was
+ # not there, is the harness lying, and no amount of machine slack makes that acceptable.
path = write_payload(
tmp_path,
[cell([{"action": "message_menu", "ran": False, "reason": "no More button in the DOM"}])],
@@ -94,9 +94,8 @@ def test_slack_never_excuses_a_scene_problem(tmp_path):
def test_an_action_that_missed_its_slot_is_not_also_counted_as_a_scene_problem(tmp_path):
- # A missed slot is recorded as `ran: False` WITH `slot_missed: True`, so an implementation
- # that checks `ran` first would file every missed slot under the category that slack cannot
- # excuse, and the flag would do nothing at all.
+ # A missed slot is recorded as `ran: False` WITH `slot_missed: True`, so an implementation that
+ # checks `ran` first would file every missed slot under the category slack cannot excuse.
path = write_payload(
tmp_path,
[
@@ -118,8 +117,8 @@ def test_an_action_that_missed_its_slot_is_not_also_counted_as_a_scene_problem(t
def test_a_tolerated_miss_still_says_the_run_is_not_quotable(tmp_path, capsys):
- # Exit 0 here claims only that the harness was not the cause. The payload still has a hole in
- # it, and saying so is the difference between tolerating a miss and hiding one.
+ # Exit 0 here claims only that the harness was not the cause. The payload still has a hole in it,
+ # and saying so is the difference between tolerating a miss and hiding one.
path = write_payload(tmp_path, [cell([ran("settings", slot_missed = True)])])
assert main(["--assert-liveness", str(path), "--allow-slot-misses", "1"]) == 0
assert "Do not quote a number from this payload" in capsys.readouterr().out
@@ -131,11 +130,11 @@ def test_negative_slack_is_treated_as_none(tmp_path):
def test_a_not_run_allowance_does_not_swallow_a_missed_slot(tmp_path):
- # Where the two rules above meet. `--allow-not-run` excuses an action the fixture cannot mount
- # at all, and a missed slot is a fact about the machine, so the slot is classified FIRST and
- # the allowance never reaches it. Checked the other way round, a listed name would vanish from
- # both buckets: not a scene problem, and not a missed slot either, so `--allow-slot-misses`
- # would silently stop counting exactly the actions most likely to overrun.
+ # Where the two rules above meet. `--allow-not-run` excuses an action the fixture cannot mount at
+ # all, and a missed slot is a fact about the machine, so the slot is classified FIRST and the
+ # allowance never reaches it. Checked the other way round, a listed name would vanish from both
+ # buckets and `--allow-slot-misses` would silently stop counting exactly the actions most likely
+ # to overrun.
path = write_payload(
tmp_path,
[cell([{"action": "image_upload", "ran": False, "slot_missed": True, "reason": "late"}])],
@@ -188,15 +187,15 @@ def test_an_action_whose_own_assertion_failed_fails(tmp_path):
def test_an_action_that_passed_its_assertion_still_passes(tmp_path):
- # The pair for the test above: `expect_ok = True` is the healthy recording, and a gate that
- # failed on it would be unusable rather than strict.
+ # The pair for the test above: `expect_ok = True` is the healthy recording, and a gate that failed
+ # on it would be unusable rather than strict.
path = write_payload(tmp_path, [cell([ran("keystroke", expect_ok = True)])])
assert main(["--assert-liveness", str(path)]) == 0
def test_an_unattempted_action_reports_not_run_rather_than_its_assertion(tmp_path):
- # `ActionResult.__post_init__` forces `expect_ok = None` when `ran` is False, so the two
- # failures never blur: the reader is told the action never happened, not that it misbehaved.
+ # `ActionResult.__post_init__` forces `expect_ok = None` when `ran` is False, so the reader is
+ # told the action never happened, not that it misbehaved.
path = write_payload(
tmp_path,
[cell([{"action": "message_menu", "ran": False, "expect_ok": None, "reason": "no slot"}])],
@@ -205,9 +204,8 @@ def test_an_unattempted_action_reports_not_run_rather_than_its_assertion(tmp_pat
def test_a_failed_assertion_is_a_scene_problem_that_slack_cannot_excuse(tmp_path):
- # The third bucket meeting the first. An assertion that failed says the surface is broken, not
- # that the machine was slow, so it belongs with the scene problems and no amount of
- # `--allow-slot-misses` may buy it a pass.
+ # The third bucket meeting the first: an assertion that failed says the surface is broken, not
+ # that the machine was slow, so it belongs with the scene problems.
path = write_payload(
tmp_path,
[cell([ran("message_menu", expect_ok = False, reason = "the menu opened with no items")])],
@@ -250,8 +248,8 @@ def test_non_cell_rows_are_ignored_but_do_not_count_as_cells(tmp_path):
cell([ran("keystroke")]),
],
)
- # The stray top-level action row must not fail the run: actions are read from inside their
- # cell, so counting them twice would make the gate depend on payload layout.
+ # The stray top-level action row must not fail the run: actions are read from inside their cell,
+ # so counting them twice would make the gate depend on payload layout.
assert main(["--assert-liveness", str(path)]) == 0
@@ -264,7 +262,6 @@ def test_blank_lines_are_skipped(tmp_path, blank):
# ── a cell that was re-run is judged on the run that finished it ─────
-
NOW = "s-now"
OLD = "s-before"
@@ -322,8 +319,8 @@ def test_the_superseded_attempt_does_not_count_as_a_second_cell(tmp_path):
assert main(["--assert-liveness", str(path)]) == 0
finally:
m._log = real
- # The summary names scene problems and missed slots apart on this branch, so the count is
- # asserted on its own rather than against one spelling of the rest of the line.
+ # The summary names scene problems and missed slots apart on this branch, so the count is asserted
+ # on its own rather than against one spelling of the rest of the line.
assert any("1 cell(s)" in line and "0 scene problem(s)" in line for line in logged), logged
diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py b/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py
index 863572fb2e3..84e56a9f809 100644
--- a/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py
+++ b/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py
@@ -35,8 +35,9 @@ from studiobench.report.overhead import ( # noqa: E402
from studiobench.scoring import Measure # noqa: E402
-# ---------------------------------------------------------------------------------------
# the layout-cost adapter
+
+
# ---------------------------------------------------------------------------------------
@@ -110,8 +111,9 @@ def test_layoutcost_js_parses():
assert result.returncode == 0, result.stderr
-# ---------------------------------------------------------------------------------------
# the overhead gate
+
+
# ---------------------------------------------------------------------------------------
diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_report.py b/tests/studio/studiobench/report/selftest/test_studiobench_report.py
index 337b0b2ab64..6041594a0d6 100644
--- a/tests/studio/studiobench/report/selftest/test_studiobench_report.py
+++ b/tests/studio/studiobench/report/selftest/test_studiobench_report.py
@@ -54,8 +54,9 @@ def _metrics(keystroke_ms: float = 25.0) -> dict[str, Measure]:
}
-# ---------------------------------------------------------------------------------------
# incremental payload
+
+
# ---------------------------------------------------------------------------------------
@@ -151,8 +152,9 @@ def test_an_empty_excluded_block_is_still_printed_as_a_claim():
render_excluded({"excluded_cells": None})
-# ---------------------------------------------------------------------------------------
# editorial policy
+
+
# ---------------------------------------------------------------------------------------
diff --git a/tests/studio/studiobench/runtime/ab.py b/tests/studio/studiobench/runtime/ab.py
index b107ad55f27..fbfad2a5abc 100644
--- a/tests/studio/studiobench/runtime/ab.py
+++ b/tests/studio/studiobench/runtime/ab.py
@@ -197,27 +197,23 @@ def order_is_balanced(plan: list[tuple[Target, Cell, RungPlan]]) -> bool:
continue
seen.add(key)
first_counts[target.label] += 1
- # Every label is seeded at zero first. Counting only the labels that DID run first reports a
- # single-rep plan -- where one side always goes first and nothing cancels -- as balanced,
- # which is the one answer this function exists to prevent.
+ # Every label is seeded at zero first. Counting only the labels that DID run reports a single-rep
+ # plan, where one side always goes first and nothing cancels, as balanced, which is the one
+ # answer this function exists to prevent.
return len(labels) > 1 and len(set(first_counts.values())) == 1
#: The per-cell gates whose failure means the cell's TIMINGS ARE NOT A READING OF THE BUILD, and
#: therefore the only ones that may take the whole cell out of the ratios.
-#:
-#: NAMED RATHER THAN "ANY FAILED GATE", because a per-cell gate is not automatically fatal and the
-#: one that is not says so itself. `timer_clamp` fails whenever idle calibration cannot establish a
-#: floor -- an overloaded machine, or simply the frames instrument not being loaded -- and
-#: `session.py` is explicit that this is "NOT fatal, and NOT silently zero": blocked time is a
-#: subtraction against that floor, so `busy_pct` is null with the reason attached AND EVERY OTHER
-#: COLUMN STANDS. Excluding the cell for it would delete keystroke latency, frame and census
-#: readings that were measured correctly, and would do it most often on exactly the machines least
-#: able to spare a repetition.
-#:
+#: NAMED RATHER THAN 'ANY FAILED GATE', because a per-cell gate is not automatically fatal.
+#: `timer_clamp` fails whenever idle calibration cannot establish a floor, and `session.py` is
+#: explicit that this is 'NOT fatal, and NOT silently zero': `busy_pct` is null with the reason
+#: attached AND EVERY OTHER COLUMN STANDS. Excluding the cell would delete keystroke latency,
+#: frame and census readings that were measured correctly, most often on the machines least able
+#: to spare a repetition.
#: The two below are different in kind: both say the FILM ITSELF was wrong. A thread that lost
-#: messages and a reply that stopped being rendered do not produce a suspect column, they produce a
-#: cheaper cell, and there is no metric in it that can be trusted afterwards.
+#: messages and a reply that stopped being rendered produce a cheaper cell, not a suspect column,
+#: and no metric in it can be trusted afterwards.
INVALIDATING_CELL_GATES: frozenset[str] = frozenset({"thread_complete", "follows_the_stream"})
@@ -304,10 +300,9 @@ def failed_invalidating_gates(records: Sequence[Mapping[str, Any]]) -> dict[str,
if keep is not None and row.get("session_id") not in (None, keep):
continue
detail = row.get("detail") if isinstance(row.get("detail"), dict) else {}
- # NOT MEASURED IS NOT FAILED. See `gate_detail_is_unmeasured`, which is shared with
- # `sweep/ui_parity.py` so the two admission lists cannot drift apart on it. Readiness now
- # refuses a cell with no thread viewport outright, so that narrowing is the second of two
- # doors on the same hole.
+ # NOT MEASURED IS NOT FAILED. See `gate_detail_is_unmeasured`, shared with `sweep/ui_parity.py` so
+ # the two admission lists cannot drift. Readiness now refuses a cell with no thread viewport
+ # outright, so that narrowing is the second of two doors on the same hole.
if gate_detail_is_unmeasured(detail):
continue
why = detail.get("reason") or detail.get("coverage_reason") or "the cell's own self-check"
@@ -339,14 +334,13 @@ def unmeasured_planned_cells(
"""
from ..scoring.from_payload import latest_attempt_rows
- # THE SAME TWO FILTERS `readings_by_arm` APPLIES, because this function exists to notice the
- # holes that one punches. It drops a cell for `completed is not True` AND for a failed
- # invalidating gate; reading only the first left the second kind of hole invisible. A cell that
- # completed but lost its thread's middle, or whose reply stopped being rendered, is removed
- # from the ratios here and takes its healthy partner with it through the arm intersection in
- # `compare_arms`, while this said the plan was whole -- so `ab.md` published a verdict over
- # the rungs that survived instead of the VOID that is the point of the guard. That is the
- # partial-plan selection bias, arriving by the gate road instead of the crash road.
+ # THE SAME TWO FILTERS `readings_by_arm` APPLIES, because this function exists to notice the holes
+ # that one punches: it drops a cell for `completed is not True` AND for a failed invalidating
+ # gate, and reading only the first left the second kind invisible. A cell that completed but lost
+ # its thread's middle is removed from the ratios and takes its healthy partner with it through
+ # the arm intersection, while this said the plan was whole, so `ab.md` published a verdict over
+ # the surviving rungs instead of the VOID that is the point of the guard.
+ # The intersection is `compare_arms`.
failed = failed_invalidating_gates(records)
complete: set = set()
for row in latest_attempt_rows(records):
@@ -400,8 +394,8 @@ def readings_by_arm(
"""
from ..scoring.from_payload import latest_attempt_rows, measures_by_cell
- # The session filter below scopes the CELL rows, but `action` and `window` rows are collected
- # by `cell_id` alone, and a resumed retry reuses the cell id of the attempt that died. Without
+ # The session filter below scopes the CELL rows, but `action` and `window` rows are collected by
+ # `cell_id` alone and a resumed retry reuses the cell id of the attempt that died, so without
# this the completed-cell filter admitted the dead attempt's windows into the retry's reading.
records = list(latest_attempt_rows(records))
failed_gates = failed_invalidating_gates(records)
@@ -460,10 +454,9 @@ def compare_arms(
weights_id = weights_id() if callable(weights_id) else str(weights_id),
session_id = session_id,
)
- # Paired PER REPETITION, matching (rung, rep) on both sides. Repetition r of the base and
- # repetition r of the treatment ran adjacent in time, so pairing them is what makes the
- # comparison paired at all; pooling reps into one reading per rung throws away every
- # observation but the first and leaves the bootstrap with nothing to resample.
+ # Paired PER REPETITION, matching (rung, rep) on both sides: repetition r of each arm ran adjacent
+ # in time, which is what makes the comparison paired at all. Pooling reps into one reading per
+ # rung throws away every observation but the first and leaves the bootstrap nothing to resample.
pairs = []
for key in sorted(set(base) & set(treatment)):
rung, _rep = key
diff --git a/tests/studio/studiobench/runtime/browser.py b/tests/studio/studiobench/runtime/browser.py
index 7b327a6f493..4a2d223329d 100644
--- a/tests/studio/studiobench/runtime/browser.py
+++ b/tests/studio/studiobench/runtime/browser.py
@@ -23,12 +23,11 @@ from pathlib import Path
from typing import Any, Callable, Optional
-# The engine matching the tester's desktop webview family. Unsloth ships as a Tauri app, so the
-# thing a user actually looks at is a system webview, not the browser they happen to have.
-# Windows -> WebView2, which is Chromium: `channel="msedge"` is the closest available.
-# macOS -> WKWebView, which is WebKit.
-# Linux -> WebKitGTK. Playwright's `webkit` is NOT WebKitGTK; it is a different embedding of
-# the same engine, so it is labelled A PROXY and never as the real thing.
+# The engine matching the tester's desktop webview family. Unsloth ships as a Tauri app, so what a
+# user looks at is a system webview, not the browser they happen to have: Windows -> WebView2,
+# which is Chromium, so `channel="msedge"` is the closest available; macOS -> WKWebView, which is
+# WebKit; Linux -> WebKitGTK, which Playwright's `webkit` is NOT (a different embedding of the same
+# engine), so it is labelled A PROXY.
def default_engine() -> tuple[str, dict, str]:
system = platform.system()
if system == "Windows":
@@ -118,9 +117,9 @@ def launch(
browser = factory.launch(headless = headless)
context = browser.new_context(
viewport = {"width": viewport[0], "height": viewport[1]},
- # Clipboard read is what proves the Copy action actually copied. Without it the action
- # still runs and the assertion reports that it could not be proved, which is the honest
- # outcome rather than a pass.
+ # Clipboard read is what proves the Copy action actually copied. Without it the action still runs
+ # and the assertion reports that it could not be proved, which is the honest outcome rather than a
+ # pass.
permissions = ["clipboard-read", "clipboard-write"] if chosen == "chromium" else None,
)
if robust is not None:
@@ -131,11 +130,11 @@ def launch(
for script in init_scripts or []:
context.add_init_script(script)
page = context.new_page()
- # A GLOBAL CEILING on Playwright's actionability waits. The default is 30 seconds, which is
- # longer than most of this suite's slot budgets: one action waiting out the default on an
- # element that is present but not visible overran a 9-second slot by more than three times,
- # and the scheduler cannot intervene because the action still holds its own window. Actions
- # that legitimately need longer pass an explicit timeout.
+ # A GLOBAL CEILING on Playwright's actionability waits. The default is 30 seconds, longer than most
+ # of this suite's slot budgets: one action waiting out the default on an element that is present
+ # but not visible overran a 9-second slot by more than three times, and the scheduler cannot
+ # intervene because the action still holds its own window. Actions that legitimately need longer
+ # pass an explicit timeout.
page.set_default_timeout(8000)
cdp = None
if chosen == "chromium":
diff --git a/tests/studio/studiobench/runtime/bundle_guard.py b/tests/studio/studiobench/runtime/bundle_guard.py
index 2125f724020..31ccf54b7f1 100644
--- a/tests/studio/studiobench/runtime/bundle_guard.py
+++ b/tests/studio/studiobench/runtime/bundle_guard.py
@@ -39,10 +39,10 @@ from typing import Optional
_WINDOW = 4000
# BACKTICKS as well as quotes. Unsloth's production bundle is minified with a pass that rewrites
-# short string literals as template literals, so the marker on the shipped build reads
-# ``rendererPackageName:`react-dom` `` and a quote-only pattern misses it on every asset -- which
-# reads as "build mode could not be established" on a perfectly good production build, i.e. the
-# gate fails exactly where it is supposed to pass. Verified against the shipped bundle.
+# short string literals as template literals, so the marker reads with backticks and a quote-only
+# pattern misses it on every asset, which reads as "build mode could not be established" on a
+# perfectly good production build. Verified against the shipped bundle.
+# The marker reads ``rendererPackageName:`react-dom` ``.
_RENDERER_RE = re.compile(r'rendererPackageName\s*:\s*["\'`]react-dom["\'`]')
_BUNDLETYPE_RE = re.compile(r"bundleType\s*:\s*([01])")
@@ -66,8 +66,8 @@ class BundleVerdict:
"entry_url": self.entry_url,
"entry_bytes": self.entry_bytes,
"checked": self.checked,
- # Attempted flags, because "bundle_type is None" and "bundle_type is 0" are opposite
- # findings and a bare null cannot say which.
+ # Attempted flags, because "bundle_type is None" and "bundle_type is 0" are opposite findings and a
+ # bare null cannot say which.
"bundle_type_attempted": self.entry_url is not None,
"vite_probe_attempted": self.vite_client_status is not None,
}
@@ -115,8 +115,8 @@ def _entry_urls(base_url: str, html: str) -> list[str]:
for src in re.findall(r'<script[^>]+src=["\']([^"\']+)["\']', html):
url = src if src.startswith("http") else base_url.rstrip("/") + "/" + src.lstrip("/")
out.append(url)
- # A modulepreload is how Vite names the real entry chunk when the script tag points at a
- # loader shim, so it is worth following when the script tags carry nothing.
+ # A modulepreload is how Vite names the real entry chunk when the script tag points at a loader
+ # shim, so it is worth following when the script tags carry nothing.
for href in re.findall(
r'<link[^>]+rel=["\']modulepreload["\'][^>]+href=["\']([^"\']+)["\']', html
):
@@ -182,9 +182,8 @@ def check_bundle(base_url: str) -> BundleVerdict:
if match is None:
checked.append(f"{url}: no rendererPackageName marker, {len(raw)} bytes")
continue
- # The SAME chunk, and within a small window of the marker. A bundle this size carries
- # several `bundleType` occurrences and picking the wrong one is picking a different
- # package's answer to a question about react-dom.
+ # The SAME chunk, and within a small window of the marker: a bundle this size carries several
+ # `bundleType` occurrences and picking the wrong one is picking a different package's answer.
lo = max(0, match.start() - _WINDOW)
hi = min(len(text), match.end() + _WINDOW)
found = _BUNDLETYPE_RE.search(text, lo, hi)
diff --git a/tests/studio/studiobench/runtime/lifecycle.py b/tests/studio/studiobench/runtime/lifecycle.py
index f0917111049..d690c304f1d 100644
--- a/tests/studio/studiobench/runtime/lifecycle.py
+++ b/tests/studio/studiobench/runtime/lifecycle.py
@@ -44,9 +44,8 @@ class StudioInstall:
bootstrap_password: Optional[str] = None
port: Optional[int] = None
pid: Optional[int] = None
- #: The commit `branch` RESOLVED TO, which is the build that was installed. `branch` is what
- #: the caller typed, and a branch or a movable tag is not a build. See
- #: `__main__.commit_problems`.
+ #: The commit `branch` RESOLVED TO, which is the build that was installed. `branch` is what the
+ #: caller typed, and a branch or a movable tag is not a build. See `__main__.commit_problems`.
commit: Optional[str] = None
@property
@@ -54,14 +53,14 @@ class StudioInstall:
return f"http://127.0.0.1:{self.port}"
-#: What the backend gives an access token: `auth/authentication.py`, ACCESS_TOKEN_EXPIRE_MINUTES.
-#: Only a FALLBACK. The expiry actually enforced is the `exp` claim in the token this run was
-#: handed, which is read from the token itself; this number is what to assume when the server
-#: hands back something that is not a readable JWT.
+#: What the backend gives an access token (`auth/authentication.py`,
+#: ACCESS_TOKEN_EXPIRE_MINUTES). Only a FALLBACK: the expiry enforced is the `exp` claim read
+#: from the token itself, and this is what to assume when the server hands back something that
+#: is not a readable JWT.
ACCESS_TOKEN_TTL_S = 60 * 60
#: How far ahead of `exp` a token is replaced. Seeding a 1M-token thread is ONE request with a
-#: 900 second timeout, so a token that is merely valid at the moment the request is written is not
-#: good enough: it has to still be valid when the server finishes reading the body.
+#: 900 second timeout, so a token merely valid when the request is written is not good enough:
+#: it must still be valid when the server finishes reading the body.
TOKEN_REFRESH_MARGIN_S = 15 * 60
@@ -115,7 +114,7 @@ class StudioAuth:
base_url: str
username: str
password: str
- #: Unix seconds, from the token's own `exp`. None means "unknown", treated as `ACCESS_TOKEN_TTL_S`
+ #: Unix seconds, from the token's own `exp`. None means unknown, treated as `ACCESS_TOKEN_TTL_S`
#: from the moment this object was built.
expires_at: Optional[float] = None
#: Called with `self` after the token is replaced. The browser context seeds its localStorage
@@ -123,9 +122,9 @@ class StudioAuth:
on_rotate: Optional[Callable[["StudioAuth"], None]] = None
rotations: int = field(default = 0, init = False)
#: Turned off the first time a FRESH token still reads as expiring, which means the `exp` this
- #: process reads and the server's own clock do not agree. See `rotate`.
+ #: process reads and the server's clock do not agree. See `rotate`.
proactive: bool = field(default = True, init = False)
- #: The last `on_rotate` failure, kept rather than raised. See `rotate`.
+ #:The last `on_rotate` failure, kept rather than raised. See `rotate`.
hook_error: Optional[str] = field(default = None, init = False)
def __post_init__(self) -> None:
@@ -281,9 +280,9 @@ def register_provider(base_url: str, auth: StudioAuth, provider: ProviderSeed) -
"""
existing = auth_request_json(auth, f"{base_url.rstrip('/')}/api/providers/") or []
for row in existing:
- # Idempotent across runs. Every run binds a NEW ephemeral pacer port, so a stale entry
- # from a previous run points at a port nothing is listening on, and leaving it there gives
- # the picker two identically named models of which one is dead.
+ # Idempotent across runs. Every run binds a NEW ephemeral pacer port, so a stale entry points at
+ # a port nothing is listening on and gives the picker two identically named models of which one
+ # is dead.
if row.get("display_name") == provider.name:
try:
auth_request_json(
@@ -410,8 +409,8 @@ def checkout_ref(repo: Path, ref: str) -> str:
"""
fetched = _run(["git", "fetch", "--tags", "origin", ref], cwd = repo, check = False)
if fetched.returncode != 0:
- # A ref the remote will not serve by name (an old server, or `ref^1`, which is a local
- # expression rather than something to ask for). Fetch everything and resolve it here.
+ # A ref the remote will not serve by name (an old server, or `ref^1`, a local expression). Fetch
+ # everything and resolve it here.
_run(["git", "fetch", "--tags", "origin"], cwd = repo, check = False)
candidates = [] if fetched.returncode != 0 else ["FETCH_HEAD"]
candidates += [f"origin/{ref}", ref]
@@ -432,10 +431,9 @@ def checkout_ref(repo: Path, ref: str) -> str:
)
-#: What `install.sh` is allowed. A multi-gigabyte download and build, documented in the README as
-#: "budgeted at up to 45 minutes" and explicitly NOT part of a tier's wall clock. Named rather
-#: than inlined because the run's watchdog has to add it to a deadline it would otherwise fire in
-#: the middle of: see `watchdog_deadline_s`.
+#: What `install.sh` is allowed. A multi-gigabyte download and build, documented as 'budgeted at
+#: up to 45 minutes' and explicitly NOT part of a tier's wall clock. Named rather than inlined
+#: because the run's watchdog has to add it to a deadline it would otherwise fire inside.
INSTALL_TIMEOUT_S = 60 * 45
@@ -452,11 +450,11 @@ def install_studio(
if not (reuse_clone and (repo / ".git").exists()):
if repo.exists():
shutil.rmtree(repo)
- # Cloned WITHOUT `--branch`, then moved onto the ref locally, so one code path serves a
- # branch, a tag and a commit sha instead of two that disagree about what a ref is.
+ # Cloned WITHOUT `--branch`, then moved onto the ref locally, so one code path serves a branch,
+ # a tag and a commit sha instead of two that disagree about what a ref is.
_run(["git", "clone", remote, str(repo)])
- # KEPT, not discarded. `checkout_ref` is the only place that knows which commit a ref names,
- # and the ref alone cannot tell a resumed run that `main` moved underneath it.
+ # KEPT, not discarded: `checkout_ref` is the only place that knows which commit a ref names, and
+ # the ref alone cannot tell a resumed run that `main` moved underneath it.
commit = checkout_ref(repo, branch)
install_sh = repo / "install.sh"
if not install_sh.exists():
@@ -507,8 +505,8 @@ def _read_bootstrap_password(home: Path, log_path: Path, deadline: float) -> Opt
#: How long to keep looking for the launched server's pid. It appears once the server has forked
-#: and exec'd, which is after `setsid -f` has already returned, so this is a poll rather than a
-#: read. Named so a test can set it to zero instead of sleeping through it.
+#: and exec'd, after `setsid -f` has returned, so this is a poll rather than a read. Named so a
+#: test can set it to zero.
PID_DISCOVERY_TIMEOUT_S = 15.0
@@ -566,20 +564,18 @@ def launch_studio(
healthz_timeout_s: int = 240,
password_timeout_s: int = 30,
) -> StudioInstall:
- # AN OCCUPIED PORT IS REFUSED BEFORE ANYTHING IS LAUNCHED, and this is the half of the
- # abandoned-server failure that no cleanup can reach. `--keep-studio` asks for an Unsloth to be
- # LEFT RUNNING on this port, so the next run's `unsloth studio -p <port>` finds one of our own
- # servers there and aborts rather than binding (`studio/backend/run.py`, `_resolve_port` with
- # `avoid_own_studio`); when the pid record is not readable it falls back to the NEXT port
- # instead. Either way nothing this run launched is on `port`.
- #
- # Everything downstream then agrees that the launch worked. `_discover_pid` pgreps for
- # `unsloth studio.*-p <port>` and finds the OLD process; `wait_for_healthz` takes its 200 from
- # it; and `authenticate` retries with `BENCH_PASSWORD`, which a previous studiobench run has
- # already rotated that Unsloth to, so the login succeeds as well. The run then measures the
- # build the PREVIOUS invocation installed while `run_meta` records the ref this one asked for,
- # and `stop_studio` kills the server the caller asked to keep. There is no reading anywhere
- # that says which build answered, so this is refused rather than reported.
+ # AN OCCUPIED PORT IS REFUSED BEFORE ANYTHING IS LAUNCHED, the half of the abandoned-server
+ # failure no cleanup can reach. `--keep-studio` leaves an Unsloth running on this port, so the
+ # next run's `unsloth studio -p <port>` finds one of our own servers and aborts rather than
+ # binding, or falls back to the NEXT port when the pid record is unreadable. Either way nothing
+ # this run launched is on `port`.
+ # `studio/backend/run.py::_resolve_port`, `avoid_own_studio`.
+ # Everything downstream then agrees the launch worked: `_discover_pid` pgreps and finds the OLD
+ # process, `wait_for_healthz` takes its 200 from it, and `authenticate` retries with
+ # `BENCH_PASSWORD`, which a previous run has already rotated that Unsloth to. The run measures
+ # the build the PREVIOUS invocation installed while `run_meta` records the ref this one asked
+ # for, and `stop_studio` kills the server the caller asked to keep. No reading anywhere says
+ # which build answered.
if port_is_busy(port):
holder = _discover_pid(port, 0.0)
raise RuntimeError(
@@ -595,7 +591,7 @@ def launch_studio(
bin_path = _find_unsloth_bin(install)
env = {"UNSLOTH_STUDIO_HOME": str(install.home), **(extra_env or {})}
# NOT `UNSLOTH_STUDIO_BLOCK_PRIVATE_PROVIDER_URLS=1`: that opt-in SSRF guard rejects any
- # non-global address, which is exactly what the pacer's 127.0.0.1 base URL is.
+ # non-global address, which is exactly the pacer's 127.0.0.1 base URL.
env.pop("UNSLOTH_STUDIO_BLOCK_PRIVATE_PROVIDER_URLS", None)
cmd = [
"setsid",
@@ -615,19 +611,17 @@ def launch_studio(
install.bootstrap_password = _read_bootstrap_password(
install.home, log_path, time.time() + password_timeout_s
)
- # BEFORE the health check, not after it. An Unsloth that starts and stays unhealthy used to raise
- # here with `install.pid` still None, and `stop_studio` has nothing to kill without it -- so the
- # detached server was left running on the requested port while the CLI unwound. It is not idle
- # there: the next attempt's `unsloth studio -p <port>` finds one of our own servers on the port
- # and aborts rather than binding (`run.py:_resolve_port`, `avoid_own_studio`), while
- # `wait_for_healthz` gets its 200 from the STALE process -- which by then may have finished
- # starting. That run measures the build the previous attempt installed and records the ref this
- # one asked for, which is the one failure this harness may never produce quietly.
+ # BEFORE the health check, not after. An Unsloth that starts and stays unhealthy used to raise
+ # here with `install.pid` still None, so `stop_studio` had nothing to kill and the detached
+ # server was left on the requested port. The next attempt then aborts rather than binding while
+ # `wait_for_healthz` gets its 200 from the STALE process, measuring the build the previous
+ # attempt installed while recording the ref this one asked for: the one failure this harness may
+ # never produce quietly.
install.pid = _discover_pid(port)
healthy = wait_for_healthz(install.base_url, healthz_timeout_s)
if install.pid is None:
- # One more look: a server slow enough to miss the window above is exactly the one whose
- # health check just timed out, and it is the one that most needs to be terminated.
+ # One more look: a server slow enough to miss the window above is exactly the one whose health
+ # check just timed out, and the one that most needs terminating.
install.pid = _discover_pid(port, 0.0)
if not healthy:
stop_studio(install)
@@ -678,11 +672,10 @@ def authenticate(
the run simply fails to seed a thread and reports an empty one. So the gate is read from
`/api/auth/status` up front and cleared through the one endpoint that accepts the gated token.
"""
- # Try the supplied password, then the one a PREVIOUS studiobench run rotated to. Unsloth mints
- # a bootstrap password and demands it be changed; this function is what changes it, so the
- # second run against the same home is handed a bootstrap password that no longer exists and
- # gets a 401 whose message sends you to `reset-password`. Measured on the second run against
- # a home the first run had already set up. Trying both makes reruns and `--resume` work.
+ # Try the supplied password, then the one a PREVIOUS studiobench run rotated to. Unsloth mints a
+ # bootstrap password and demands it be changed, and this function is what changes it, so the
+ # second run against the same home is handed a password that no longer exists and gets a 401
+ # pointing at `reset-password`. Trying both makes reruns and `--resume` work.
attempts = [password, new_password] if password != new_password else [password]
auth = None
last: Optional[Exception] = None
diff --git a/tests/studio/studiobench/runtime/readiness.py b/tests/studio/studiobench/runtime/readiness.py
index 7d37162444a..f298849bfac 100644
--- a/tests/studio/studiobench/runtime/readiness.py
+++ b/tests/studio/studiobench/runtime/readiness.py
@@ -114,21 +114,21 @@ from dataclasses import dataclass, field
from typing import Any, Callable, Optional
#: The two gate modes. `full` is the historical behaviour plus the settle and end-present
-#: conditions; it is STRICTLY STRONGER than what shipped and is what every normal arm runs.
-#: `windowed` is the one an arm that deliberately mounts fewer nodes may ask for.
+#: conditions, STRICTLY STRONGER than what shipped, and is what every normal arm runs.
+#: `windowed` is for an arm that deliberately mounts fewer nodes.
MODE_FULL = "full"
MODE_WINDOWED = "windowed"
MODES = (MODE_FULL, MODE_WINDOWED)
-#: How far apart the two agreeing samples must be. 600ms rather than the old 500ms poll, because
-#: the thing being ruled out is a mount loop that pauses for a frame, and a gap shorter than a
-#: couple of React commits can be spanned by one.
+#: How far apart the two agreeing samples must be. 600ms rather than the old 500ms poll,
+#: because what is being ruled out is a mount loop that pauses for a frame, and a shorter gap
+#: can be spanned by one.
STABLE_GAP_MS = 600
-#: How many consecutive agreeing samples are needed. Two, i.e. one confirmed repeat.
+#:How many consecutive agreeing samples are needed. Two, i.e. one confirmed repeat.
STABLE_SAMPLES = 2
-#: Bottom tolerance, in CSS pixels. Not zero: a virtualised list settles on an estimated total
-#: height and corrects it as real rows are measured, so it lands a few pixels off the exact bottom
-#: and stays there. 24px is under one line of text, so it cannot hide a missing message.
+#: Bottom tolerance in CSS pixels. Not zero: a virtualised list settles on an estimated total
+#: height and lands a few pixels off. 24px is under one line of text, so it cannot hide a
+#: missing message.
BOTTOM_TOLERANCE_PX = 24
DEFAULT_TIMEOUT_S = 180
@@ -146,8 +146,8 @@ class ThreadNotReady(TimeoutError):
self.detail = detail
-#: One reading of the page. Cheap enough to run every poll: the expensive part is
-#: `getElementsByTagName('*').length`, which is a live HTMLCollection length and not a walk.
+#: One reading of the page. Cheap enough to run every poll:
+#: `getElementsByTagName('*').length` is a live HTMLCollection length, not a walk.
PROBE_JS = """
(args) => {
const [marker, tailChars] = args;
@@ -241,10 +241,9 @@ PROBE_JS = """
}
"""
-#: The keys that must AGREE between two samples for the page to count as settled. Deliberately
-#: three quantities that move for three different reasons: the message list growing, any subtree
-#: growing (a highlighter still colouring a fence moves `elements` while `mounted` sits still), and
-#: the laid-out height changing (a virtualised list correcting its estimated row heights).
+#: The keys that must AGREE between two samples for the page to count as settled. Three
+#: quantities that move for three different reasons: the message list growing, any subtree
+#: growing (a highlighter still colouring a fence), and the laid-out height changing.
SETTLE_KEYS = ("mounted", "elements", "scroll_height")
@@ -296,27 +295,20 @@ def evaluate(probe: dict, previous: Optional[dict], expected_messages: int, mode
"composer_present": bool(probe.get("composer")),
"any_message_mounted": (probe.get("mounted") or 0) > 0,
"settled": bool(settled),
- # The END of the thread is on screen. Both halves matter: the marker for the last user turn
- # is mounted, AND it is near the end of the mounted run rather than in the middle of it.
- # `2` because the thread ends user, assistant -- the marker's own row plus the reply.
+ # The END of the thread is on screen: the last user turn's marker is mounted AND near the end
+ # of the mounted run rather than in the middle. `2` because the thread ends user, assistant.
"end_present": bool(probe.get("marker_found"))
and (probe.get("marker_from_end") is not None and probe["marker_from_end"] <= 2),
}
if mode == MODE_WINDOWED:
- # THE VIRTUALIZER'S OWN CLAIM about the thread's length, and it has to be the truth the
- # seeder wrote. An arm that mounts a window without publishing aria-setsize has not given
- # anyone -- this harness or a screen reader -- any way to know how long the thread is, and
- # it is refused here rather than scored on the assumption that it is fine.
- #
- # UNLESS THE WHOLE THREAD IS MOUNTED, in which case there is nothing left to declare: the
- # DOM is the total. This matters at the small rungs, where the thread is shorter than the
- # window and a virtualised build mounts every message exactly like the shipped one. The
- # requirement has to be true of a fully-mounted thread as well as of a correctly windowed
- # one, or `windowed` becomes a mode that only some rungs of the same arm can pass.
- #
- # It cannot be used to slip past the gate: this branch needs `mounted >= expected`, which
- # is the full-mount condition itself, so a thread that has mounted 9 of 18 gets none of it.
+ # THE VIRTUALIZER'S OWN CLAIM about the thread's length, which has to be the truth the seeder
+ # wrote. An arm that windows without publishing aria-setsize has given nobody, this harness or
+ # a screen reader, a way to know how long the thread is.
+ # UNLESS THE WHOLE THREAD IS MOUNTED, where the DOM is the total. This matters at small rungs,
+ # where a virtualised build mounts every message like the shipped one, or `windowed` becomes a
+ # mode only some rungs of an arm can pass. It cannot be used to slip past the gate: this
+ # branch needs `mounted >= expected`, the full-mount condition itself.
fully_mounted = (probe.get("mounted") or 0) >= expected_messages
out["total_declared"] = fully_mounted or probe.get("setsize") is not None
out["total_matches_seeded"] = fully_mounted or probe.get("setsize") == expected_messages
@@ -325,22 +317,16 @@ def evaluate(probe: dict, previous: Optional[dict], expected_messages: int, mode
probe.get("posinset_count") == probe.get("mounted") and (probe.get("mounted") or 0) > 0
)
# ORDINALS THAT ARE ACTUALLY POSITIONS. The condition above only asks whether a number was
- # published on every row, and three malformed shapes satisfy it while carrying no
- # information at all: every row publishing `0`, every row publishing the same ordinal, and
- # a window numbered from 1 rather than from where it sits in the thread. Each is a real
- # virtualizer bug, each leaves the mounted set unlocatable, and each used to pass here.
- #
- # So the numbers have to behave like positions in a set: at least 1, no two rows claiming
- # the same one, and none pointing past the end those same rows declare in `aria-setsize`.
- # `setsize` is the arm's own claim and is already required to equal the seeded total by
- # `total_matches_seeded`; the seeded total stands in when no setsize was published, which
- # is the fully-mounted case below.
- #
- # THE WAIVER, and it is the same one `total_declared` carries: a thread short enough to be
- # mounted whole publishes no ordinals -- the shipped build publishes none anywhere -- and
- # there is nothing to validate. It needs `mounted >= expected`, the full-mount condition
- # itself, AND no ordinals at all, so an arm that publishes malformed ones cannot buy its
- # way out of the check by also mounting everything.
+ # published, and three malformed shapes satisfy it carrying no information: every row
+ # publishing `0`, every row publishing the same ordinal, and a window numbered from 1 rather
+ # than from where it sits. Each is a real virtualizer bug and each used to pass here.
+ # So the numbers must behave like positions in a set: at least 1, no duplicates, and none past
+ # the end those rows declare in `aria-setsize` (already required to equal the seeded total by
+ # `total_matches_seeded`; the seeded total stands in when none was published).
+ # THE WAIVER, the same one `total_declared` carries: a thread mounted whole publishes no
+ # ordinals, and the shipped build publishes none anywhere, so there is nothing to validate. It
+ # needs `mounted >= expected` AND no ordinals at all, so an arm publishing malformed ones
+ # cannot buy its way out by also mounting everything.
declared = probe.get("setsize")
if declared is None:
declared = expected_messages
@@ -353,49 +339,41 @@ def evaluate(probe: dict, previous: Optional[dict], expected_messages: int, mode
and probe["max_posinset"] <= declared
)
# AND THE WINDOW IS AT THE END OF THE THREAD BY ITS OWN NUMBERING. `end_present` proves the
- # last message's TEXT is mounted; this proves the ordinals agree with it. Without it a
- # bottom-anchored window numbered 1..6 out of 18 -- a virtualizer publishing the index
- # within the window instead of the position in the thread -- passes every other condition.
+ # last message's TEXT is mounted; this proves the ordinals agree. Without it a bottom-anchored
+ # window numbered 1..6 out of 18 passes every other condition.
out["posinset_reaches_end"] = (fully_mounted and published == 0) or (
probe.get("max_posinset") == expected_messages
)
# THE APP'S ANSWER FIRST, the arithmetic only when the app has not given one. A virtualised
- # list settles on an estimated total height and corrects it as real rows are measured, so
- # its scrollTop arithmetic can read tens of pixels off the bottom while the app -- which
- # knows it is pinned -- is perfectly happy. Trusting the arithmetic over the app would fail
- # a correct arm for a rounding error in a height estimate.
+ # list's scrollTop arithmetic can read tens of pixels off the bottom while the app, which
+ # knows it is pinned, is happy; trusting the arithmetic would fail a correct arm for a
+ # rounding error.
app_bottom = probe.get("app_says_at_bottom")
near_bottom = (
probe.get("from_bottom") is not None and probe["from_bottom"] <= BOTTOM_TOLERANCE_PX
)
out["anchored_at_end"] = bool(app_bottom) if app_bottom is not None else near_bottom
- # THE VIEWPORT ITSELF, ASSERTED RATHER THAN INFERRED. Every windowed condition above is
- # about what is inside the scroller, and each one degrades to a pass when the scroller is
- # not there: `from_bottom` is null so the arithmetic is skipped, and the app's own answer
- # is read off `.aui-thread-scroll-to-bottom`, which is a DESCENDANT of the viewport but is
- # looked up at DOCUMENT scope. So renaming the viewport class while leaving that control
- # mounted leaves `app_says_at_bottom` true, `anchored_at_end` true, and the cell admitted
- # with no viewport at all.
- #
- # What follows a cell admitted that way is silent in every direction: the completeness
- # probe returns `probe_attempted: false`, the scroll actions return `not_run` and a not-run
- # action blanks only its own timings, and the census viewport fields go null. Nothing
- # refuses, so the film is scored without the surface it was measuring.
- #
- # The signal is already probed and costs nothing to read. It is asserted only here because
- # a fully mounted arm that scrolls the window instead of a div is a shape this harness
- # supports, and `MODE_FULL` must not start requiring a scroller it never needed.
+ # THE VIEWPORT ITSELF, ASSERTED RATHER THAN INFERRED. Every windowed condition above degrades
+ # to a pass without the scroller: `from_bottom` is null so the arithmetic is skipped, and the
+ # app's own answer is read off `.aui-thread-scroll-to-bottom`, a DESCENDANT of the viewport
+ # looked up at DOCUMENT scope. So renaming the viewport class admits a cell with no viewport.
+ # What follows is silent in every direction: the completeness probe returns
+ # `probe_attempted: false`, scroll actions return `not_run` and blank only their own timings,
+ # and the census viewport fields go null. Nothing refuses, so the film is scored without the
+ # surface it was measuring.
+ # `app_says_at_bottom` and `anchored_at_end` both read true on such a cell.
+ # Asserted only here because a fully mounted arm that scrolls the window instead of a div is a
+ # shape this harness supports, and `MODE_FULL` must not start requiring a scroller.
out["viewport_present"] = bool(probe.get("viewport_present"))
- # And the pin must have FINISHED. `--aui-scroll-stabilizer` is on the viewport while
- # use-intent-aware-autoscroll is still pinning and comes off when it settles, so a page
- # carrying it is mid-scroll no matter what its scrollTop says.
+ # And the pin must have FINISHED: `--aui-scroll-stabilizer` is on the viewport while
+ # use-intent-aware-autoscroll is still pinning, so a page carrying it is mid-scroll whatever
+ # its scrollTop says.
out["pin_settled"] = not probe.get("pinning")
- # Not a gate, a sanity check with teeth: a "windowed" arm that mounted MORE rows than the
- # thread has is not windowed, it is broken, and the reading would be nonsense.
+ # Not a gate, a sanity check with teeth: a 'windowed' arm that mounted MORE rows than the
+ # thread has is broken, and the reading would be nonsense.
out["not_over_mounted"] = (probe.get("mounted") or 0) <= expected_messages
else:
out["all_messages_mounted"] = (probe.get("mounted") or 0) >= expected_messages
- # Recorded, never gated, in full mode. See the module docstring.
out["total_declared"] = None
out["total_matches_seeded"] = None
out["posinset_on_every_row"] = None
@@ -412,9 +390,8 @@ def _describe(conditions: dict, probe: dict, expected: int, mode: str) -> str:
bits = [
f"mounted {probe.get('mounted')} of {expected}",
f"aria-setsize {probe.get('setsize')}",
- # The ordinals THEMSELVES, not just how many rows carried one: "posinset 0..0 on 6 rows,
- # 1 distinct" is the whole diagnosis of a malformed contract, and reading it off the raw
- # probe afterwards is what this message exists to save.
+ # The ordinals THEMSELVES, not just how many rows carried one: 'posinset 0..0 on 6 rows, 1
+ # distinct' is the whole diagnosis of a malformed contract.
f"aria-posinset {probe.get('min_posinset')}..{probe.get('max_posinset')} on "
f"{probe.get('posinset_count')} of {probe.get('mounted')} rows, "
f"{probe.get('posinset_distinct')} distinct",
@@ -605,15 +582,14 @@ async ([toTop, steps, stepPx]) => {
}
"""
-#: How the traversal is stepped. 400 steps of 2,000px covers 800,000px of thread, which clears the
-#: tallest rung this benchmark runs, and a step that lands at either end breaks out early.
+#: How the traversal is stepped. 400 steps of 2,000px covers 800,000px, clearing the tallest
+#: rung, and a step landing at either end breaks out early.
TRAVERSE_STEPS = 400
TRAVERSE_STEP_PX = 2000
-#: The ordinals mounted right now, read exactly the way PROBE_JS reads them. Run once at the top of
-#: the thread, after the wait for the head marker: the last stop of the gesture reads its rows the
-#: instant the paint lands, and an arm that materialises a row a beat later would otherwise have
-#: that row counted as one the traversal never saw.
+#: The ordinals mounted right now, read exactly as PROBE_JS reads them. Run once at the top
+#: after the wait for the head marker: the gesture's last stop reads its rows the instant the
+#: paint lands, so an arm materialising a row a beat later would be counted as never seen.
COLLECT_ORDINALS_JS = """
() => {
const out = [];
@@ -627,30 +603,26 @@ COLLECT_ORDINALS_JS = """
}
"""
-#: How many missing ordinals a verdict names. The COUNT is always exact; the list is capped so a
-#: thread that lost half of itself does not write a thousand integers into every payload.
+#: How many missing ordinals a verdict names. The COUNT is always exact; the list is capped so
+#: a thread that lost half of itself does not write a thousand integers into every payload.
MISSING_ORDINALS_LISTED = 40
-#: WHY `ordinal_coverage` GAVE THE ANSWER IT GAVE, which the three-valued verdict on its own cannot
-#: say and which the `thread_complete` gate has to know.
-#:
-#: `not_applicable` and `unmeasured` are BOTH `None` and they are opposites. The first is a question
-#: that does not arise on this arm: a fully mounted thread publishes no `aria-posinset` anywhere --
-#: the shipped build publishes none -- so there are no ordinals to cover and none missing. The
-#: second is a question that does arise and was not answered: the gesture stopped short of the top,
-#: or its consecutive stops did not overlap, so rows nobody scrolled past are rows nobody looked at.
-#:
-#: A gate that passes both reads "we could not tell" as "it was fine", which is the store that kept
-#: only its first and last page staying scoreable. A gate that fails both fails the shipped build on
-#: every cell for publishing no ordinals. So the two are recorded apart, and only the second is
-#: withheld from a pass.
+#: WHY `ordinal_coverage` GAVE THE ANSWER IT GAVE, which the three-valued verdict cannot say
+#: and the `thread_complete` gate has to know.
+#: `not_applicable` and `unmeasured` are BOTH `None` and are opposites. The first is a question
+#: that does not arise: a fully mounted thread publishes no `aria-posinset`, so nothing is
+#: missing. The second is a question that arose and was not answered: the gesture stopped short
+#: of the top, or its stops did not overlap.
+#: A gate passing both reads 'we could not tell' as 'it was fine', which is the store that kept
+#: only its first and last page staying scoreable; a gate failing both fails the shipped build
+#: on every cell. So the two are recorded apart and only the second is withheld from a pass.
COVERAGE_COMPLETE = "complete"
COVERAGE_INCOMPLETE = "incomplete"
COVERAGE_NOT_APPLICABLE = "not_applicable"
COVERAGE_UNMEASURED = "unmeasured"
#: The coverage states a cell may still be scored on. `unmeasured` is deliberately absent, and
-#: `incomplete` is excluded by the verdict itself rather than by this tuple.
+#: `incomplete` is excluded by the verdict itself.
COVERAGE_STATES_SCOREABLE = (COVERAGE_COMPLETE, COVERAGE_NOT_APPLICABLE)
@@ -703,8 +675,8 @@ def ordinal_coverage(
seen.update(int(n) for n in (extra_seen or ()))
expected = set(range(1, expected_messages + 1)) if expected_messages > 0 else set()
missing = sorted(expected - seen)
- # Intersected with "never seen anywhere", so a row that was briefly absent while its window
- # was still materialising and mounted at the next stop is not reported as a lost message.
+ # Intersected with 'never seen anywhere', so a row briefly absent while its window materialised
+ # is not reported as a lost message.
holes = sorted({int(n) for n in (traverse.get("ordinals_in_window_holes") or [])} - seen)
out: dict[str, Any] = {
"ordinals_seen_count": len(seen),
@@ -718,9 +690,9 @@ def ordinal_coverage(
"traversal_stops": traverse.get("traversal_stops"),
"coverage_reason": None,
}
- # NOT APPLICABLE BEFORE UNMEASURED. An arm publishing no ordinals anywhere has nothing to cover
- # whatever the gesture did, and asking about the gesture first would report the shipped build's
- # own shape as a measurement this probe failed to take -- which the gate now refuses to score.
+ # NOT APPLICABLE BEFORE UNMEASURED. An arm publishing no ordinals has nothing to cover whatever
+ # the gesture did, and asking about the gesture first would report the shipped build's own
+ # shape as a failed measurement.
if not seen:
out["ordinal_coverage_complete"] = None
out["ordinal_coverage_state"] = COVERAGE_NOT_APPLICABLE
@@ -827,25 +799,24 @@ def probe_thread_completeness(
"mounted_at_top": seen.get("mounted"),
"setsize_at_top": seen.get("setsize"),
"scroll_height_at_top": top.get("scroll_height"),
- # DID THE GESTURE ACTUALLY REACH THE TOP? Without this, "the head never mounted" and
- # "the viewport never left the bottom" are the same reading, and the second one is a
- # defect in this probe being reported as data loss in the app.
+ # DID THE GESTURE ACTUALLY REACH THE TOP? Without this, 'the head never mounted' and 'the
+ # viewport never left the bottom' read the same, and the second is a defect in this probe
+ # reported as data loss in the app.
"reached_top": top.get("reached_target"),
"scroll_top_after_gesture": top.get("scroll_top"),
"traverse_step_px": step_px,
}
)
out.update(coverage)
- # `min_posinset_seen` before there were any ordinals to see: the head marker mounting means
- # position 1 was on the page, whether or not the arm numbers its rows. Kept so an arm that
- # publishes nothing still fills the field the way it always did.
+ # `min_posinset_seen` before there were ordinals to see: the head marker mounting means
+ # position 1 was on the page, whether or not the arm numbers its rows.
if out.get("min_posinset_seen") is None and found:
out["min_posinset_seen"] = 1
# And back to the end, so the cell resumes from the state the readiness gate described.
page.evaluate(TRAVERSE_JS, [False, steps, step_px])
if not found and not top.get("reached_target"):
- # NOT A VERDICT ABOUT THE ARM. The gesture did not get there, so nothing was learned about
- # what the arm holds, and saying otherwise would blame the app for the probe.
+ # NOT A VERDICT ABOUT THE ARM: the gesture did not get there, so nothing was learned, and
+ # saying otherwise would blame the app for the probe.
out["head_reached"] = None
out["reason"] = (
f"the scroll gesture never reached the top of the thread (stopped at "
@@ -860,14 +831,14 @@ def probe_thread_completeness(
)
log(f" COMPLETENESS FAILED: {out['reason']}")
elif out.get("ordinal_coverage_complete") is False:
- # THE HEAD ARRIVED AND THE THREAD IS STILL INCOMPLETE. This is the case the marker check
- # alone reported as a pass: first page kept, last page kept, middle gone.
+ # THE HEAD ARRIVED AND THE THREAD IS STILL INCOMPLETE: the case the marker check alone reported
+ # as a pass, first page kept, last page kept, middle gone.
out["reason"] = f"the head of the thread mounted, but {out['coverage_reason']}"
log(f" COMPLETENESS FAILED: {out['reason']}")
elif out.get("ordinal_coverage_state") == COVERAGE_UNMEASURED:
- # THE HEAD ARRIVED AND THE MIDDLE WAS NEVER INSPECTED. Not a finding about the arm, and
- # not a pass either: this is the same first-page-and-last-page store re-entering through
- # the unknown state, so the cell carries a reason and the gate declines to score it.
+ # THE HEAD ARRIVED AND THE MIDDLE WAS NEVER INSPECTED. Not a finding about the arm and not a
+ # pass: the same store re-entering through the unknown state, so the cell carries a reason and
+ # the gate declines to score it.
out["reason"] = (
f"the head of the thread mounted, but coverage of the middle was NOT ESTABLISHED: "
f"{out['coverage_reason']}"
@@ -881,7 +852,6 @@ def probe_thread_completeness(
f"({out.get('ordinals_seen_count')} of {expected_messages} ordinals seen)"
)
if out.get("ordinal_coverage_complete") is None:
- # Only the not-applicable kind reaches here; the unmeasured kind is caught above and
- # carries a reason of its own.
+ # Only the not-applicable kind reaches here; the unmeasured kind is caught above with its own reason.
log(f" coverage DOES NOT APPLY: {out.get('coverage_reason')}")
return out
diff --git a/tests/studio/studiobench/runtime/seeder.py b/tests/studio/studiobench/runtime/seeder.py
index 90c877d1aeb..3b88916d8eb 100644
--- a/tests/studio/studiobench/runtime/seeder.py
+++ b/tests/studio/studiobench/runtime/seeder.py
@@ -35,8 +35,8 @@ from ..fixture.corpus import RungPlan, Unit
from .lifecycle import StudioAuth, auth_request_json
# How close the two paths must land to count as equivalent. Not zero: a streamed reply carries a
-# usage record and a duration the seeded one does not, and the composer state differs, so a
-# handful of elements always differ. 2% on the quantities that scale with content.
+# usage record and a duration the seeded one does not, and the composer state differs. 2% on the
+# quantities that scale with content.
EQUIVALENCE_TOLERANCE = 0.02
@@ -55,11 +55,10 @@ def _assistant_content(unit: Unit) -> list[dict]:
parts: list[dict] = []
if unit.reasoning:
parts.append({"type": "reasoning", "text": unit.reasoning})
- # Tool calls sit BETWEEN the reasoning and the answer, which is where a real turn puts them:
- # the model thinks, calls tools, then answers. reasoning.tsx groups adjacent tool-call parts
- # with the reasoning above them, so the order decides whether a tool group renders inside the
- # collapsible pane or as its own block, and those are different components with different
- # costs.
+ # Tool calls sit BETWEEN the reasoning and the answer, where a real turn puts them. reasoning.tsx
+ # groups adjacent tool-call parts with the reasoning above them, so the order decides whether a
+ # tool group renders inside the collapsible pane or as its own block, and those are different
+ # components with different costs.
for call in unit.tool_calls:
parts.append(dict(call))
if unit.content:
@@ -85,10 +84,10 @@ class SeededThread:
seeded_chars: int
seconds: float
turns: int
- # The markers on the FIRST and LAST user turns. The readiness gate uses `last_marker` to prove
- # the end of the thread is mounted, and the completeness probe uses `first_marker` to prove a
- # windowed arm still holds the head of the conversation. Plain text written by this harness,
- # so neither is a guess about what a markdown renderer will do to the corpus.
+ # The markers on the FIRST and LAST user turns: the readiness gate uses `last_marker` to prove the
+ # end of the thread is mounted, and the completeness probe uses `first_marker` to prove a windowed
+ # arm still holds the head. Plain text written by this harness, so neither is a guess about what a
+ # markdown renderer will do.
first_marker: Optional[str] = None
last_marker: Optional[str] = None
@@ -109,10 +108,8 @@ class Seeder:
def create_thread(self, title: str = "studiobench") -> str:
thread_id = str(uuid.uuid4())
- # `auth_request_json`, not `request_json`, and that is not a spelling preference: the
- # seeder is asked for a thread once per cell for as long as the run lasts, and an access
- # token is good for 60 minutes. See `StudioAuth`: the token is re-minted before it expires
- # and once more if a 401 arrives anyway.
+ # `auth_request_json`, not `request_json`: the seeder is asked for a thread once per cell for as
+ # long as the run lasts and an access token is good for 60 minutes. See `StudioAuth`.
auth_request_json(
self.auth,
self._url("/api/chat/threads"),
@@ -173,8 +170,8 @@ class Seeder:
parent = assistant_id
started = time.monotonic()
if messages:
- # pruneMissing so this REPLACES the thread rather than merging into whatever a
- # previous cell left behind. A merge would make every rung after the first cumulative.
+ # pruneMissing so this REPLACES the thread rather than merging into whatever a previous cell left
+ # behind; a merge would make every rung after the first cumulative.
auth_request_json(
self.auth,
self._url(f"/api/chat/threads/{thread_id}/messages"),
@@ -228,19 +225,14 @@ def compare_signatures(
elements legitimately differ and gating on exact equality would fail every time for a reason
that has nothing to do with fidelity.
"""
- # GATED ON CONTENT, REPORTED ON REASONING.
- #
- # A collapsed reasoning pane in a SEEDED thread does not mount its children, while a streamed
- # one does: it was open while the text arrived, and collapsing afterwards leaves the subtree
- # in place. Measured directly, the same text carried 1,485 reasoning spans one way and 0 the
- # other, and open-then-close on the seeded side unmounted them again, so this is not something
- # seeding can be made to reproduce -- it is a property of how the app builds a thread.
- #
+ # GATED ON CONTENT, REPORTED ON REASONING. A collapsed reasoning pane in a SEEDED thread does not
+ # mount its children while a streamed one does, because it was open while the text arrived.
+ # Measured, the same text carried 1,485 reasoning spans one way and 0 the other, so this is a
+ # property of how the app builds a thread rather than something seeding can reproduce.
# Gating on total `highlight_spans` therefore asked a question seeding can never pass, and the
- # answer moved with whatever pane state the film happened to leave behind: two runs of the
- # same rung reported 2.1% and 36.4% drift. The question worth asking is whether the same text
- # renders the same CONTENT, which is what these keys are. The reasoning difference is measured
- # and reported below rather than swept into a pass or a fail.
+ # answer moved with whatever pane state the film left behind: two runs of the same rung reported
+ # 2.1% and 36.4% drift. The question worth asking is whether the same text renders the same
+ # CONTENT, and the reasoning difference is measured and reported below.
keys = ("assistant_messages", "content_code_blocks", "content_spans", "reasoning_panes")
fields: dict = {}
equivalent = True
@@ -354,9 +346,9 @@ def measure_chars_per_token(
}
except Exception: # noqa: BLE001
pass
- # Last resort, and LABELLED. Counting whitespace-delimited words plus punctuation is a rough
- # stand-in for a BPE tokeniser and is off by tens of percent on dense code, which is most of
- # this corpus.
+ # Last resort, and LABELLED: counting whitespace-delimited words plus punctuation is a rough
+ # stand-in for a BPE tokeniser and is off by tens of percent on dense code, which is most of this
+ # corpus.
words = len(sample.split())
punct = sum(1 for c in sample if not c.isalnum() and not c.isspace())
est = max(1, words + punct // 2)
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py
index cf9cb49e337..e21fd6d33f9 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py
@@ -86,13 +86,12 @@ def test_origin_gate_names_the_exact_origin_and_strips_the_slash():
assert "return" in script
-# ── the gate compares against an ORIGIN, so it has to be given one ──────────────────────────
-#
-# `window.location.origin` is the URL standard's canonical origin, not the URL the caller typed.
-# Every expectation below was read out of chromium, by navigating a real document to the spelling
-# on the left and asking the page for `window.location.origin`.
+# The gate compares against an ORIGIN, so it has to be given one. `window.location.origin` is the
+# URL standard's canonical origin, not the URL the caller typed, and every expectation below was
+# read out of chromium by navigating a real document to the spelling on the left.
+# ── the gate compares against an ORIGIN, so it has to be given one ──────────────────────────
@pytest.mark.parametrize(
("spelled", "origin"),
[
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py
index c309f40137c..f6a087c3bc5 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py
@@ -32,7 +32,7 @@ SIDES = [
{"label": "treatment", "ref": "fix", "base_url": "http://127.0.0.1:5400", "owns": True},
]
-#: (rung, tokens, base p95, treatment p95). 10K regresses by 100%, 100K improves by 20%.
+#:(rung, tokens, base p95, treatment p95). 10K regresses by 100%, 100K improves by 20%.
LADDER = (("10K", 10_000, 100.0, 200.0), ("100K", 100_000, 100.0, 80.0))
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py
index 8d20e329a53..0bc99473146 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py
@@ -38,8 +38,8 @@ def _target(label):
return Target(label = label, ref = label, base_url = f"http://x/{label}", seeder = None, runner = None)
-#: Each rung needs its OWN token count: `measures_by_cell` keys on `(rung_tokens, rep)`, so two
-#: rungs sharing one number collapse into one pair and a two-rung table renders as a one-rung one.
+#: Each rung needs its OWN token count: `measures_by_cell` keys on `(rung_tokens, rep)`, so two rungs
+#: sharing one number collapse into one pair and a two-rung table renders as a one-rung one.
RUNG_TOKENS = {"1K": 1_000, "10K": 10_000, "100K": 100_000, "500K": 500_000, "1M": 1_000_000}
@@ -188,9 +188,9 @@ def _resumed_table(tmp_path, *, pair_granular: bool) -> str:
def test_the_resumed_session_measures_both_arms_and_gets_a_table(tmp_path):
table = _resumed_table(tmp_path, pair_granular = True)
- # Both arms measured, so the pair exists and the table has a reading. One pair carries no
- # bootstrap CI, so the verdict is INCONCLUSIVE; the contrast with the test below, where the
- # arms never pair at all and the table says NO READING, is the thing under test.
+ # Both arms measured, so the pair exists and the table has a reading. One pair carries no bootstrap
+ # CI, so the verdict is INCONCLUSIVE; the contrast with the test below, where the arms never pair
+ # and the table says NO READING, is the thing under test.
assert "VERDICT: INCONCLUSIVE" in table
assert "NO READING" not in table
@@ -227,8 +227,8 @@ def _two_rung_resumed_table(tmp_path, *, whole_table: bool) -> str:
if whole_table:
done = skippable_cells(work, recorded)
else:
- # THE PRE-FIX RULE, inline so the contrast is the change itself: skip any pair whose every
- # arm is already recorded, and let the rest of the table be whatever is left.
+ # THE PRE-FIX RULE, inline so the contrast is the change itself: skip any pair whose every arm is
+ # already recorded, and let the rest of the table be whatever is left.
by_pair: dict = {}
for _t, cell, _p in work:
by_pair.setdefault((cell.rung, cell.rep), []).append(cell.cell_id)
@@ -265,9 +265,9 @@ def test_skipping_the_recorded_pair_publishes_a_verdict_over_the_remainder(tmp_p
table = _two_rung_resumed_table(tmp_path, whole_table = False)
assert "keystroke_p95_ms 1" in table, table
- # The remainder is a single pair, so it can no longer be published as a direction; the bug
- # this documents is unchanged and is the two assertions around this one: the 30% regression
- # is gone from the table and nothing names the rung it came from.
+ # The remainder is a single pair, so it can no longer be published as a direction; the bug this
+ # documents is the two assertions around it: the 30% regression is gone from the table and nothing
+ # names the rung it came from.
assert "VERDICT: INCONCLUSIVE" in table, table
assert "10K" not in table
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py
index ac40ad770f8..615319bea58 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py
@@ -45,7 +45,7 @@ from studiobench.runtime.readiness import ( # noqa: E402
from studiobench.runtime.session import record_completeness_gate # noqa: E402
from studiobench.runtime.types import Cell, Recorder, make_cell_id # noqa: E402
-#: The arm the whole windowed readiness gate exists for, at a rung big enough to be windowed.
+#:The arm the whole windowed readiness gate exists for, at a rung big enough to be windowed.
CELL = Cell(
cell_id = make_cell_id("100K", "B1", 0),
rung = "100K",
@@ -271,9 +271,8 @@ def test_every_per_cell_gate_names_its_cell():
src = inspect.getsource(S)
def _calls(text: str) -> list:
- # A PAREN COUNTER, not a regex. `rec.gate("follows_the_stream", bool(...), ...)` contains a
- # nested call, and a non-greedy regex stops at the inner closing paren -- reporting the
- # outer call as missing an argument that is two lines further down.
+ # A PAREN COUNTER, not a regex: `rec.gate("follows_the_stream", bool(...), ...)` contains a nested
+ # call, and a non-greedy regex stops at the inner closing paren.
out = []
for marker in ("rec.gate(", "recorder.gate("):
start = 0
@@ -290,9 +289,8 @@ def test_every_per_cell_gate_names_its_cell():
depth -= 1
j += 1
args = text[i + len(marker) : j - 1]
- # Skip prose. `session.py` carries a comment reading "WHY THIS IS NOT
- # `recorder.gate(...)`", and a scanner that counts it reports a defect in a
- # sentence.
+ # Skip prose: `session.py` carries a comment reading "WHY THIS IS NOT `recorder.gate(...)`", and a
+ # scanner that counts it reports a defect in a sentence.
line_start = text.rfind("\n", 0, i) + 1
is_comment = text[line_start:i].lstrip().startswith("#")
if args.strip() != "..." and not is_comment:
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py
index 6ecb6e8b617..2b47bf02402 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py
@@ -87,8 +87,7 @@ def test_composer_click_ms_excludes_the_instrument_hooks():
runner = _run()
got = runner._composer_click_ms
assert got is not None
- # The click is 100 ms and the hooks are 800 ms between them. Timed around the window this
- # came back near 900.
+ # The click is 100 ms and the hooks are 800 ms between them. Timed around the window this came back near 900.
assert CLICK_S * 1000 <= got < CLICK_S * 1000 + TEARDOWN_S * 1000
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py
index 2352260d149..a61e40fa9f4 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py
@@ -61,15 +61,14 @@ def context():
b = p.chromium.launch(args = ["--no-sandbox"])
except Exception as exc: # noqa: BLE001
pytest.skip(f"chromium could not be launched: {exc}")
- # The real harness's browser factory requests the same two. Without them the clipboard
- # read-back throws and the reading is "could not be measured", which is exactly the
- # NOT COMPARABLE outcome the scoring layer now produces rather than a pass.
+ # The real harness's browser factory requests the same two. Without them the clipboard read-back
+ # throws and the reading is "could not be measured", which is the NOT COMPARABLE outcome the
+ # scoring layer produces rather than a pass.
ctx = b.new_context(permissions = ["clipboard-read", "clipboard-write"])
- # `navigator.clipboard` DOES NOT EXIST outside a secure context, and `set_content` leaves
- # the page on about:blank, which is not one. The reading came back as
- # "Cannot read properties of undefined" rather than as an empty clipboard, which would have
- # been easy to misread as "the copy did not work". Fulfilled from a route so no server is
- # needed and no network is touched.
+ # `navigator.clipboard` DOES NOT EXIST outside a secure context, and `set_content` leaves the page
+ # on about:blank, which is not one. The reading came back as "Cannot read properties of undefined"
+ # rather than as an empty clipboard, which would have been easy to misread. Fulfilled from a route
+ # so no server is needed.
ctx.grant_permissions(["clipboard-read", "clipboard-write"], origin = ORIGIN)
yield ctx
ctx.close()
@@ -105,8 +104,7 @@ def _page(context, mode: str, copy_from_store: bool):
#: Exactly what scene/actions.select_all_copy does: focus the viewport, select its contents, then a
-#: REAL Control+C so the app's own copy handler runs. Anything else would be testing a different
-#: code path from the one the benchmark drives.
+#: REAL Control+C so the app's own copy handler runs.
SELECT_JS = """
async () => {
const v = window.__sb.dom.viewport();
@@ -180,8 +178,8 @@ def test_the_handler_puts_the_whole_conversation_on_the_clipboard(context):
page.close()
assert got["mounted"] == WINDOW < got["total"] == MESSAGES
assert _markers_present(got["clipboard"]) == TURNS, got["clipboard"][:400]
- # The selection is still short, and that is correct: it can only cover mounted nodes. This is
- # the reading the old alarm was wired to, and it is why the alarm had to be moved.
+ # The selection is still short, and that is correct: it can only cover mounted nodes. This is the
+ # reading the old alarm was wired to, and why the alarm had to be moved.
assert got["selected_chars"] < got["clipboard_chars"]
@@ -226,6 +224,6 @@ def test_a_partial_selection_is_not_replaced_by_the_whole_conversation(context):
assert clip != "SENTINEL"
assert len(clip) < 1000, clip[:300]
else:
- # Chromium gave no selection for an off-screen row, so no copy was performed at all. Said
- # out loud rather than passed silently: this run did not exercise the substitution guard.
+ # Chromium gave no selection for an off-screen row, so no copy was performed at all. Said out loud
+ # rather than passed silently: this run did not exercise the substitution guard.
assert clip == "SENTINEL"
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py
index 327610089e0..e8fb50a6295 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py
@@ -43,7 +43,7 @@ from studiobench.runtime.session import ( # noqa: E402
)
from studiobench.sweep.ui_parity import incomplete_cells # noqa: E402
-#: The coverage actually observed on the run this carve-out was derived from, not a round number.
+#:The coverage actually observed on the run this carve-out was derived from, not a round number.
OBSERVED_COVERAGE = 0.481
@@ -167,11 +167,10 @@ def test_low_coverage_does_not_launder_a_real_pinning_failure(tmp_path) -> None:
assert refused
-#: Every shape a failed `follows_the_stream` row can take, and whether it must still be fatal.
-#: The two admission lists have to agree on all of them: `INVALIDATING_CELL_GATES` was centralised
-#: so the scorers could not disagree about what invalidates a cell, and a predicate copied into
-#: both consumers reintroduces exactly that drift one level down, where it is harder to see
-#: because each copy reads correctly on its own.
+#: Every shape a failed `follows_the_stream` row can take, and whether it must still be fatal. The
+#: two admission lists have to agree on all of them: `INVALIDATING_CELL_GATES` was centralised so
+#: the scorers could not disagree about what invalidates a cell, and a predicate copied into both
+#: consumers reintroduces that drift one level down, where each copy reads correctly on its own.
_AGREEMENT_CASES: list[tuple[str, dict, bool]] = [
("coverage-only shortfall", _coverage_short(OBSERVED_COVERAGE), False),
("sliver", _coverage_short(0.13), False),
@@ -236,9 +235,9 @@ _AGREEMENT_CASES: list[tuple[str, dict, bool]] = [
True,
),
("absent instrument", {"follow_attempted": False, "reason": "sampler is not installed"}, False),
- # The narrowing that must survive the refactor: `probe_attempted: False` has two producers and
- # only one is an absent instrument. A missing thread viewport is the ARM missing the surface
- # under test, and waiving it once let a real failure ride the instrument allowance.
+ # The narrowing that must survive the refactor: `probe_attempted: False` has two producers and only
+ # one is an absent instrument. A missing thread viewport is the ARM missing the surface under
+ # test, and waiving it once let a real failure ride the instrument allowance.
("no thread viewport", {"probe_attempted": False, "reason": "no thread viewport"}, True),
]
@@ -258,11 +257,12 @@ def test_both_admission_lists_agree_on_every_shape(tmp_path) -> None:
assert not wrong, "wrong verdict: " + "; ".join(wrong)
-# ---------------------------------------------------------------------------------------
-# the WRITER. Everything above drives the two consumers with a hand-built gate detail, which
-# cannot see a defect in the thing that PRODUCES that detail: a `follow_verdict` that set
+# the WRITER. Everything above drives the two consumers with a hand-built gate detail, which cannot
+# see a defect in the thing that PRODUCES that detail: a `follow_verdict` that set
# `stream_coverage_unmeasured` on every low reading would waive real follow failures and every
-# consumer test would still pass. These drive the real writer.
+# consumer test would still pass.
+
+
# ---------------------------------------------------------------------------------------
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py
index 12cbe558bff..c6a12153023 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py
@@ -257,8 +257,8 @@ def test_a_refused_resume_rolls_back_every_row_it_wrote(tmp_path):
assert dropped > 0
assert paths.payload_jsonl.read_bytes() == before
- # The two ways the leftovers were visible: a rung the payload never owed, and a failed gate
- # charged to somebody else's evidence.
+ # The two ways the leftovers were visible: a rung the payload never owed, and a failed gate charged
+ # to somebody else's evidence.
assert recorded_ladder(paths.payload_jsonl) == ["1K", "10K", "100K"]
assert [
r for r in _rows(paths) if r.get("row_type") == "gate" and r.get("passed") is False
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py
index 56bf2facebd..d607a5b96a7 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py
@@ -59,8 +59,8 @@ def launched(monkeypatch, tmp_path):
}
# `raising = False` so this fixture also builds against a lifecycle without the constant, which
- # is what makes the test below fail on the unfixed code for the reason it is about rather than
- # on the way in.
+ # makes the test below fail on the unfixed code for the reason it is about rather than on the way
+ # in.
monkeypatch.setattr(lifecycle, "PID_DISCOVERY_TIMEOUT_S", 0.0, raising = False)
# Stubbed for the same reason and, for every test but the two about it, so that whatever this
# machine happens to have on :5399 cannot decide the answer.
@@ -70,8 +70,7 @@ def launched(monkeypatch, tmp_path):
monkeypatch.setattr(lifecycle, "_find_unsloth_bin", lambda install: "/bin/true")
monkeypatch.setattr(lifecycle, "_read_bootstrap_password", lambda *a, **k: "secret")
monkeypatch.setattr(lifecycle, "wait_for_healthz", lambda *a, **k: state["healthy"])
- # Recorded rather than dropped: a launch refused before the spawn has to be shown not to have
- # spawned anything.
+ # Recorded rather than dropped: a launch refused before the spawn has to be shown not to have spawned anything.
monkeypatch.setattr(subprocess, "Popen", lambda *a, **k: state["spawned"].append(a))
def fake_run(cmd, *a, **k):
@@ -200,9 +199,8 @@ def test_the_probe_itself_gives_both_answers_against_a_real_socket():
port = listener.getsockname()[1]
assert lifecycle.port_is_busy(port) is True
- # Closed, so the same port is now the negative case. A port the kernel has just released can
- # linger in TIME_WAIT for a connect, which is why the assertion below is on a port that was
- # never bound at all rather than on this one.
+ # Closed, so the same port is now the negative case. A port the kernel has just released can linger
+ # in TIME_WAIT for a connect, which is why the assertion below is on a port never bound at all.
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
probe.bind(("127.0.0.1", 0))
free_port = probe.getsockname()[1]
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py
index 3b026602a98..21345ce890c 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py
@@ -48,8 +48,8 @@ def _git(
"GIT_COMMITTER_NAME": "studiobench",
"GIT_COMMITTER_EMAIL": "[email protected]",
# No user or system git config: an `init.defaultBranch`, a `commit.gpgsign` or a
- # `clone.defaultRemoteName` on the machine running the tests would otherwise decide
- # what this repository looks like.
+ # `clone.defaultRemoteName` on the machine running the tests would otherwise decide what this
+ # repository looks like.
"GIT_CONFIG_GLOBAL": os.devnull,
"GIT_CONFIG_SYSTEM": os.devnull,
"HOME": str(cwd),
@@ -70,8 +70,8 @@ def _origin(tmp_path: Path) -> tuple[Path, str, str]:
(origin / "install.sh").write_text("second\n")
_git("commit", "-am", "second", cwd = origin)
second = _git("rev-parse", "HEAD", cwd = origin).stdout.strip()
- # A bare mirror, because a fetch from a non-bare checkout of the same branch is a special case
- # and the real remote is a server.
+ # A bare mirror, because a fetch from a non-bare checkout of the same branch is a special case and
+ # the real remote is a server.
bare = tmp_path / "origin.git"
_git("clone", "--bare", str(origin), str(bare), cwd = tmp_path)
return bare, first, second
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py
index 64a4e5f4217..fc8f7b33f2b 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py
@@ -41,15 +41,12 @@ import pytest
REPO_ROOT = Path(__file__).resolve().parents[5]
-#: SIZED BY MEASUREMENT, not by guesswork. Under `spawn` the per-trial collision rate on the broken
-#: guard is well below the 50% a barrier reaches under `fork`, because each contender pays its own
-#: interpreter startup before arriving. At 10 trials the two headline tests missed the defect in one
-#: run out of four; at 40 they caught it in six runs out of six, and the whole file still costs
-#: about ten seconds. A concurrency test that only usually fails on the broken code is not much
-#: better than one that never does.
-#:
+#: SIZED BY MEASUREMENT. Under `spawn` the per-trial collision rate on the broken guard is well below
+#: the 50% a barrier reaches under `fork`, because each contender pays its own interpreter startup
+#: first. At 10 trials the two headline tests missed the defect in one run out of four; at 40 they
+#: caught it in six runs out of six, and the file still costs about ten seconds.
#: `spawn` rather than `fork` because pytest has already started threads by the time this runs, and
-#: forking a multi-threaded process is deprecated for the good reason that the child can deadlock.
+#: forking a multi-threaded process is deprecated because the child can deadlock.
TRIALS = 40
@@ -68,8 +65,8 @@ def _contend(repo_root: str, outdir: str, index: int, start, hold, q) -> None:
except Exception as exc: # noqa: BLE001 - reported rather than lost
q.put((False, f"UNEXPECTED {type(exc).__name__}: {exc}"))
finally:
- # Nobody releases the marker until everyone has attempted, so an admission is genuine
- # overlap rather than sequential reuse of a directory the first run already let go.
+ # Nobody releases the marker until everyone has attempted, so an admission is genuine overlap rather
+ # than sequential reuse of a directory the first run already let go.
try:
hold.wait(timeout = 60)
except Exception: # noqa: BLE001
@@ -197,14 +194,12 @@ def test_a_crashed_run_does_not_lock_the_directory_forever(tmp_path):
rec.close()
-# ── a retained record is not a holder ────────────────────────────────
-#
-# The marker is deliberately never unlinked, so on a directory that has been used before it already
-# contains the PREVIOUS run's `pid session` line. These two are deterministic on purpose: they stand
-# in the window rather than racing for it, so they say the same thing on a loaded runner as on an
-# idle one.
+# A retained record is not a holder. The marker is deliberately never unlinked, so a directory used
+# before already contains the PREVIOUS run's `pid session` line. These two are deterministic on
+# purpose: they stand in the window rather than racing for it.
+# ── a retained record is not a holder ────────────────────────────────
def _stalled_holder(
marker: Path,
write_after_s: float,
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py
index 796b234d4c0..f6f7f37e012 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py
@@ -455,15 +455,14 @@ def test_a_session_that_died_before_its_first_cell_declares_no_mode(tmp_path):
)
-# ── the two fixture axes ────────────────────────────────────────────────────────────────────
-#
-# `--stream-tail-chars` and `--corpus-dollars` change what the last turn STREAMS, and neither of
-# them moves `corpus_hash`, which covers the frozen units on disk and the generator's parameters.
-# Nor does either move a `cell_id`. They are on `IDENTITY_AXES` for the same reason the tier is:
-# without that, a resume under a changed fixture skips cells measured against a different film and
-# the payload silently becomes one ladder built from two.
+# The two fixture axes. `--stream-tail-chars` and `--corpus-dollars` change what the last turn
+# STREAMS, and neither moves `corpus_hash` (which covers the frozen units on disk and the
+# generator's parameters) or a `cell_id`. They are on `IDENTITY_AXES` for the same reason the
+# tier is: without that, a resume under a changed fixture skips cells measured against a
+# different film and the payload silently becomes one ladder built from two.
+# ── the two fixture axes ────────────────────────────────────────────────────────────────────
def _fixture_payload(tmp_path, name, session, **fixture):
paths = Paths.under(tmp_path / name)
_record(
@@ -710,21 +709,19 @@ def test_a_payload_from_before_the_fixture_axes_is_refused_under_a_non_default_f
assert ("stream_tail_chars" if "--stream-tail-chars" in flags else "corpus_dollars") in (
message
)
- # The refusal has to say WHY an axis the payload never mentions is a difference, or the
- # reader's next move is to go looking for a key that was never going to be there.
+ # The refusal has to say WHY an axis the payload never mentions is a difference, or the reader's
+ # next move is to go looking for a key that was never going to be there.
assert "predates this axis" in message
-# ── the measurement-mode axis ───────────────────────────────────────────────────────────────
-#
-# `--click-probe` runs a full `page.click`, a real mouse click, a dispatch, a focus and a hover
-# over the thread BEFORE the film starts, and the tool's own help text for the flag says it "makes
-# the cell's timings incomparable with a cell that did not run it". The cell then carries a
-# `composer_click_ms` measured on a composer all of those paths have already been through, and a
-# `click_attribution` block a cell without the flag does not have at all. None of that moves the
-# cell id, so it is on `IDENTITY_AXES` for the same reason the tier is.
+# The measurement-mode axis. `--click-probe` runs a full `page.click`, a real mouse click, a
+# dispatch, a focus and a hover over the thread BEFORE the film starts, and the flag's own help
+# text says it 'makes the cell's timings incomparable with a cell that did not run it'. The cell
+# then carries a `composer_click_ms` measured on a composer all those paths have been through,
+# and a `click_attribution` block a cell without the flag lacks. None of that moves the cell id.
+# ── the measurement-mode axis ───────────────────────────────────────────────────────────────
def test_resuming_after_toggling_the_click_probe_is_refused(tmp_path):
"""REGRESSION. Both directions, because either one produces the same mixed ladder.
@@ -935,8 +932,8 @@ def test_the_report_a_mode_change_would_have_produced(tmp_path):
changed_mode = score_payload(_payload("changed", "base").payload_jsonl, [10_000])
# Same mode: the retry reuses the cell id, so the crash is superseded and never reaches the
- # ladder. Change the mode and the same crash is a cell of its own, first at its rung, and it
- # is what the report prints -- over a rung the run being reported measured cleanly.
+ # ladder. Change the mode and the same crash is a cell of its own, first at its rung, and it is
+ # what the report prints, over a rung the run being reported measured cleanly.
assert "died at 10K" not in (same_mode.rungs[0].incomplete_reason or "")
assert "died at 10K" in (changed_mode.rungs[0].incomplete_reason or "")
@@ -1008,15 +1005,13 @@ def test_the_readme_sequence_no_longer_scores_two_runs_as_one_ladder(tmp_path):
assert payload["header"]["studio_ref"] == "main"
-# ── the injection axis ──────────────────────────────────────────────────────────────────────
-#
-# `--inject-stream-cost-ms` burns a known amount of main-thread time per SSE chunk on the TREATMENT
-# arm, so a treatment cell recorded with it is not the same reading as one recorded without it --
-# and `cell_id` carries the rung, the arm and the repetition and nothing that could tell. It is on
-# `IDENTITY_AXES` for the same reason the two fixture axes above are, and for a larger perturbation
-# than either of them makes.
+# The injection axis. `--inject-stream-cost-ms` burns a known amount of main-thread time per SSE
+# chunk on the TREATMENT arm, so a treatment cell recorded with it is not the same reading as one
+# recorded without, and `cell_id` carries the rung, the arm and the repetition and nothing that
+# could tell.
+# ── the injection axis ──────────────────────────────────────────────────────────────────────
def _injected_payload(tmp_path, name, session, **fixture):
paths = Paths.under(tmp_path / name)
_record(
@@ -1131,19 +1126,17 @@ def test_an_unchanged_injection_resumes(tmp_path):
assert _resume_set(paths) == {"r10K.base.rep0", "r10K.treatment.rep0"}
-# ── the external probe axis ─────────────────────────────────────────────────────────────────
-#
-# `SBENCH_EXTRA_INIT_SCRIPT` installs a caller's own init script into the page, and the run says so
-# in its log: "this run carries an external probe and is NOT a clean measurement of the build".
-# What it cannot end in is a wrong number, because `refuse_if_probed` reads EVERY `run_meta` in the
-# file and every scoring entry point calls it. It is on `IDENTITY_AXES` for what that refusal
-# costs: the refusal is whole-file and the payload is append-only, so a resume that toggles the
-# probe takes the cells recorded before it down with it, and nothing here can put them back.
-#
-# It is also the axis most easily toggled by accident. The other four are typed on the command
-# line every time; this one is an environment variable, and a variable that is still set is
-# indistinguishable from one that was never set until the run is over.
+# The external probe axis. `SBENCH_EXTRA_INIT_SCRIPT` installs a caller's own init script into the
+# page, and the run's log says 'this run carries an external probe and is NOT a clean measurement
+# of the build'. It cannot end in a wrong number, because `refuse_if_probed` reads EVERY
+# `run_meta` and every scoring entry point calls it; it is on `IDENTITY_AXES` for what that
+# refusal costs, since the refusal is whole-file and the payload append-only, so a resume that
+# toggles the probe takes the cells recorded before it down with it.
+# It is also the axis most easily toggled by accident: the other four are typed on the command
+# line every time, while a variable that is still set is indistinguishable from one that was
+# never set until the run is over.
+# ── the external probe axis ─────────────────────────────────────────────────────────────────
PROBE = "probes/paint_counter.js"
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py
index 46725965fff..d6040984da2 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py
@@ -119,14 +119,14 @@ class _Pacer:
class _Seeder:
- #: `None`, so `measure_chars_per_token` cannot reach for the network from a unit test.
+ #:`None`, so `measure_chars_per_token` cannot reach for the network from a unit test.
auth = None
def seed(self, plan):
- # No messages, so the mount wait is the selector and not a count that never arrives, and
- # both markers `SeededThread` declares are present and `None` for the same reason:
- # `_wait_for_thread` reads `last_marker` unconditionally, and a stub that omits it fails on
- # the attribute rather than on the failure these tests are about.
+ # No messages, so the mount wait is the selector and not a count that never arrives, and both
+ # markers `SeededThread` declares are present and `None`: `_wait_for_thread` reads `last_marker`
+ # unconditionally, and a stub that omits it fails on the attribute rather than on the failure
+ # these tests are about.
return types.SimpleNamespace(
thread_id = "t-1",
seconds = 0.0,
@@ -210,8 +210,8 @@ def test_a_cell_that_dies_after_the_probe_still_reports_the_attribution(tmp_path
assert row["completed"] is False
assert row["failure"]["kind"] == "TimeoutError"
- # Out of the payload, not out of the returned dict: the file is what a failed run leaves
- # behind, and it is where the diagnostic has to be readable from.
+ # Out of the payload, not the returned dict: the file is what a failed run leaves behind, and where
+ # the diagnostic has to be readable from.
cells = _cell_rows(paths)
assert len(cells) == 1
attribution = cells[0]["click_attribution"]
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py
index 42d446de724..17e89fc44d4 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py
@@ -82,8 +82,7 @@ TURNS = 9
MESSAGES = TURNS * 2 # 18, the number in the failure this work exists to fix
WINDOW = 6
#: The fixture's row height. The completeness tests step the traversal by two rows, because a
-#: gesture whose stops do not overlap can only report NOT MEASURED, and a test that wants to see
-#: coverage refuse something has to give the probe a sweep that actually covers the thread.
+#: gesture whose stops do not overlap can only report NOT MEASURED.
ROW_PX = 120
_DOM_JS = _STUDIO_TESTS / "studiobench" / "scene" / "dom.js"
@@ -215,10 +214,10 @@ def test_full_mount_is_admitted_in_full_mode(browser):
assert r.conditions["end_present"] is True
assert r.conditions["settled"] is True
assert r.probe["mounted"] == MESSAGES
- # AND THE ORDINAL CONDITIONS ARE NOT APPLICABLE HERE, which is not the same as passing.
- # Unsloth publishes no aria-posinset anywhere, so a `full` arm has none to validate and must
- # never be gated on them; `None` is the value the parity layer and this gate both use for a
- # surface that was not measured rather than one that agreed.
+ # AND THE ORDINAL CONDITIONS ARE NOT APPLICABLE HERE, which is not the same as passing. Unsloth
+ # publishes no aria-posinset anywhere, so a `full` arm has none to validate; `None` is what the
+ # parity layer and this gate both use for a surface that was not measured rather than one that
+ # agreed.
assert r.conditions["posinset_ordinals_valid"] is None
assert r.conditions["posinset_reaches_end"] is None
assert r.probe["posinset_count"] == 0
@@ -248,7 +247,7 @@ def test_a_virtualised_thread_is_admitted_in_windowed_mode(browser):
assert r.conditions["anchored_at_end"] is True
assert r.conditions["end_present"] is True
# The ordinals of a window at the END of an eighteen-message thread: 13..18, distinct, one per
- # mounted row. This is the shape the three malformed modes below fail to produce.
+ # mounted row. The shape the three malformed modes below fail to produce.
assert (r.probe["min_posinset"], r.probe["max_posinset"]) == (MESSAGES - WINDOW + 1, MESSAGES)
assert r.probe["posinset_distinct"] == WINDOW
@@ -310,16 +309,15 @@ def _completeness(browser, mode: str, **kwargs) -> tuple[dict, list[str]]:
def test_a_virtualised_thread_passes_the_completeness_probe(browser):
out, _ = _completeness(browser, "windowed")
assert out["head_reached"] is True, out
- # AND THE COVERAGE VERDICT IS NOT MEASURED AT THE DEFAULT STEP, which is the honest answer
- # rather than a flattering one. The gesture jumps 2,000px at a time and this fixture's whole
- # thread is 2,160px, so it lands at the bottom and then at the top and the rows between the
- # two stops were never in view. Nothing is known about them, so nothing is claimed.
+ # AND THE COVERAGE VERDICT IS NOT MEASURED AT THE DEFAULT STEP, which is the honest answer. The
+ # gesture jumps 2,000px and this thread is 2,160px, so it lands at the bottom and then the top
+ # and the rows between were never in view.
assert out["ordinal_coverage_complete"] is None, out
assert out["sweep_continuous"] is False
assert "never in view" in out["coverage_reason"]
- # WHICH KIND OF NOT MEASURED. The arm publishes ordinals, so the question applies and the
- # sweep failed to answer it, which `record_completeness_gate` refuses to score the cell on.
- # The remedy is the smaller step the next test uses, not a gate that accepts the unknown.
+ # WHICH KIND OF NOT MEASURED: the arm publishes ordinals, so the question applies and the sweep
+ # failed to answer it, which `record_completeness_gate` refuses to score the cell on. The remedy
+ # is the smaller step the next test uses.
assert out["ordinal_coverage_state"] == COVERAGE_UNMEASURED, out
@@ -448,10 +446,9 @@ def test_windowed_mode_also_admits_a_thread_short_enough_to_mount_whole(browser)
assert r.ready, r.reason
assert r.probe["setsize"] is None
assert r.conditions["total_declared"] is True
- # THE SAME WAIVER COVERS THE ORDINALS, and for the same reason: there are none to publish and
- # none to validate. It is waived on `mounted >= expected` AND on the absence of ordinals
- # together, so an arm that publishes malformed ones cannot buy its way out by also mounting
- # the whole thread.
+ # THE SAME WAIVER COVERS THE ORDINALS: there are none to publish and none to validate. Waived on
+ # `mounted >= expected` AND the absence of ordinals together, so an arm publishing malformed ones
+ # cannot buy its way out by also mounting the whole thread.
assert r.probe["posinset_count"] == 0
assert r.conditions["posinset_ordinals_valid"] is True
assert r.conditions["posinset_reaches_end"] is True
@@ -479,8 +476,8 @@ def test_a_half_mounted_thread_is_refused_in_full_mode(browser):
detail = caught.value.detail
assert detail["ready"] is False
assert detail["conditions"]["all_messages_mounted"] is False
- # And the two NEW conditions caught it too, independently of the count. That matters: it is
- # what makes the windowed mode safe, because the windowed mode has no count to rely on.
+ # And the two NEW conditions caught it too, independently of the count, which is what makes the
+ # windowed mode safe, since it has no count to rely on.
assert detail["conditions"]["end_present"] is False
assert detail["probe"]["mounted"] < MESSAGES
@@ -508,8 +505,8 @@ def test_a_half_mounted_thread_is_refused_in_windowed_mode_too(browser):
finally:
page.close()
conditions = caught.value.detail["conditions"]
- # THREE independent refusals, not one. It is still growing, it has not reached the end of the
- # thread, and it publishes no total.
+ # THREE independent refusals: it is still growing, it has not reached the end of the thread, and
+ # it publishes no total.
assert conditions["settled"] is False
assert conditions["end_present"] is False
assert conditions["total_declared"] is False
@@ -631,8 +628,8 @@ def test_a_bottom_window_numbered_from_one_is_refused(browser):
assert conditions["posinset_ordinals_valid"] is True
assert detail["probe"]["max_posinset"] == WINDOW
assert conditions["posinset_reaches_end"] is False
- # The TEXT of the last message is mounted; only the numbering disagrees with it. That split is
- # the point: `end_present` reads the thread and this reads what the arm says about it.
+ # The TEXT of the last message is mounted; only the numbering disagrees. That split is the point:
+ # `end_present` reads the thread and this reads what the arm says about it.
assert conditions["end_present"] is True
assert conditions["anchored_at_end"] is True
@@ -738,8 +735,8 @@ def test_evaluate_refuses_ordinals_that_are_not_positions():
conditions = evaluate(from_one, from_one, 18, MODE_WINDOWED)
assert conditions["posinset_ordinals_valid"] is True
assert conditions["posinset_reaches_end"] is False
- # PAST THE END OF THE SET THE SAME ROWS DECLARE. 19 of 18 is not a position either, and it is
- # what an off-by-one in a virtualizer's index-to-ordinal arithmetic produces.
+ # PAST THE END OF THE SET THE SAME ROWS DECLARE. 19 of 18 is not a position either, and it is what
+ # an off-by-one in index-to-ordinal arithmetic produces.
over = _windowed_probe(max_posinset = 19)
assert evaluate(over, over, 18, MODE_WINDOWED)["posinset_ordinals_valid"] is False
# And every one of them is NOT APPLICABLE in full mode, where nothing publishes ordinals.
@@ -796,9 +793,8 @@ def test_ordinal_coverage_never_reports_a_gap_in_the_gesture_as_data_loss():
}
got_coarse = ordinal_coverage(coarse, 18)
assert got_coarse["ordinal_coverage_complete"] is None
- # And it is the kind of None that is NOT a pass: the ordinals apply to this arm and the sweep
- # did not inspect them. `not_applicable` -- the other None -- is an arm publishing no ordinals
- # at all, which is the case below in `test_ordinal_coverage_separates_...`.
+ # And it is the kind of None that is NOT a pass: the ordinals apply to this arm and the sweep did
+ # not inspect them. `not_applicable` is an arm publishing no ordinals at all.
assert got_coarse["ordinal_coverage_state"] == COVERAGE_UNMEASURED
continuous = dict(coarse, sweep_continuous = True)
got = ordinal_coverage(continuous, 18)
@@ -923,6 +919,6 @@ def test_a_windowed_thread_with_no_viewport_is_refused(browser):
finally:
page.close()
- # Named in the refusal, so the reader is told the surface is missing rather than left to infer
- # it from a condition that is really about something else.
+ # Named in the refusal, so the reader is told the surface is missing rather than left to infer it
+ # from a condition about something else.
assert "viewport_present" in str(caught.value), str(caught.value)
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py
index 7320f026147..66fa5c795a0 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py
@@ -54,9 +54,9 @@ class _Verdict:
class _Bundle:
- # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta`
- # records the engine and `requested_identity` resolves the same way, so a stub that names a
- # fixed one would make a legitimate resume look like an engine change off Linux and macOS.
+ # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta` records the
+ # engine and `requested_identity` resolves the same way, so a stub naming a fixed one would make a
+ # legitimate resume look like an engine change off Linux and macOS.
engine = browser_mod.default_engine()[0]
engine_note = "stubbed for this test"
browser = context = page = cdp = None
@@ -100,8 +100,8 @@ def studio(monkeypatch, tmp_path):
def fake_install(ref, home, *args, **kwargs):
install = StudioInstall(home = Path(home), repo = Path(home).parent / "repo", branch = ref)
# `setattr` rather than a constructor argument, so this fixture also builds against a
- # `StudioInstall` that has no commit field and the tests below fail on the subject rather
- # than on the way in.
+ # `StudioInstall` that has no commit field and the tests fail on the subject rather than the way
+ # in.
install.commit = state["commits"].get(ref, f"c-{ref}-1")
return install
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py
index a66e267ead4..1f8aae053cd 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py
@@ -57,9 +57,9 @@ class _Verdict:
class _Bundle:
- # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta`
- # records the engine and `requested_identity` resolves the same way, so a stub that names a
- # fixed one would make a legitimate resume look like an engine change off Linux and macOS.
+ # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta` records the
+ # engine and `requested_identity` resolves the same way, so a stub naming a fixed one would make a
+ # legitimate resume look like an engine change off Linux and macOS.
engine = browser_mod.default_engine()[0]
engine_note = "stubbed for this test"
browser = context = page = cdp = None
@@ -433,15 +433,14 @@ def test_a_plain_run_and_a_plain_resume_are_unaffected(studio):
)
-# ── the external probe on the record ────────────────────────────────────────────────────────
-#
-# `SBENCH_EXTRA_INIT_SCRIPT` is an ENVIRONMENT variable, not a flag, so it outlives the command
-# that wanted it. A resume under a shell that still has it set runs the rungs still owed with the
-# probe in the page and appends a probed `run_meta`, and `refuse_if_probed` reads every `run_meta`
-# in a file: the cells recorded cleanly before it stop being scorable too, permanently, because a
-# payload is append-only. So `run_meta` has to carry the probe and `--resume` has to compare it.
+# The external probe on the record. `SBENCH_EXTRA_INIT_SCRIPT` is an ENVIRONMENT variable, so it
+# outlives the command that wanted it: a resume under a shell that still has it set runs the rungs
+# still owed with the probe in the page and appends a probed `run_meta`, and `refuse_if_probed`
+# reads every `run_meta` in a file, so cells recorded cleanly before it stop being scorable too,
+# permanently. So `run_meta` has to carry the probe and `--resume` has to compare it.
+# ── the external probe on the record ────────────────────────────────────────────────────────
class _ProbedBundle(_Bundle):
"""A probe run attaches console and `pageerror` listeners, which a `None` page cannot take."""
@@ -645,8 +644,8 @@ def test_a_fresh_probe_run_replaces_the_summary_it_inherited(studio, monkeypatch
assert "NO SUMMARY" in text
assert script.name in text
assert "studiobench summary" not in text
- # The evidence itself is untouched: the refusal replaces the report, never the payload the
- # summary described, which is still on disk under its archived name.
+ # The evidence itself is untouched: the refusal replaces the report, never the payload the summary
+ # described, which is still on disk under its archived name.
assert len(list(Paths.under(studio["out"]).out.glob("payload-*.jsonl"))) == 1
@@ -843,11 +842,11 @@ def test_an_attached_null_control_registers_one_provider_for_the_one_studio(stud
args = _args(studio, "--attach", url, "--attach-b", url, "--branch", "main", "--ab", "main")
assert sb.run(args, ab_ref = "main") == 0
- # THE SYMPTOM FIRST. Whatever the bookkeeping below says, the failure a cell actually meets is
- # a seed script that SELECTS a provider the backend no longer has, so that is what this pins:
- # every id named by a script scoped to this origin is still live there, and no live id is
- # unnamed. Asserting only "nothing was deleted" would pass a future fix that deleted and then
- # re-seeded both scripts with the survivor, and would fail a fix that never selects at all.
+ # THE SYMPTOM FIRST. Whatever the bookkeeping says, the failure a cell meets is a seed script that
+ # SELECTS a provider the backend no longer has, so that is what this pins: every id named by a
+ # script scoped to this origin is still live there, and no live id is unnamed. Asserting only
+ # 'nothing was deleted' would pass a fix that deleted and re-seeded both scripts with the
+ # survivor.
selected = _selected_provider_ids(scripts, url)
assert selected == set(backend.live[url])
assert selected, "the one Unsloth must still have a provider selected"
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py
index e8eaa73cf67..e0717c42f5f 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py
@@ -138,9 +138,9 @@ class _Pacer:
return list(self.streams)
-#: (action name, timing key, value). The three actions `scoring.from_payload.ACTION_SOURCES`
-#: reads, so a cell that completes carries enough weight to be scored rather than failing the
-#: coverage floor for an unrelated reason.
+#: (action name, timing key, value). The three actions `scoring.from_payload.ACTION_SOURCES` reads,
+#: so a cell that completes carries enough weight to be scored rather than failing the coverage
+#: floor for an unrelated reason.
SCENE_ACTIONS = (
("keystroke", "p95_ms", 40.0),
("message_menu", "open_ms", 90.0),
@@ -279,8 +279,8 @@ def cell_runner(monkeypatch, tmp_path):
def _cell():
- # NOT the 10K rung: that one additionally runs the seeded-vs-streamed equivalence check, which
- # is a different subject.
+ # NOT the 10K rung: that one additionally runs the seeded-vs-streamed equivalence check, which is a
+ # different subject.
return Cell(cell_id = "r1K.A0.rep0", rung = "1K", rung_tokens = 1_000, tier = "quick")
@@ -331,8 +331,7 @@ def test_a_cell_whose_reply_never_finished_does_not_complete(cell_runner):
assert row["completed"] is False
assert row["failure"]["kind"] == "RuntimeError"
assert "never finished" in row["failure"]["message"]
- # The evidence ships in the same row as the failure: how long was waited, and how much the
- # pacer actually delivered.
+ # The evidence ships in the same row as the failure: how long was waited, and how much the pacer actually delivered.
assert row["stream"]["finished"] is False
assert row["pacer"]["last"]["chunks"] == 150
assert row["pacer"]["streams"][0]["tag"] == "r1K.A0.rep0"
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py
index 0945f4b0418..0a329263885 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py
@@ -257,7 +257,6 @@ def test_a_cell_with_nothing_planned_is_not_checked():
# ── level 2: what the cell does with it ──────────────────────────────────────────────────────
-
CENSUS = {"messages": 6, "elements": 1200, "highlight_spans": 200, "assistant_chars": 1120}
@@ -394,9 +393,9 @@ def cell_runner(monkeypatch, tmp_path):
thread_id = "t1",
seconds = 0.5,
messages = 0,
- # Both markers `SeededThread` declares, present and None: the readiness gate
- # reads `last_marker` unconditionally, so a stub that omits it fails on the
- # attribute rather than on the stream accounting these tests are about.
+ # Both markers `SeededThread` declares, present and None: the readiness gate reads `last_marker`
+ # unconditionally, so a stub that omits it fails on the attribute rather than on the stream
+ # accounting.
first_marker = None,
last_marker = None,
),
@@ -495,7 +494,7 @@ def _scene_with(rows: list[dict]):
return _Fixed
-#: A `send_turn` that RAN, loaded the pacer and pressed Enter, and whose own assertion failed.
+#:A `send_turn` that RAN, loaded the pacer and pressed Enter, and whose own assertion failed.
SEND_TURN_THAT_FAILED = {
"action": "send_turn",
"ran": True,
@@ -688,10 +687,9 @@ class _WirePage:
if "messageCount" in expr:
return self.messages
# THE THREAD'S LENGTH AS WELL AS THE MOUNTED COUNT. `send_turn` proves a send worked by
- # `threadTotal()` growing, not `messageCount()`, so that a windowed arm whose window slides
- # is not read as a send that did nothing. This page models a fully mounted arm, where the
- # two are the same number; without the second name the shipped action sees 0 both sides and
- # every follow-up reports that it never started a reply.
+ # `threadTotal()` growing, not `messageCount()`, so a windowed arm whose window slides is not read
+ # as a send that did nothing. This page models a fully mounted arm; without the second name the
+ # shipped action sees 0 both sides and every follow-up reports that it never started a reply.
if "threadTotal" in expr:
return self.messages
if "assistantChars" in expr:
@@ -776,9 +774,9 @@ def test_a_healthy_multi_turn_cell_passes_the_check_over_real_wire_bytes(monkeyp
thread_id = "t1",
seconds = 0.5,
messages = 0,
- # Both markers `SeededThread` declares, present and None: the readiness gate
- # reads `last_marker` unconditionally, so a stub that omits it fails on the
- # attribute rather than on the stream accounting these tests are about.
+ # Both markers `SeededThread` declares, present and None: the readiness gate reads `last_marker`
+ # unconditionally, so a stub that omits it fails on the attribute rather than on the stream
+ # accounting.
first_marker = None,
last_marker = None,
),
diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py
index 37f3c6d611d..2e87f2a8649 100644
--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py
+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py
@@ -45,9 +45,9 @@ from studiobench.runtime.seeder import Seeder # noqa: E402
#: How long the fake Unsloth's access tokens live. An hour compressed into something a test can sit
#: through, and long enough that a loaded machine cannot expire one mid-request.
TOKEN_TTL_S = 6.0
-#: The margin the tests below drive `token()` with. The REAL ratio matters: a margin far shorter
-#: than the token's life is what the harness runs with (15 minutes against 60), and a margin longer
-#: than the whole life would make every call rotate and prove nothing.
+#: The margin the tests below drive `token()` with. The REAL ratio matters: a margin far shorter than
+#: the token's life is what the harness runs with (15 minutes against 60), and a margin longer than
+#: the whole life would make every call rotate and prove nothing.
TEST_MARGIN_S = 1.0
PASSWORD = "studiobench-bench-password"
@@ -62,8 +62,8 @@ class _State:
self.logins = 0
self.login_attempts = 0
self.rejections = 0
- #: Set to reject EVERY token once, whatever its `exp` says: an Unsloth restarted underneath
- #: the run, or a clock this process cannot see.
+ #: Set to reject EVERY token once, whatever its `exp` says: an Unsloth restarted underneath the run,
+ #: or a clock this process cannot see.
self.reject_next = 0
self.lock = threading.Lock()
@@ -194,8 +194,8 @@ def test_the_seeder_keeps_working_after_its_token_expires(studio, monkeypatch):
logins_after_setup = state.logins
time.sleep(TOKEN_TTL_S + 0.5)
- # The token the run was handed is dead by now -- the control above proves the server refuses
- # it -- and this still has to work.
+ # The token the run was handed is dead by now (the control above proves the server refuses it) and
+ # this still has to work.
assert seeder.create_thread()
assert auth.rotations == 1
assert state.logins == logins_after_setup + 1
@@ -215,8 +215,7 @@ def test_the_token_is_replaced_before_it_expires_not_after_it_fails(studio, monk
monkeypatch.setattr(lifecycle, "TOKEN_REFRESH_MARGIN_S", TEST_MARGIN_S)
base_url, state = studio
auth = authenticate(base_url, "bench", PASSWORD, new_password = PASSWORD)
- # Inside the margin and NOT yet expired: the server would still accept the old token, and it
- # is replaced anyway.
+ # Inside the margin and NOT yet expired: the server would still accept the old token, and it is replaced anyway.
time.sleep(TOKEN_TTL_S - TEST_MARGIN_S / 2)
assert auth_request_json(auth, f"{base_url}/api/chat/threads", method = "POST", body = {})
diff --git a/tests/studio/studiobench/runtime/selftest/thread_fixture.js b/tests/studio/studiobench/runtime/selftest/thread_fixture.js
index 40b5bddfe7c..70876b0a2de 100644
--- a/tests/studio/studiobench/runtime/selftest/thread_fixture.js
+++ b/tests/studio/studiobench/runtime/selftest/thread_fixture.js
@@ -1,57 +1,49 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
// A SYNTHETIC THREAD, with a switch on how it mounts.
-//
// This exists so the readiness gate can be shown PASSING on a correctly virtualised thread and
-// FAILING on a broken one, in a real browser, against the real `scene/dom.js` adapter and the real
-// `runtime/readiness.py` probe. A gate that has only ever been shown passing is not a gate, and
-// neither is one that has only ever been shown against a mock of itself.
-//
-// It reproduces exactly the contract the shipping app publishes and the readiness gate reads --
-// `.aui-thread-root`, `.aui-thread-viewport`, `[data-role]`, `.aui-thread-scroll-to-bottom` with
-// its `invisible` toggle, the composer textarea -- and nothing else. It is not a replica of Unsloth
-// and must never be mistaken for one: it cannot tell you anything about the app's own timing. What
-// it can do is put the gate in front of each of the situations it has to tell apart.
+// FAILING on a broken one, in a real browser, against the real `scene/dom.js` adapter and the
+// real `runtime/readiness.py` probe.
+// It reproduces exactly the contract the shipping app publishes and the gate reads
+// (`.aui-thread-root`, `.aui-thread-viewport`, `[data-role]`, `.aui-thread-scroll-to-bottom`
+// with its `invisible` toggle, the composer textarea) and nothing else. It is not a replica of
+// Unsloth and can say nothing about the app's own timing.
(() => {
const MODES = [
// Everything mounted, the way the app ships today.
"full",
- // Mounting is still in progress: the first K messages, growing. This is the state the gate
- // exists to refuse, and the state the old count-based gate DID correctly refuse.
+ // Mounting still in progress: the first K messages, growing. The state the gate exists to
+ // refuse, and the one the old count-based gate did correctly refuse.
"mounting",
// A correct window: the tail of the thread, aria-setsize/aria-posinset published, anchored at
- // the bottom, and the head materialises when you scroll to the top.
+ // the bottom, head materialising when you scroll to the top.
"windowed",
// A window with no aria-setsize. Nothing outside the app can learn how long this thread is.
"windowed_no_total",
- // A window over the HEAD of the thread rather than its end: settled, correct total, but the
- // last message is nowhere.
+ // A window over the HEAD of the thread rather than its end: settled, correct total, last message nowhere.
"windowed_at_top",
- // A window at the end, correct total published, and a store that only holds what is mounted.
- // Indistinguishable from `windowed` while you stand at the bottom of the thread, which is why
- // the completeness probe has to walk to the top.
+ // A window at the end with a correct total published and a store that only holds what is
+ // mounted. Indistinguishable from `windowed` while you stand at the bottom, which is why the
+ // completeness probe walks to the top.
"windowed_lost_head",
// A store that kept the first page AND the last page and lost the middle. The head marker
- // arrives when you scroll to the top, so the marker check alone calls it complete; the
- // ordinals of one mounted window run 1,2,3 then 16,17,18, which is the loss.
+ // arrives when you scroll to the top, so the marker check alone calls it complete; the ordinals
+ // run 1,2,3 then 16,17,18, which is the loss.
"windowed_lost_middle",
- // The same correct window, but with the ordinals written directly on the [data-role] element
- // instead of on a row wrapper. Both placements must be accepted.
+ // The same correct window with the ordinals written directly on the [data-role] element instead
+ // of a row wrapper. Both placements must be accepted.
"windowed_flat",
- // THE THREE MALFORMED ORDINAL CONTRACTS. Each publishes aria-posinset on every mounted row --
- // enough for a gate that only counts attributes -- and each publishes numbers that are not
- // positions in the thread, so nothing outside the app can locate the mounted window.
- //
- // Every row says 0. The attribute is there and it is out of range: aria-posinset is 1-based.
+ // THE THREE MALFORMED ORDINAL CONTRACTS. Each publishes aria-posinset on every mounted row,
+ // enough for a gate that only counts attributes, and each publishes numbers that are not
+ // positions, so nothing outside the app can locate the mounted window.
+ // Every row says 0. The attribute is there and out of range: aria-posinset is 1-based.
"windowed_zero_ordinals",
// Every row says the same number, so six mounted rows claim one position between them.
"windowed_duplicate_ordinals",
// A window at the BOTTOM of an 18-message thread numbered 1..6: the index within the window
- // published as though it were the position in the thread. The most likely of the three to be
- // written by accident, and the one that makes a window at the end look like a window at the
- // start.
+ // published as the position in the thread. The likeliest of the three to be written by
+ // accident, and the one that makes a window at the end look like a window at the start.
"windowed_from_one",
];
@@ -96,8 +88,8 @@
root.appendChild(composer);
document.body.appendChild(root);
- // Every message the "store" holds. `windowed_lost_head` throws most of them away, which is
- // the data loss the completeness probe is looking for.
+ // Every message the 'store' holds. `windowed_lost_head` throws most of them away, which is the
+ // data loss the completeness probe looks for.
const ROW_PX = 120;
let store = [];
for (let i = 0; i < turns; i += 1) {
@@ -106,9 +98,9 @@
}
const declaredTotal = total;
if (mode === "windowed_lost_head") store = store.slice(-windowSize);
- // THE HEAD AND THE TAIL, AND NOTHING BETWEEN THEM. Half a window at each end, so the store
- // is exactly one window long and every scroll position mounts all of it: the first message
- // is always there for the marker check, and the hole is always there for the ordinals.
+ // THE HEAD AND THE TAIL, AND NOTHING BETWEEN THEM. Half a window at each end, so the store is
+ // exactly one window long and every scroll position mounts all of it: the first message is
+ // always there for the marker check, and the hole is always there for the ordinals.
if (mode === "windowed_lost_middle") {
const keep = Math.max(1, Math.floor(windowSize / 2));
store = store.slice(0, keep).concat(store.slice(-keep));
@@ -116,11 +108,9 @@
const state = { mode, store, declaredTotal, windowSize, ROW_PX, total };
- // WHAT EACH ROW PUBLISHES AS ITS POSITION. Everything but the three malformed modes
- // publishes the message's real position in the thread, which is what aria-posinset means.
- // The malformed ones publish a number that is not one, each in a different way, so the gate
- // can be shown refusing each of them separately rather than refusing "something about the
- // ordinals".
+ // WHAT EACH ROW PUBLISHES AS ITS POSITION. Everything but the three malformed modes publishes
+ // the message's real position, which is what aria-posinset means; the malformed ones each
+ // publish a different sort of non-position, so the gate can be shown refusing each separately.
function publishedPos(message, indexInWindow) {
if (mode === "windowed_zero_ordinals") return 0;
if (mode === "windowed_duplicate_ordinals") return state.declaredTotal;
@@ -138,11 +128,10 @@
el.setAttribute("data-role", m.role);
el.style.cssText = "height:" + ROW_PX + "px;overflow:hidden;";
el.textContent = m.text;
- // WHERE THE SHIPPING VIRTUALIZER PUTS THE ORDINALS: on the positioned row wrapper, not
- // on the message. `thread-message-virtualizer.tsx` renders an absolutely positioned div
- // per item and mounts `ThreadPrimitive.MessageByIndex` inside it, so the element that is
- // a member of the set is the wrapper. `wrapped` reproduces that shape exactly; the
- // unwrapped placement is kept too, because the gate must accept both.
+ // WHERE THE SHIPPING VIRTUALIZER PUTS THE ORDINALS: on the positioned row wrapper, not the
+ // message. `thread-message-virtualizer.tsx` renders an absolutely positioned div per item with
+ // `ThreadPrimitive.MessageByIndex` inside, so the wrapper is the member of the set. `wrapped`
+ // reproduces that; the unwrapped placement is kept because the gate must accept both.
if (publishTotals && mode !== "windowed_flat") {
const row = document.createElement("div");
row.setAttribute("aria-setsize", String(state.declaredTotal));
@@ -157,9 +146,9 @@
}
list.appendChild(el);
}
- // Spacers, so the scroll extent describes the WHOLE thread even though only a window is
- // mounted. A virtualizer that omits these has a scrollbar that lies, which is one of the
- // behavioural invariants in analysis/behaviour.py.
+ // Spacers, so the scroll extent describes the WHOLE thread even though only a window is mounted.
+ // A virtualizer that omits these has a scrollbar that lies, one of the behavioural invariants in
+ // analysis/behaviour.py.
const above = mode === "full" || mode === "mounting" ? 0 : startIndex * ROW_PX;
const below =
mode === "full" || mode === "mounting"
@@ -175,10 +164,10 @@
}
state.renderWindowAround = (scrollTop) => {
- // A store SHORTER than the thread it claims cannot be indexed by the thread's own
- // indices: `windowed_lost_middle` holds one window's worth of messages for an
- // eighteen-message thread, so every scroll position mounts the same rows and the ordinals
- // on them are the only thing that says which messages they are.
+ // A store SHORTER than the thread it claims cannot be indexed by the thread's own indices:
+ // `windowed_lost_middle` holds one window's worth for an eighteen-message thread, so every
+ // scroll position mounts the same rows and the ordinals are the only thing saying which
+ // messages they are.
if (mode === "windowed_lost_middle") {
const start = Math.max(
0,
@@ -191,8 +180,8 @@
0,
Math.min(state.declaredTotal - windowSize, Math.floor(scrollTop / ROW_PX)),
);
- // `windowed_lost_head` only has the tail in its store, so an index into the full thread
- // has to be translated. It will simply have nothing to show near the top.
+ // `windowed_lost_head` only has the tail in its store, so an index into the full thread has to
+ // be translated; it will simply have nothing to show near the top.
const offset = mode === "windowed_lost_head" ? state.declaredTotal - state.store.length : 0;
render(Math.max(0, first - offset), windowSize, true);
};
@@ -201,9 +190,9 @@
render(0, state.store.length, false);
pin();
} else if (mode === "mounting") {
- // GROWING. One message every 250ms, from the head, and it never finishes inside the
- // window any test here gives it. The count climbs, the element count climbs, the scroll
- // height climbs, and the last message is never reached.
+ // GROWING: one message every 250ms from the head, never finishing inside the window any test
+ // gives it. The count, element count and scroll height climb, and the last message is never
+ // reached.
let n = 1;
render(0, n, false);
pin();
@@ -222,8 +211,8 @@
pin();
} else {
// windowed, windowed_flat, windowed_lost_head, windowed_lost_middle and the three
- // malformed-ordinal modes: all of them are a window at the END of the thread, and they
- // differ only in what they publish about it.
+ // malformed-ordinal modes are all a window at the END of the thread, differing only in what
+ // they publish about it.
render(Math.max(0, state.store.length - windowSize), windowSize, true);
pin();
viewport.addEventListener("scroll", () => {
@@ -234,13 +223,11 @@
);
});
}
- // COPY-FROM-STORE, the shipping fix, reproduced on the same contract.
- //
- // Mirrors `decideThreadCopy` in
- // studio/frontend/src/components/assistant-ui/thread-copy-from-store.ts: intervene only when
- // the selection spans the whole mounted list AND the store holds more than is mounted. The
- // unit tests over there stub `containsNode`; this is the only place the REAL Selection
- // semantics are exercised, which is the part that could actually be wrong.
+ // COPY-FROM-STORE, the shipping fix, reproduced on the same contract. Mirrors `decideThreadCopy`
+ // in thread-copy-from-store.ts: intervene only when the selection spans the whole mounted list
+ // AND the store holds more than is mounted. The unit tests there stub `containsNode`; this is
+ // the only place the REAL Selection semantics are exercised.
+ // Full path: studio/frontend/src/components/assistant-ui/thread-copy-from-store.ts.
if (opts.copyFromStore) {
viewport.addEventListener("copy", (event) => {
const selection = window.getSelection();
@@ -261,8 +248,7 @@
},
};
- // The two-rAF paint promise the real harness installs from instruments/frames.js. The
- // completeness probe awaits it.
+ // The two-rAF paint promise the real harness installs from instruments/frames.js. The completeness probe awaits it.
if (!window.__sbNextPaint) {
window.__sbNextPaint = () =>
new Promise((resolve) =>
diff --git a/tests/studio/studiobench/runtime/session.py b/tests/studio/studiobench/runtime/session.py
index fca5f7c84a9..a5e1062066c 100644
--- a/tests/studio/studiobench/runtime/session.py
+++ b/tests/studio/studiobench/runtime/session.py
@@ -53,10 +53,9 @@ from .readiness import (
from .seeder import Seeder, SeededThread, compare_signatures, dom_signature, measure_chars_per_token
from .types import BenchContext, Cell, Paths, Recorder, Window, make_cell_id, new_session_id
-# A 1x1 PNG, written to disk once per run so the image-upload action has a real file to attach.
-# Generated rather than shipped as a binary asset: the artifact is a zipapp and a tester's
-# machine has no fixture directory, and an action that reports `ran = False, no image path` at
-# every rung is a hole in the suite that looks like a decision.
+# A 1x1 PNG, written once per run so image-upload has a real file. Generated rather than shipped:
+# the artifact is a zipapp with no fixture directory, and an action reporting `ran = False` at
+# every rung is a hole that looks like a decision.
_PNG_1X1 = bytes.fromhex(
"89504e470d0a1a0a0000000d494844520000000100000001080600000"
"01f15c4890000000d49444154789c6360000002000100ffff03000006"
@@ -73,29 +72,21 @@ def ensure_probe_image(paths: Paths) -> Path:
IDLE_CALIBRATION_MS = 1500
# The rung the seeded-vs-streamed equivalence is CHECKED at. 10K, because both paths are
-# affordable there: streaming it takes under a minute at field cadence, and seeding it is instant.
-# Below it the thread is one turn and there is nothing to seed; above it, streaming the whole
-# thread is hours.
+# affordable there; below it there is nothing to seed, above it streaming the thread is hours.
EQUIVALENCE_RUNG = "10K"
MOUNT_TIMEOUT_S = 180
#: How much of the streaming phase the thread must stay pinned for `follows_the_stream` to pass.
-#: 0.95 rather than 1.0: the sampler ticks four times a second and a legitimate pin lands a tick
-#: or two late after a large append, which is a rendering delay and not a failure to follow. It is
-#: paired with `ever_fell_behind`, which is absolute -- a thread that drifted past the tolerance
-#: even once fails, however quickly it was yanked back, because being yanked back is itself the
-#: other half of the intent contract being broken.
+#: 0.95 rather than 1.0, since the sampler ticks four times a second and a legitimate pin can land
+#: a tick late. Paired with `ever_fell_behind`, which is absolute: being yanked back is the other
+#: half of the intent contract being broken.
FOLLOW_PINNED_MIN = 0.95
-#: How much of the STREAMING TIME the attached phases must cover before `pinned_fraction` is
-#: allowed to stand as a verdict.
-#:
-#: `pinned_fraction` is computed over attached samples only. With `detached` latching on the first
-#: deliberate scroll and never clearing, the shipped film -- which scrolls 1.5s into an 18s opening
-#: stream and then streams twice more -- produced a verdict from the first ~3s: 11 attached samples
-#: against 72 detached, 13% coverage, reported as 100% pinned and read as "the thread follows the
-#: stream". The latch is fixed, and this makes the coverage a condition rather than a footnote, so
-#: a future regression that strands the sampler cannot produce a confident pass over a sliver.
+#: How much of the STREAMING TIME the attached phases must cover before `pinned_fraction` may
+#: stand as a verdict. It is computed over attached samples only, and with `detached` latching on
+#: the first deliberate scroll the shipped film produced a verdict from the first ~3s -- 13%
+#: coverage, reported as 100% pinned. The latch is fixed; this makes the coverage a condition.
+# That scroll is 1.5s into an 18s opening stream.
FOLLOW_MIN_STREAM_COVERAGE = 0.50
@@ -145,30 +136,19 @@ def follow_verdict(follow: Mapping[str, Any]) -> tuple[bool, dict[str, Any]]:
pinned_ok = pinned is not None and pinned >= FOLLOW_PINNED_MIN
fell_behind = bool(follow.get("ever_fell_behind"))
coverage_short = coverage is None or coverage < FOLLOW_MIN_STREAM_COVERAGE
- # THE DETACHMENT HAS TO BE THE SCHEDULE'S, AND ONLY A RE-ATTACHMENT PROVES THAT IT WAS.
- #
- # The waiver's whole justification is that `attached_fraction_of_stream` is set by the film and
- # is therefore identical on both arms. Half of it is: the harness scrolls away on a fixed
- # schedule. The other half is the BUILD'S -- `scene/dom.js` clears `detached` only when a run
- # that began after the gesture is observed at the bottom, so an arm that stops re-pinning when
- # the user submits a new turn never re-attaches and every remaining sample lands in the
- # detached branch. That branch `return`s before the pinned and fell-behind accounting, so the
- # opening stream's `pinned_fraction: 1.0` and `ever_fell_behind: False` survive untouched while
- # coverage collapses -- exactly the shape this conjunction was about to waive, on the one
- # failure the gate exists to catch. A reply that leaves the viewport is unmounted and stops
- # costing anything to paint, so the cell is CHEAPER for the defect, and `readings_by_arm` would
- # have admitted it to the ratios against a healthy partner.
- #
- # `reattachments` is the sampler's own record of the app coming back, and it is >= 1 on every
- # cell the carve-out was derived from: the measured 0.481 coverage is unreachable without one,
- # since `detached` latches 1.5s into an 18s stream and only a re-attachment resumes counting
- # attached samples. Absent (an old payload, or a sampler that never ran) is not evidence of a
- # re-attachment, and the absent-instrument case is already waived by `follow_attempted`.
+ # THE DETACHMENT HAS TO BE THE SCHEDULE'S, AND ONLY A RE-ATTACHMENT PROVES IT. The waiver rests
+ # on `attached_fraction_of_stream` being set by the film and identical on both arms, but half of
+ # it is the BUILD'S: `scene/dom.js` clears `detached` only when a run begun after the gesture is
+ # observed at the bottom, so an arm that stops re-pinning never re-attaches and every sample
+ # lands in the detached branch, leaving `pinned_fraction: 1.0` intact while coverage collapses.
+ # A reply that leaves the viewport stops costing anything to paint, so the cell is CHEAPER for
+ # the defect and `readings_by_arm` would have admitted it against a healthy partner.
+ # `reattachments` is the sampler's own record of the app coming back.
reattached = bool(follow.get("reattachments"))
- # THE COVERAGE AS A NUMBER, WHATEVER THE VERDICT, next to the floor it is read against. It was
- # only ever visible as a pass or a fail, and that is why it took a campaign to notice the floor
- # sat above the film's ceiling: every reader saw "FAILED its stream-follow gate", nobody saw
- # 0.481. A quantity that decides whether a run is quotable has to be legible as a quantity.
+ # THE COVERAGE AS A NUMBER, WHATEVER THE VERDICT, next to the floor it is read against: it was
+ # only ever visible as a pass or a fail, which is why it took a campaign to notice the floor sat
+ # above the film's ceiling.
+ # Every reader saw "FAILED its stream-follow gate"; nobody saw the 0.481.
recorded: dict[str, Any] = {
"stream_coverage": coverage,
"stream_coverage_floor": FOLLOW_MIN_STREAM_COVERAGE,
@@ -187,12 +167,10 @@ def follow_verdict(follow: Mapping[str, Any]) -> tuple[bool, dict[str, Any]]:
# How long the composer may take to accept the click that starts the film. Not a performance
-# budget: the point is that the cell survives and the number gets recorded. See `_press_send`.
-# 90s because it has to be well clear of the worst real reading and still bounded, since a click
-# that never lands is a different fact from a slow one.
+# budget: the point is that the cell survives and the number gets recorded. 90s, well clear of
+# the worst real reading and still bounded.
COMPOSER_CLICK_TIMEOUT_S = 90
-# Above this the log says so out loud, because a multi-second click is the user complaint itself
-# and it should not be something you only find by reading the payload afterwards.
+# Above this the log says so out loud, because a multi-second click is the user complaint itself.
SLOW_COMPOSER_CLICK_MS = 1_000
@@ -281,8 +259,7 @@ class Session:
yield w
finally:
w.t_close_ms = self._now_ms()
- # REVERSE order on close, so an instrument that wrapped another's state unwinds after
- # the one it wrapped.
+ # REVERSE order on close, so an instrument that wrapped another's state unwinds after the one it wrapped.
for inst in sorted(self.instruments, key = lambda i: i.name, reverse = True):
got = self._safe(inst, "close", w)
if got is not None:
@@ -340,39 +317,31 @@ class CellRunner:
log: Callable[[str], None] = print
image_path: Optional[Path] = None
cadence: str = "field"
- #: Record the NORMALISED signature text beside each digest. Off by default and deliberately
- #: so: the text is megabytes per capture. `sweep/parity_null_control.py --hunt` is the only
- #: consumer, and it needs it because a digest pair can only say THAT two DOMs differ, never
- #: which bytes moved -- and "which bytes" is the entire difference between fixing a
- #: normalisation gap and guessing at one.
+ #: Record the NORMALISED signature text beside each digest. Off by default: the text is megabytes
+ #: per capture, and `sweep/parity_null_control.py --hunt` is the only consumer -- it needs it
+ #: because a digest pair can say THAT two DOMs differ, never which bytes moved.
parity_raw: bool = False
- #: Directory for the per-action viewport PNGs, or None. Set only when the caller intends to
- #: produce before/after evidence; the encode is cheap but the files are not free.
+ #: Directory for the per-action viewport PNGs, or None. Set only when the caller intends
+ #: before/after evidence; the encode is cheap but the files are not free.
parity_shots: Optional[str] = None
- #: Which ARM this runner drives. It is burned into every filename, because a screenshot pair
- #: whose halves cannot be told apart is worse than no pair: both arms share a fixture, a
- #: password and a film, so the image itself carries nothing that identifies the side.
+ #: Which ARM this runner drives, burned into every filename: both arms share a fixture, a password
+ #: and a film, so the image itself carries nothing that identifies the side.
arm_label: str = "base"
- # Set once the 10K check fails, and it then labels every LARGER rung, which is the whole
- # point: those rungs are mostly seeded and their fidelity depends on this one answer.
+ # Set once the 10K check fails, and it then labels every LARGER rung: those rungs are mostly
+ # seeded and their fidelity depends on this one answer.
equivalence_failed: bool = False
- # Run the click attribution probe before the film starts. Off by default: it costs a handful
- # of seconds at small rungs and a great deal at large ones, and it makes the cell's timings
- # incomparable with a cell that did not run it. See `_click_attribution`.
+ # Run the click attribution probe before the film. Off by default: it costs a great deal at large
+ # rungs and makes the cell's timings incomparable with a cell that did not run it.
click_probe: bool = False
- # WHICH READINESS GATE. `full` is the default and is what every normal arm runs: every seeded
- # message must be mounted, plus the settle and end-present conditions the gate gained. An arm
- # that mounts a WINDOW on purpose sets `windowed` and is then held to a different set of
- # conditions, none of them weaker. See runtime/readiness.py.
- #
- # It is a per-TARGET setting, so a base-versus-virtualised A/B runs the base arm on `full` and
- # only the treatment arm on `windowed`, and the base arm keeps the strict gate it always had.
+ # WHICH READINESS GATE. `full` is the default every normal arm runs; an arm that mounts a WINDOW
+ # on purpose sets `windowed` and is held to a different set of conditions, none of them weaker.
+ # See runtime/readiness.py.
+ # Per TARGET, so a base-versus-virtualised A/B keeps the base arm on its strict gate.
readiness_mode: str = MODE_FULL
- # Scroll a windowed thread to its top once per cell, before the measured window, to prove the
- # arm still holds the head of the conversation. Costs a full traversal, so it is off for
- # `full` (where the whole thread is mounted and there is nothing to prove) and on by default
- # for `windowed`, where it is the only check that separates a virtualised thread from one that
- # has lost most of its messages.
+ # Scroll a windowed thread to its top once per cell, before the measured window, to prove the arm
+ # still holds the head of the conversation. Costs a full traversal, so it is off for `full` and
+ # on by default for `windowed`, where it is the only check that separates a virtualised thread
+ # from one that has lost most of its messages.
completeness_probe: Optional[bool] = None
def run(self, cell: Cell, plan: RungPlan) -> dict:
@@ -398,9 +367,8 @@ class CellRunner:
"target_tokens": plan.target_tokens,
"instruments": {},
}
- # Cleared HERE, not beside the click that produces it, so the preservation in the `finally`
- # below cannot attach the PREVIOUS cell's attribution to a cell that died before its own
- # probe ran. A reading filed under the wrong cell id is worse than a missing one.
+ # Cleared HERE, not beside the click that produces it, so the preservation in the `finally` below
+ # cannot attach the PREVIOUS cell's attribution to a cell that died before its own probe ran.
self._click_attribution_result = None
try:
self._run_inner(cell, plan, row)
@@ -412,10 +380,8 @@ class CellRunner:
"traceback": traceback.format_exc()[-3000:],
}
if isinstance(exc, ThreadNotReady):
- # WHICH CONDITION, not just that it timed out. The old gate's message was "the
- # thread mounted 9 of 18 messages", which named a count and left the reader to
- # guess whether the app was slow, the thread was short or the arm mounts a window
- # on purpose. The verdict now carries every condition and the last probe reading.
+ # WHICH CONDITION, not just that it timed out: the old message named a count and left the reader
+ # to guess whether the app was slow, the thread short, or the arm windowed on purpose.
row["failure"]["readiness"] = exc.detail
row["readiness"] = exc.detail
rec.gate(
@@ -430,35 +396,23 @@ class CellRunner:
dump_diagnostics(page, self.paths.logs, f"fail_{cell.cell_id}", self.log)
finally:
row["instruments"].update(s.each_instrument("end_cell", cell))
- # A cell that could not complete is a FIRST-CLASS RESULT with its failure mode and its
- # RSS at death, not a gap in the table.
+ # A cell that could not complete is a FIRST-CLASS RESULT with its failure mode and its RSS at
+ # death, not a gap in the table.
rss = row["instruments"].get("rss") or {}
row["rss_at_death_mb"] = rss.get("rss_peak_mb") if not row["completed"] else None
- # AND SO IS THE PROBE THAT ALREADY RAN. `--click-probe` finishes inside `_press_send`,
- # and everything after it there -- the `page.fill`, the send button lookup and its
- # click -- still runs under the default 8s action timeout, which is exactly what a
- # large rung exceeds. Assigned only on the way out of `_run_inner`, the whole
- # attribution was dropped from the cell it was measured for, and unlike
- # `composer_click_ms` it has no window row of its own to survive in: nothing else in
- # the payload records it. So it is filed from here, on both paths.
+ # AND SO IS THE PROBE THAT ALREADY RAN: `--click-probe` finishes inside `_press_send`, and
+ # everything after it there still runs under the default 8s action timeout, which a large rung
+ # exceeds. Assigned only on the way out of `_run_inner`, the attribution was dropped from the
+ # cell it was measured for, and unlike `composer_click_ms` it has no window row to survive in.
if self._click_attribution_result is not None:
row["click_attribution"] = self._click_attribution_result
rec.emit(row)
- # A TERMINAL MARKER FOR A CELL THAT DID NOT FINISH, so a reader scanning FORWARD can
- # discard its windows without having to join backwards to the cell row.
- #
- # `window` rows are written as the film runs; the `cell` row is written when the film
- # ends. An in-flight or aborted cell therefore leaves a complete-looking set of window
- # rows behind with no completed cell owning them, and an analysis that reads windows
- # directly picks them up. `floor_table.cell_metrics` has always guarded this; nothing
- # else did.
- #
- # This cost a headline. Reading `stream:gap` windows without the guard reported the
- # 1M rung at 28.7 fps and 21.8% of frames over 33 ms against a 46.7 fps baseline, an
- # apparent large regression at the rung closest to the user complaint, drawn entirely
- # from a cell that had not finished. An incomplete cell is not a shorter film but a
- # different one: at 1M the completed cell emitted 6 `stream:gap` windows against 17 at
- # 100K, so an in-flight cell contributes whichever phase it had reached.
+ # A TERMINAL MARKER FOR A CELL THAT DID NOT FINISH, so a reader scanning FORWARD can discard its
+ # windows without joining backwards. `window` rows are written as the film runs and the `cell`
+ # row when it ends, so an aborted cell leaves a complete-looking set nothing owns. It cost a
+ # headline: reading `stream:gap` windows without the guard reported the 1M rung at 28.7 fps
+ # against a 46.7 fps baseline, drawn entirely from an unfinished cell.
+ # And 21.8% of frames over 33 ms.
if not row["completed"]:
rec.emit(
{
@@ -495,8 +449,8 @@ class CellRunner:
raise ValueError(f"unknown readiness mode {self.readiness_mode!r}")
readiness = self._wait_for_thread(page, seeded)
row["readiness"] = readiness.as_dict()
- # RECORDED AS A GATE, so no reader can pick up a windowed cell's frame rate without also
- # seeing that its readiness was established a different way.
+ # RECORDED AS A GATE, so no reader can pick up a windowed cell's frame rate without also seeing
+ # that its readiness was established a different way.
rec.gate(
f"thread_ready:{self.readiness_mode}",
True,
@@ -504,9 +458,8 @@ class CellRunner:
cell_id = cell.cell_id,
)
- # THE COMPLETENESS PROBE, before the idle window and therefore before anything is measured.
- # It scrolls the whole thread, which mounts rows and dirties the page, so it has to be
- # followed by the idle window rather than preceding a measurement directly.
+ # THE COMPLETENESS PROBE, before the idle window and therefore before anything is measured. It
+ # scrolls the whole thread, which mounts rows and dirties the page, so the idle window follows.
do_probe = (
self.completeness_probe
if self.completeness_probe is not None
@@ -521,8 +474,8 @@ class CellRunner:
)
row["completeness"] = completeness
record_completeness_gate(rec, cell, completeness)
- # Back to the resting state the gate described, or the idle calibration below runs
- # against a page that is still settling from the traversal.
+ # Back to the resting state the gate described, or the idle calibration below runs against a page
+ # still settling from the traversal.
self._wait_for_thread(page, seeded)
# ── the enforced idle window ────────────────────────────────
@@ -536,9 +489,8 @@ class CellRunner:
w.note("clamp", clamp)
row["clamp"] = clamp
if clamp.get("clampMs") is None:
- # NOT fatal, and NOT silently zero. Blocked time is a subtraction against this floor,
- # so without it busy_pct is null with the reason attached and every other column
- # stands.
+ # NOT fatal, and NOT silently zero: blocked time is a subtraction against this floor, so without
+ # it busy_pct is null with the reason attached and every other column stands.
self.log(f" timer clamp NOT established: {clamp.get('reason')}")
rec.gate("timer_clamp", False, clamp, cell_id = cell.cell_id)
else:
@@ -575,7 +527,6 @@ class CellRunner:
f"(measured via {cpt.get('source')})"
)
- # ── the film ────────────────────────────────────────────────
unit = plan.streamed_unit
self.pacer.reset()
self.pacer.load(
@@ -592,24 +543,19 @@ class CellRunner:
f"content chars, cadence {self.cadence}, {expected_ms / 1000:.0f}s expected"
)
- # RESET THE FOLLOW SAMPLER FOR THIS CELL, immediately before the film starts.
- #
- # Necessary because the counters were just made to survive a navigation, by persisting to
- # sessionStorage -- and a cell boundary IS a navigation, to the next seeded thread on the
- # same origin. Without this, cell 2 reports cell 1's samples plus its own, cell 3 reports
- # all three, and a single bad cell early in a session poisons every reading after it while
- # each one still looks like a per-cell number.
+ # RESET THE FOLLOW SAMPLER FOR THIS CELL, immediately before the film starts: the counters now
+ # survive a navigation via sessionStorage, and a cell boundary IS a navigation, so without this
+ # cell 2 reports cell 1's samples plus its own and one bad cell poisons every later reading.
with contextlib.suppress(Exception):
page.evaluate("() => window.__sb.follow && window.__sb.follow.reset()")
before_metrics = cdp_metrics(s.ctx.cdp)
self._composer_click_ms = None
- # `click_attribution` is NOT filed here. It is filed in `run`'s `finally`, because a cell
- # that dies after the probe has to keep it: see there.
+ # `click_attribution` is NOT filed here but in `run`'s `finally`, because a cell that dies after
+ # the probe has to keep it.
t0 = self._press_send(page)
- # On the cell rather than in `actions`, because it happens before the first slot opens and
- # filing it as an action would put a reading outside the film into a list the scoring layer
- # pairs by slot. It is still a per-cell timing and grows with the rung like any other.
+ # On the cell rather than in `actions`, because it happens before the first slot opens and filing
+ # it as an action would put a reading outside the film into a list scoring pairs by slot.
row["composer_click_ms"] = self._composer_click_ms
scene = scene_schedule.SCENES.get(self.tier, scene_schedule.QUICK)
@@ -630,15 +576,13 @@ class CellRunner:
"parity_shots": self.parity_shots,
"arm_label": self.arm_label,
"image_path": str(self.image_path) if self.image_path else None,
- # The follow-up turns `send_turn` streams mid-film, and
- # the pacer it reloads to serve them.
+ # The follow-up turns `send_turn` streams mid-film, and the pacer it reloads to serve them.
"_pacer": self.pacer,
"_stream_queue": [
{"reasoning": u.reasoning, "content": u.content, "kind": u.kind}
for u in (plan.follow_up_units or [])
],
- # Shared and MUTABLE, so consecutive sends advance
- # through the queue. See send_turn.
+ # Shared and MUTABLE, so consecutive sends advance through the queue. See send_turn.
"_stream_cursor": {"i": 0},
"_input_instrument": next((i for i in s.instruments if i.name == "input"), None),
},
@@ -654,15 +598,11 @@ class CellRunner:
drained = self._drain_stream(page, expected_ms)
w.note("drained", drained)
row["stream"] = drained
- # DID THE THREAD FOLLOW THE STREAM? Read once, here, after the last window of the film has
- # closed, so the reading is charged to nothing. See the sampler in scene/dom.js.
- #
- # This is a GATE, not a column, because of the specific way it fails. A thread that stops
- # following lets the streamed message leave the viewport; a windowed list then unmounts
- # it, and the streaming cost collapses because the renderer is no longer rendering the
- # thing being measured. The frame rate that comes out is excellent and is about nothing.
- # A number with a caveat attached is still a number people quote without the caveat, so
- # the caveat is a gate row that a reader has to step over.
+ # DID THE THREAD FOLLOW THE STREAM? Read once, here, after the last window has closed, so the
+ # reading is charged to nothing. A GATE, not a column, because of how it fails: a thread that
+ # stops following lets the streamed message leave the viewport, a windowed list unmounts it, and
+ # the streaming cost collapses -- an excellent frame rate about nothing. A number with a caveat
+ # attached is still quoted without the caveat.
follow = self._read_follow(page)
row["follow"] = follow
pinned = follow.get("pinned_fraction")
@@ -670,29 +610,18 @@ class CellRunner:
passed, recorded = follow_verdict(follow)
follow.update(recorded)
rec.gate("follows_the_stream", passed, follow, cell_id = cell.cell_id)
- # THE OTHER HALF OF THE CONTRACT, RECORDED AND DELIBERATELY NOT GATED.
- #
- # It was a gate for one run and it failed on BOTH arms at nearly the same rate (32 of 79
- # on the base, 38 of 85 on the treatment), which is the signature of a reading about the
- # film rather than about the build. The film re-pins legitimately and often: `send_turn`
- # and `stop_generation` each START A RUN, and pinning to the bottom on run start is the
- # intended behaviour, not a violation; `scroll_after` ends its own gesture near the bottom
- # by design. Separating a legitimate re-pin from a yank requires knowing which pins the
- # app was ASKED for, which this sampler does not know.
+ # THE OTHER HALF OF THE CONTRACT, RECORDED AND DELIBERATELY NOT GATED. It was a gate for one run
+ # and failed on BOTH arms at nearly the same rate, the signature of a reading about the film:
+ # `send_turn` and `stop_generation` each START A RUN, where pinning is intended, and
+ # `scroll_after` ends its gesture near the bottom by design. Separating a legitimate re-pin from
+ # a yank needs to know which pins the app was ASKED for, which this sampler does not know, so it
+ # is a per-arm figure compared BETWEEN arms where the confounds cancel.
#
- # So it is reported as a per-arm figure to be compared BETWEEN arms, where the confounds
- # are common to both and cancel, and it carries the reason it is not a verdict. Gating on
- # it would have failed the shipped build.
row["scroll_intent"] = {
- # THE ATTESTATION, without which this block fails the bare-zero ban and takes the whole
- # report with it. `yanked_back_samples: 0` beside a non-zero `detached_samples` is the
- # GOOD outcome -- the user scrolled away and the app left them there -- and the walker
- # in scoring/schema.py cannot tell that from a counter nobody wrote. `follow` itself
- # already carries `follow_attempted` and is covered by it; this block is derived from
- # the same read and had nothing, so a real CI session refused to render at all with
- # `bare zeros found: $.cells[0].scroll_intent.yanked_back_samples = 0`. False here
- # rather than absent: the sampler that was never installed reports None for both
- # counters, so "not measured" stays distinguishable from "measured zero".
+ # THE ATTESTATION, without which this block fails the bare-zero ban: `yanked_back_samples: 0`
+ # beside a non-zero `detached_samples` is the GOOD outcome, and the walker in scoring/schema.py
+ # cannot tell that from a counter nobody wrote. False here rather than absent, so "not measured"
+ # stays distinguishable from "measured zero".
"follow_attempted": bool(follow.get("follow_attempted")),
"detached_samples": follow.get("detached_samples"),
"yanked_back_samples": follow.get("yanked_back_samples"),
@@ -722,11 +651,10 @@ class CellRunner:
f"after the user scrolled away"
+ (" -- THE USER WAS YANKED DOWN" if follow.get("yanked_after_scroll") else "")
)
- # EVERY STREAM THE CELL SERVED, not just the last one. `last_stats()` describes whichever
- # turn finished last, so for a multi-turn cell it says nothing at all about the opening
- # reply -- and the opening reply is the one the rung is named for. Everything stays under
- # the `pacer` key because that subtree is exempt from the payload's bare-zero rule: a
- # pacer counter of 0 is a true reading, not a missing one.
+ # EVERY STREAM THE CELL SERVED, not just the last one: `last_stats()` describes whichever turn
+ # finished last, so for a multi-turn cell it says nothing about the opening reply the rung is
+ # named for. Everything stays under `pacer` because that subtree is exempt from the bare-zero
+ # rule.
streams = self.pacer.all_stats()
planned = self._planned_streams(cell, plan, row)
row["pacer"] = {
@@ -734,58 +662,32 @@ class CellRunner:
"streams": streams,
"check": check_planned_streams(streams, planned),
}
- # A REPLY THAT NEVER FINISHED IS A FAILED CELL, not a completed one with a note.
- #
- # `_drain_stream` reports rather than raises, and the value it reports was recorded here
- # and read by nothing: no gate, no report column, no `--assert-liveness` check and no
- # exit code. So the one state this tool exists to catch -- the app still generating three
- # times past its own cadence and 120 s beyond that, after the whole film has run -- came
- # back as `completed: true`, `ok` in the summary and exit 0, with its actions and its
- # frame windows scored and paired into the A/B ratio against an arm that DID finish.
- # That is the crash-beats-limp rule inverted: a build that cannot finish reads as a build
- # that had nothing to say.
- #
- # Raised AFTER the drain reading AND the pacer's own counters are on the row, so the two
- # facts that say WHY it never finished -- how long was waited, and how much the pacer
- # actually delivered -- ship in the same row as the failure. `CellRunner.run` catches
- # this, records `failure`, dumps the diagnostics and keeps the cell as a first-class
- # incomplete result; the rung then scores INCOMPLETE, which is exactly how this harness
- # reports a build that died at 500K. The censuses below are not taken, because a census of
- # a thread that is still growing describes nothing that was measured.
+ # A REPLY THAT NEVER FINISHED IS A FAILED CELL, not a completed one with a note. `_drain_stream`
+ # reports rather than raises, and the value was read by nothing, so an app still generating three
+ # times past its own cadence came back as `completed: true`, exit 0, paired into the A/B ratio.
+ # And 120 s beyond that, after the whole film had run.
+ # Raised AFTER the drain reading and the pacer's counters are on the row, so how long was waited
+ # ships with the failure. The censuses below are not taken, because a census of a still-growing
+ # thread describes nothing that was measured.
if not drained.get("finished"):
raise RuntimeError(
f"the reply never finished: {drained.get('reason') or 'the run was still going'} "
f"({drained.get('drain_ms')}ms waited, {drained.get('expected_ms')}ms expected)"
)
- # A CELL THAT DID NOT STREAM WHAT IT PLANNED IS A FAILED CELL, for the same reason.
- #
- # The drain check above only asks whether the UI stopped running, and a later turn that
- # finishes satisfies it on behalf of an earlier one that did not. So an opening reply that
- # disconnected part way through left a complete-looking cell whose thread was thousands of
- # characters short of its rung, averaged into the A/B ratio against arms that streamed in
- # full. Raised here, after the check is on the row, so the reason and the per-turn evidence
- # ship with the failure and the cell is excluded by name rather than quietly included.
+ # A CELL THAT DID NOT STREAM WHAT IT PLANNED IS A FAILED CELL, for the same reason: the drain
+ # check only asks whether the UI stopped running, and a later turn that finishes satisfies it on
+ # behalf of an earlier one that did not, so an opening reply that disconnected left a
+ # complete-looking cell thousands of characters short of its rung.
check = row["pacer"]["check"]
if check["checked"] and not check["ok"]:
self.log(f" the cell did not stream what it planned: {check['reason']}")
raise RuntimeError(f"the cell did not stream what it planned: {check['reason']}")
- # AND THE SAME RULE FOR THE TURN THAT STREAMED BUT NEVER LANDED.
- #
- # The stream check asks whether the bytes went out. `send_turn` asserts something the pacer
- # cannot see: that the thread GREW. A send whose bytes were served in full but whose reply
- # never joined the thread leaves every later action, the peak census and the equivalence
- # mirror reading a thread one turn short, and the stream check alone would pass it.
- #
- # Scoped to `send_turn` DELIBERATELY, and this is the whole of the generalisation. An
- # action whose own assertion fails already has its own timing voided by
- # `scoring/from_payload._action_measure`, which returns `Measure.failed` for
- # `expect_ok is False` and says so. What that does NOT cover is an action whose failure
- # changed the workload the REST of the cell measured, and `send_turn` is the only one that
- # can: it is the only action whose outcome decides how much content the thread carries for
- # everything after it. A `select_text` that selected nothing or a `message_menu` that did
- # not open leaves the workload intact, and failing the cell for those would throw away a
- # whole cell's frame readings -- which really were taken, over the real thread -- for a
- # gesture that missed.
+ # AND THE SAME RULE FOR THE TURN THAT STREAMED BUT NEVER LANDED: the stream check asks whether
+ # the bytes went out, while `send_turn` asserts that the thread GREW. Scoped to `send_turn`
+ # DELIBERATELY -- an action whose own assertion fails already has its timing voided, and what
+ # that does not cover is an action whose failure changed the workload the REST of the cell
+ # measured. `send_turn` is the only one that can.
+ # The covered case is `expect_ok is False`.
missed_turns = [
a
for a in (row["actions"] or [])
@@ -802,38 +704,26 @@ class CellRunner:
row["census_after"] = dom_signature(page)
row["cdp"] = cdp_counters(before_metrics, cdp_metrics(s.ctx.cdp))
- # THE PEAK, over every window's census, not the state at the end of the film.
- #
- # The film ENDS with thread_reopen and delete_message, so an end-of-cell census reports
- # the thread the benchmark has just deleted. The first working run recorded 0 assistant
- # messages and 0 characters against a reply the pacer's own log proved it had delivered in
- # full: 150 chunks, 3,581 characters, at exactly the 73ms cadence. The occupancy that
- # every per-action cost has to be read against is the peak, and it is now recovered from
- # the per-window censuses rather than from a single reading taken at the worst moment.
+ # THE PEAK, over every window's census, not the state at the end of the film: the film ENDS with
+ # thread_reopen and delete_message, so an end-of-cell census reports the thread the benchmark has
+ # just deleted. The first working run recorded 0 assistant messages against a delivered reply.
+ # The pacer's own log proved it: 150 chunks, 3,581 characters, at exactly the 73 ms cadence.
+ # 0 messages and 0 characters.
censuses = [w.get("census") for w in row["actions"] if isinstance(w.get("census"), dict)]
censuses = [c for c in censuses if c.get("elements")]
peak = max(censuses, key = lambda c: c.get("elements", 0)) if censuses else {}
row["census_peak"] = peak
row["census_peak_attempted"] = bool(censuses)
- # WHICH ACTION THE PEAK CAME FROM, and a standing refusal to compare it across arms.
- #
- # `census_peak` is whichever per-action census had the most elements. The census attached
- # to an action is taken after the action returns, and `reasoning_toggle` opens every pane
- # and then closes them again, so that census races the close. Which action wins the max()
- # therefore depends on the race and DIFFERS BETWEEN ARMS.
- #
- # Measured on a null control -- the same bundle served on both arms -- the winner flipped
- # between `settings` (panes collapsed, 64,648 elements) and `reasoning_toggle` (panes open,
- # 106,067), a 70.1% swing in `highlight_spans` WITHIN one arm.
- #
- # This produced a published wrong number: main was reported as mounting 48% more Shiki
- # spans than a pre-campaign baseline, and an attribution bisect was nearly commissioned to
- # find the cause. Settled, the two trees mount the same document to within 0.3%.
- #
- # It stays in the payload because it is useful as a diagnostic high-water mark. It carries
- # its provenance and an explicit refusal so that no analysis can quote it across arms
- # without stepping over a flag that says not to.
+ # WHICH ACTION THE PEAK CAME FROM, and a standing refusal to compare it across arms. The census
+ # attached to an action is taken after it returns, and `reasoning_toggle` opens every pane and
+ # closes them again, so that census races the close and which action wins the max() DIFFERS
+ # BETWEEN ARMS. Measured on a null control the winner flipped between two actions, a 70.1% swing
+ # WITHIN one arm, which produced a published wrong number. Kept as a diagnostic high-water mark,
+ # carrying its provenance and an explicit refusal.
+ # The winner flipped between `settings` at 64,648 elements and `reasoning_toggle` at 106,067,
+ # published as main mounting 48% more Shiki spans.
+ # Settled, the two trees mount the same document to within 0.3%.
peak_from = next(
(
w.get("action")
@@ -858,10 +748,9 @@ class CellRunner:
if chars is None:
chars = page.evaluate("() => window.__sb.dom.assistantChars()")
row["assistant_chars_in_dom"] = chars
- # The span density the fixture ACHIEVED, measured in the DOM rather than assumed. The
- # field capture ran 5.6 characters per span; a corpus that lands far from that is not
- # standing in for the same highlighter load per character, and the report should say so
- # rather than quietly compare two different workloads.
+ # The span density the fixture ACHIEVED, measured in the DOM rather than assumed: the field
+ # capture ran 5.6 characters per span, and a corpus far from that is not standing in for the same
+ # highlighter load per character.
row["chars_per_span"] = round(chars / spans, 2) if spans else None
row["chars_per_span_target"] = 5.6
self.log(
@@ -882,9 +771,9 @@ class CellRunner:
cell_id = cell.cell_id,
)
if not eq.get("equivalent"):
- # A FINDING, printed, not a bug to hide. It says exactly which of this tool's
- # numbers are about the streaming path and which are about a thread that was put
- # there, and it is the reason the higher rungs carry a fidelity label at all.
+ # A FINDING, printed, not a bug to hide: it says which of this tool's numbers are about the
+ # streaming path and which are about a thread that was put there, and it is why the higher rungs
+ # carry a fidelity label at all.
self.log(
" SEEDED IS NOT EQUIVALENT TO STREAMED at the 10K rung. Rungs above it "
"are labelled fidelity: seeded_only."
@@ -901,9 +790,8 @@ class CellRunner:
" seeded and streamed agree on CONTENT at the 10K rung within "
f"{eq['tolerance']:.0%}"
)
- # Passing the content gate is not the same as the two threads being identical,
- # and the difference is large enough that leaving it unsaid would mislead: a
- # seeded rung carries the same rendered content and materially less mounted DOM.
+ # Passing the content gate is not the same as the two threads being identical: a seeded rung
+ # carries the same rendered content and materially less mounted DOM.
fields = eq.get("fields") or {}
for key in ("reasoning_spans", "highlight_spans", "assistant_chars"):
field = fields.get(key) or {}
@@ -995,17 +883,11 @@ class CellRunner:
"""
s = self.session
page = s.ctx.page
- # THE STREAMED SIDE IS THE PEAK, AND THE PEAK IS TAKEN AT AN UNSTABLE MOMENT. Recorded on
- # the row rather than left implicit, because the seeded side below is read after an
- # explicit 4 s wait -- a settled moment -- so this gate differences a racing census against
- # a stable one. That is the same shape as the defect that made `census_peak` unquotable
- # across arms.
- #
- # It is NOT the same harm and the behaviour is deliberately left alone here: both sides of
- # this comparison come from ONE cell on ONE build, so the instability widens the gate's
- # tolerance rather than pointing a difference in a direction. Changing which census feeds
- # it would change what the gate asserts, and that cannot be justified without measuring it
- # against a browser. Said out loud so the next reader does not have to rediscover it.
+ # THE STREAMED SIDE IS THE PEAK, AND THE PEAK IS TAKEN AT AN UNSTABLE MOMENT, recorded on the row
+ # because the seeded side below is read after an explicit 4 s wait -- so this gate differences a
+ # racing census against a stable one, the same shape as the defect that made `census_peak`
+ # unquotable across arms. NOT the same harm: both sides come from ONE cell on ONE build, so the
+ # instability widens the tolerance rather than pointing a difference in a direction.
streamed = row.get("census_peak") or row.get("census_after") or {}
streamed_from = "census_peak" if row.get("census_peak") else "census_after"
follow_ups = self._streamed_follow_ups(plan, row)
@@ -1042,20 +924,17 @@ class CellRunner:
out["seeded_census_settled"] = True
out["readiness_mode"] = self.readiness_mode
if self.readiness_mode == MODE_WINDOWED:
- # SAID OUT LOUD RATHER THAN SCORED QUIETLY. Both sides of this check are loaded by the
- # SAME build, so under a windowed arm both censuses count the mounted window and not
- # the thread. The comparison is still like for like and its pass still means something
- # -- the two paths render the same window the same way -- but it is no longer evidence
- # that seeding reproduces the whole streamed thread, because neither side has the whole
- # thread in the DOM to compare.
+ # SAID OUT LOUD RATHER THAN SCORED QUIETLY: both sides are loaded by the SAME build, so under a
+ # windowed arm both censuses count the mounted window. The comparison is still like for like, but
+ # it is no longer evidence that seeding reproduces the whole streamed thread.
out["scope"] = "the mounted window only, not the whole thread"
out["caveat"] = (
"this arm mounts a window, so `assistant_messages`, `content_spans` and "
"`content_code_blocks` are counts over what is mounted at the end of the thread. "
"A pass is equivalence of the WINDOW, not of the thread."
)
- # What the mirror was built from, so a drift can be read against the corpus it compared
- # rather than against an assumption about which turns were in the thread.
+ # What the mirror was built from, so a drift can be read against the corpus it compared rather
+ # than an assumption about which turns were in the thread.
out["mirrored_follow_ups"] = len(follow_ups)
out["planned_follow_ups"] = len(plan.follow_up_units or [])
return out
@@ -1142,47 +1021,29 @@ class CellRunner:
page.evaluate("() => document.body.offsetHeight")
return (time.monotonic() - started) * 1000.0
- # FIRST, before anything else touches the page: the same trivial operation N times.
- #
- # This exists because the biggest number in the whole ladder is one I could not attribute.
- # The first thing touched after a large thread mounts costs 11 to 24 seconds, and in every
- # probe the cost vanished, because whatever ran first absorbed it and by the time the
- # interesting path ran the page was warm. Measuring the decay directly is the way out: if
- # the first reading is seconds and the rest are milliseconds, the cost is ONE TIME and its
- # size is the first reading minus the steady state. If they are all expensive, it is a
- # per-interaction cost and the ladder's number was never a one-off.
- #
- # A no-op body on purpose. Every reading is the same work, so any difference between them
- # is the page's state and not the operation.
+ # FIRST, before anything else touches the page: the same trivial operation N times. The biggest
+ # number in the ladder is one nobody could attribute -- the first thing touched after a large
+ # thread mounts costs 11 to 24 seconds, and in every probe the cost vanished because whatever ran
+ # first absorbed it. Measuring the decay directly is the way out: a first reading in seconds and
+ # the rest in milliseconds means the cost is ONE TIME. A no-op body on purpose.
decay = [settled(lambda: None) for _ in range(5)]
out: dict[str, Any] = {
- # The harness layer's attestation, and it is load-bearing rather than decorative.
- # `scoring/schema._walk_for_bare_zeros` rejects any bare zero that is not covered by a
- # sibling `*_attempted` flag, and this block has legitimate zeros: a thread with no
- # mounted code blocks records `code_token_spans: 0`, and `blur_inpage_ms` and
- # `forced_layout_ms` come from `performance.now()`, which Chromium coarsens to 100 us
- # outside a cross-origin-isolated context, so an operation shorter than that reads
- # exactly 0. Without this flag `--report` refuses the whole payload of a completed
- # probe run.
+ # The harness layer's attestation, load-bearing rather than decorative:
+ # `scoring/schema._walk_for_bare_zeros` rejects a bare zero with no sibling `*_attempted` flag,
+ # and this block has legitimate zeros -- a thread with no mounted code blocks records
+ # `code_token_spans: 0`, and Chromium coarsens `performance.now()` to 100 us.
+ # Which is what `forced_layout_ms` is built from.
"click_attribution_attempted": True,
"first_touch_ms": decay[0],
"settled_touch_ms": min(decay[1:]),
"touch_decay_ms": [round(v, 1) for v in decay],
}
- # The blur, timed, and timed again from INSIDE the page.
- #
- # The decay series above says the first touch after mount costs 10.6 ms at 500K, so there
- # is no expensive first interaction. Yet the reading taken right after the first blur came
- # back at 10,052 ms and 10,017 ms in two of three runs. Two things about that: it sits
- # between the cheap decay series and the cheap readings that follow, so the blur is what it
- # attaches to; and 10.0 seconds to three digits is the shape of a TIMEOUT, not of work that
- # happens to take ten seconds.
- #
- # `blur_inpage_ms` decides which. It runs the same blur and the same forced layout inside a
- # single evaluate and times them with `performance.now()`, so the protocol is excluded. If
- # the page reports ten seconds, it is real work. If the page reports a millisecond while
- # the outer reading is ten seconds, the ten seconds is the driver or the transport and not
- # the app, and must never be quoted as a user cost.
+ # The blur, timed, and timed again from INSIDE the page. The decay series says the first touch
+ # after mount costs 10.6 ms at 500K, yet the reading right after the first blur came back at
+ # 10,052 ms in two of three runs -- and 10.0 seconds to three digits is the shape of a TIMEOUT.
+ # And 10,017 ms in the other.
+ # `blur_inpage_ms` decides which: it runs the same blur inside one evaluate, so if the page
+ # reports a millisecond while the outer reading is ten seconds, the ten seconds is the driver.
out["blur_outer_ms"] = settled(
lambda: page.evaluate("() => document.activeElement && document.activeElement.blur()")
)
@@ -1210,14 +1071,12 @@ class CellRunner:
page.mouse.move(2, 2)
page.wait_for_timeout(250)
out["hover_thread_ms"] = settled(lambda: page.mouse.move(x, 300))
- # The reading that decides what `roundtrip_ms` meant. If the first forced layout of a huge
- # thread is expensive and every later one is cheap, this comes back near zero and the cost
- # is a ONE-TIME layout of the mounted thread, not a per-interaction cost. If it is
- # expensive again, every interaction pays it and the story is the opposite.
+ # The reading that decides what `roundtrip_ms` meant: near zero means the cost is a ONE-TIME
+ # layout of the mounted thread, expensive again means every interaction pays it.
blur()
out["roundtrip_again_ms"] = settled(lambda: None)
- # Measured INSIDE the page, so the protocol round trip is not in the number. `offsetHeight`
- # is read after a write that dirties layout, so it cannot be served from a clean tree.
+ # Measured INSIDE the page, so the protocol round trip is not in the number. `offsetHeight` is
+ # read after a write that dirties layout, so it cannot be served from a clean tree.
out["forced_layout_ms"] = page.evaluate(
"() => { const t = performance.now();"
" document.body.style.minHeight = (1 + Math.random()) + 'px';"
@@ -1268,24 +1127,16 @@ class CellRunner:
page.wait_for_selector(selector, timeout = 60_000)
if self.click_probe:
self._click_attribution_result = self._click_attribution(page, selector)
- # In a window, so every instrument covers it. At 500K this single click is the largest
- # cost in the whole run by an order of magnitude, and it was the one moment the tool could
- # not see inside: no window, so no frame recorder, no CPU profile, no RSS delta. A cost
- # that big being outside every instrument is how it stayed unattributed.
- #
- # `setup`, NOT `action`. The scoring layer pools every non-excluded window of the cell into
- # `max_frame_ms`, `jank_index` and `time_in_jank_pct`, and this window is mostly Playwright's
- # own injected actionability script running ON THE PAGE'S MAIN THREAD -- it blocks frames
- # exactly as app work would, so the frame recorder cannot tell them apart. Filed as an
- # `action` it would put an 11 s driver stall into three weighted headline metrics whose
- # `max_frame_ms` anchor calls 2,000 ms the worst case, on EVERY run including the ones that
- # never asked for --click-probe. It is measured, reported and kept out of the film.
+ # In a window, so every instrument covers it: at 500K this single click is the largest cost in the
+ # run by an order of magnitude and was the one moment the tool could not see inside. `setup`, NOT
+ # `action`: the scoring layer pools every non-excluded window into the three frame metrics, and
+ # this window is mostly Playwright's own actionability script running ON THE PAGE'S MAIN THREAD,
+ # so filed as an `action` it would put an 11 s driver stall into three weighted headline metrics.
+ # It would peg `max_frame_ms`, `jank_index` and `time_in_jank_pct`.
with self.session.window("setup:composer_click", kind = "setup"):
- # Timed INSIDE the window, like `Window.duration_ms` itself: the session opens every
- # instrument before this block and closes them after it, and at instrument level 1-3
- # those hooks stop a CPU profile, collect coverage and write and analyse a trace.
- # Timed around the `with`, `composer_click_ms` would grow with the instrument level
- # and stop being the click.
+ # Timed INSIDE the window, like `Window.duration_ms`: the session opens every instrument before
+ # this block and closes them after, and at instrument level 1-3 those hooks stop a CPU profile,
+ # collect coverage and analyse a trace, so timing around the `with` would grow with the level.
clicked_at = time.monotonic()
page.click(selector, timeout = COMPOSER_CLICK_TIMEOUT_S * 1000)
self._composer_click_ms = (time.monotonic() - clicked_at) * 1000.0
@@ -1301,16 +1152,15 @@ class CellRunner:
raise RuntimeError("the send button is not on the page, so no reply can be started")
t0 = time.monotonic()
send.click()
- # The composer must be EMPTY for the rest of the film, or the Stop control is replaced by
- # a Queue control and the stop action presses the wrong button. Sending clears it, but the
- # keystroke action refills it, which is why the stop action clears it again itself.
+ # The composer must be EMPTY for the rest of the film, or the Stop control is replaced by a Queue
+ # control and the stop action presses the wrong button. Sending clears it, but the keystroke
+ # action refills it, which is why the stop action clears it again itself.
return t0
def _drain_stream(self, page, expected_ms: float) -> dict:
"""Wait for the run to end, or say plainly that it did not."""
- # Generous: the whole point of deficit scheduling is that the stream's own duration is
- # machine-independent, so anything much past it is the RENDERER failing to keep up, which
- # is a finding rather than a timeout to paper over.
+ # Generous: deficit scheduling makes the stream's own duration machine-independent, so anything
+ # much past it is the RENDERER failing to keep up, which is a finding rather than a timeout.
deadline = time.monotonic() + (expected_ms / 1000) * 3 + 120
started = time.monotonic()
while time.monotonic() < deadline:
@@ -1340,10 +1190,9 @@ def make_context(
out_lock = None,
) -> tuple[BenchContext, Session]:
session_id = new_session_id()
- # THE LOCK THE CALLER IS ALREADY HOLDING. `run()` takes the output directory before it archives
- # a payload or installs an Unsloth, so the `Recorder` adopts that lock rather than opening a
- # second one against the same path. Without a caller's lock it takes its own, which is what
- # every test and every other consumer of a payload does.
+ # THE LOCK THE CALLER IS ALREADY HOLDING: `run()` takes the output directory before it archives a
+ # payload, so the `Recorder` adopts that lock rather than opening a second against the same path.
+ # Without a caller's lock it takes its own.
recorder = Recorder(paths.payload_jsonl, session_id, lock = out_lock)
ctx = BenchContext(
browser = browser_bundle.browser,
@@ -1376,12 +1225,11 @@ def make_context(
return ctx, Session(ctx = ctx, instruments = instruments)
-#: The sources that may SIZE a rung. `measure_chars_per_token` also has a last-resort
-#: whitespace-and-punctuation estimate, which it labels itself as "off by tens of percent on dense
-#: code" -- and this corpus is mostly dense code, where that estimate reads 6.7 against tiktoken's
-#: 3.3. Sizing the ladder from it would not make the axis honest, it would move the error and, past
+#: The sources that may SIZE a rung. `measure_chars_per_token`'s last-resort whitespace estimate
+#: labels itself "off by tens of percent on dense code", and this corpus is mostly dense code,
+#: where it reads 6.7 against tiktoken's 3.3 -- sizing from it would move the error and, past
#: `MANIFEST_CHARS_PER_TOKEN`, make `plan_rung` refuse the whole run on any machine with no
-#: tokeniser. A real tokeniser sizes the rungs; the estimate is still measured and still reported.
+#: tokeniser. The estimate is still measured and reported.
LADDER_RATIO_SOURCES = ("tiktoken/cl100k", "studio /api/inference/chat/count_tokens")
@@ -1478,10 +1326,9 @@ def build_cells(
}
out: list[tuple[Cell, RungPlan]] = []
for rung in rungs:
- # The ladder is sized by `ratio`, not by the raw `chars_per_token` argument: the argument
- # may be None, meaning "measure it", and the measured value is what every cell's `meta`
- # reports. Passing the argument straight through here would size the rungs from a number
- # the payload does not carry.
+ # The ladder is sized by `ratio`, not the raw `chars_per_token` argument: the argument may be None
+ # meaning "measure it", and the measured value is what every cell's `meta` reports, so passing the
+ # argument through would size the rungs from a number the payload does not carry.
plan = plan_rung(
corpus,
rung,
diff --git a/tests/studio/studiobench/runtime/types.py b/tests/studio/studiobench/runtime/types.py
index ce077d98307..08b41502a51 100644
--- a/tests/studio/studiobench/runtime/types.py
+++ b/tests/studio/studiobench/runtime/types.py
@@ -30,27 +30,24 @@ ROW_TYPES = frozenset(
"action",
"sample",
"failure",
- # The A/B run order, recorded before the first cell. Written even when the order is
- # UNBALANCED, because whether linear drift cancelled is a property of the run that a reader
- # of the table has no other way to recover.
+ # The A/B run order, recorded before the first cell. Written even when UNBALANCED, because
+ # whether linear drift cancelled is a property of the run a reader cannot otherwise recover.
"ab_plan",
- # A cell that did not finish, announced the moment it fails so a reader scanning FORWARD
- # can discard its window rows without joining backwards to a cell row it may never reach.
+ # A cell that did not finish, announced the moment it fails so a reader scanning FORWARD can
+ # discard its window rows without joining backwards.
"cell_aborted",
- # The comparability key: everything that must match for two payloads to be compared at
- # all, hashed into one quotable token. Its own row rather than a field on `run_meta` so a
- # reader can find it without parsing the whole meta block.
+ # The comparability key: everything that must match for two payloads to be compared, hashed
+ # into one quotable token. Its own row so a reader need not parse the whole meta block.
"comparability",
- # One UI surface swept by the optional `--surfaces` phase: a route, a settings tab, a menu.
- # A row type of its own rather than an `action` row with a different name, because a surface
- # has no slot, no budget and no timing to miss -- and reusing `action` would put forty rows
- # with a null `timings` into the column the report scores actions from.
+ # One UI surface swept by the optional `--surfaces` phase. A row type of its own because a
+ # surface has no slot, no budget and no timing to miss, and reusing `action` would put forty
+ # null-`timings` rows into the column the report scores actions from.
"surface",
}
)
-# Required keys per row type. Enforced in Recorder.emit, because a row that silently lost its
-# `ran` flag reads downstream as a fast action rather than as a missing one.
+# Required keys per row type. Enforced in Recorder.emit, because a row that lost its `ran` flag
+# reads downstream as a fast action rather than a missing one.
ROW_REQUIRED: dict[str, tuple[str, ...]] = {
"run_meta": (
"tier",
@@ -69,16 +66,12 @@ ROW_REQUIRED: dict[str, tuple[str, ...]] = {
"comparability": ("key", "fields"),
"sample": ("t_ms",),
"failure": ("kind", "detail"),
- # `reason` is REQUIRED, not optional. A surface row that lost its reason reads as a surface
- # that was reached, which is the one thing a coverage sweep may never claim by default. It is
- # null only on the success path, where `reached` is true.
+ # `reason` is REQUIRED: a surface row that lost it reads as a surface that was reached, the one
+ # thing a coverage sweep may never claim by default. Null only on the success path.
"surface": ("surface", "reached", "reason", "parity"),
}
-# ── the cell ────────────────────────────────────────────────────────
-
-
@dataclass(frozen = True)
class Cell:
"""One measured configuration: one rung, one arm, one repetition."""
@@ -122,15 +115,14 @@ def make_cell_id(rung: str, arm: str, rep: int) -> str:
return f"r{rung}.{arm}.rep{rep}"
-# ── the window ──────────────────────────────────────────────────────
-
#: `gap` is the quiet stretch the scheduler holds between two slots. It is NOT `stream`: a gap
-#: window is opened before every slot, so most of them sit long after the reply finished. See
-#: SceneRunner._gap_window for what that mislabelling cost.
-#: `setup` is work the harness has to do before the film can start -- the composer click is the
-#: one that costs anything -- and it is NOT `action`. It is dominated by the driver rather than by
-#: the app, so the scoring layer keeps it out of the frame pool. See
+#: window opens before every slot, so most sit long after the reply finished. See
+#: SceneRunner._gap_window.
+#: `setup` is pre-film harness work (the composer click is the costly one) and is NOT `action`:
+#: dominated by the driver rather than the app, so scoring keeps it out of the frame pool. See
#: `scoring/from_payload.UNSCORED_WINDOW_KINDS`.
+# ── the window ──────────────────────────────────────────────────────
+
WINDOW_KINDS = frozenset({"action", "stream", "gap", "idle", "setup", "settle", "teardown"})
@@ -168,9 +160,6 @@ class Window:
}
-# ── actions ─────────────────────────────────────────────────────────
-
-
@dataclass
class ActionResult:
"""The outcome of one action.
@@ -184,23 +173,17 @@ class ActionResult:
expect_ok: Optional[bool] = None
expect: dict = field(default_factory = dict)
timings: dict = field(default_factory = dict)
- # CORRECTNESS INVARIANTS, not timings, and kept apart from them on purpose.
- #
- # A count here answers "did the action still do the whole job", where a timing answers "how
- # long did it take". They are scored the same way and they mean opposite things when they
- # move: a timing falling is the result, a count falling is a regression.
- #
+ # CORRECTNESS INVARIANTS, not timings, and kept apart on purpose. A count answers 'did the
+ # action still do the whole job'; they move oppositely, a timing falling is the result and a
+ # count falling is a regression.
# This exists because `select_all_copy` asserted only `chars > 0`. Its selection is taken over
- # the viewport's DOM, so any change that stops mounting the whole thread -- windowing,
- # virtualization, a progressive mount that never widens -- truncates the clipboard silently
- # and still passes. From a user's point of view a copy that drops most of the conversation is
- # data loss, and it is the classic regression of every list that starts unmounting rows.
- #
- # The reference is the OTHER ARM rather than an absolute threshold. Both arms of an A/B seed a
- # byte-identical thread, so a treatment that truncates reads as a large negative delta scored
- # against the null control's own spread, and nothing has to be calibrated per rung or per
- # platform. A count is therefore only meaningful in a paired comparison, which is the only
- # place it is read.
+ # the viewport's DOM, so anything that stops mounting the whole thread truncates the clipboard
+ # silently and still passes: data loss, and the classic regression of every list that starts
+ # unmounting rows.
+ # The reference is the OTHER ARM rather than an absolute threshold: both arms seed a
+ # byte-identical thread, so a truncating treatment reads as a large negative delta against the
+ # null control's own spread with nothing calibrated per rung or platform. A count is therefore
+ # only meaningful in a paired comparison.
counts: dict = field(default_factory = dict)
reason: Optional[str] = None
slot_missed: bool = False
@@ -208,9 +191,8 @@ class ActionResult:
def __post_init__(self) -> None:
if not self.ran:
self.timings = {}
- # Same rule as `timings`, for the same reason: an action that did not happen has no
- # invariant to report, and a zero left here would read as "the whole job was done, and
- # it did nothing", which is the exact inversion of what happened.
+ # Same rule as `timings`: an action that did not happen has no invariant to report, and a zero
+ # here would read as 'the whole job was done, and it did nothing'.
self.counts = {}
self.expect_ok = None
if not self.reason:
@@ -287,9 +269,6 @@ class Instrument:
def detach(self) -> None: ...
-# ── paths and context ───────────────────────────────────────────────
-
-
@dataclass
class Paths:
out: Path
@@ -377,9 +356,9 @@ class OutDirLock:
"""
lock = cls(out)
lock.out.mkdir(parents = True, exist_ok = True)
- # The legacy per-session names, swept once. A directory left by an older build still has to
- # be read, or the guard would quietly switch itself off on exactly the runs it was added
- # for. Only the fixed name is ever a mutex.
+ # The legacy per-session names, swept once: a directory left by an older build still has to be
+ # read, or the guard switches itself off on exactly the runs it was added for. Only the fixed
+ # name is ever a mutex.
lock._refuse_if_legacy_marker_is_live()
lock._acquire(session_id)
return lock
@@ -475,12 +454,11 @@ class OutDirLock:
if fd is None:
return
self._fd = None
- # BLANKED BEFORE IT IS RELEASED, and only while this process still holds the lock, so
- # nothing can be reading it as authoritative. The file stays (see above); what goes is
- # its CONTENT, because a retained `pid session` line outlives the run that wrote it and
- # the next contender to lose a race would otherwise be handed it as the holder. Not a
- # substitute for the liveness test in `_read_marker_once_written`: a killed run never
- # reaches this line.
+ # BLANKED BEFORE IT IS RELEASED, and only while this process holds the lock, so nothing can read
+ # it as authoritative. The file stays; the CONTENT goes, because a retained `pid session` line
+ # outlives its run and the next contender to lose a race would be handed it as the holder. Not
+ # a substitute for the liveness test in `_read_marker_once_written`: a killed run never reaches
+ # this line.
try:
os.ftruncate(fd, 0)
except OSError:
@@ -562,21 +540,15 @@ class OutDirLock:
deadline = time.monotonic() + budget_s
while True:
got = cls._read_marker(path)
- # A RETAINED RECORD IS NOT A HOLDER. The marker is deliberately never unlinked, so on
- # any directory that has been used before it already contains the PREVIOUS run's
- # `pid session` line. That record is non-empty, so a bare "did it read" test stops on
- # it and the refusal names a run that finished hours ago as the current holder --
- # worse than saying nothing, because it sends the reader after a specific dead pid.
- # Measured: a clean `close()` leaves `2235618 sessionAAAA`, and a contender that meets
- # a NEW holder inside this window was told `session sessionAAAA is still running as
- # pid 2235618` while the actual holder was pid 2235621.
- #
- # Liveness is what separates them. The holder is by definition a running process, and
- # the run that wrote a retained record has exited. `close()` also blanks the marker
- # while it still holds the lock, so a clean exit leaves nothing to mistake; this covers
- # the unclean ones. What is left is a retained record whose pid has been RECYCLED onto
- # a live process, which needs an unclean exit and a wrapped pid space in the same
- # directory.
+ # A RETAINED RECORD IS NOT A HOLDER. The marker is never unlinked, so a reused directory already
+ # holds the PREVIOUS run's `pid session` line; a bare 'did it read' test stops on it and names
+ # a run that finished hours ago, sending the reader after a specific dead pid. Measured: a
+ # clean `close()` leaves `2235618 sessionAAAA`, and a contender meeting a NEW holder in this
+ # window was told that pid while the actual holder was 2235621.
+ # Liveness separates them: the holder is by definition running, and the run that wrote a
+ # retained record has exited. `close()` also blanks the marker under the lock, so this covers
+ # only unclean exits. What is left is a retained record whose pid was RECYCLED onto a live
+ # process, needing an unclean exit and a wrapped pid space in one directory.
if got is not None and cls._alive(got[1]):
return got
if time.monotonic() >= deadline:
@@ -629,48 +601,34 @@ class Recorder:
self.session_id = session_id
self.t0 = t0 if t0 is not None else time.monotonic()
# REFUSE A SECOND LIVE SESSION IN ONE OUTPUT DIRECTORY.
- #
- # Appending is correct across SHARDS, which are deliberate and sequential. It is never
- # correct for two runs going at once: they contend with each other, so neither measures
- # the machine the other thought it had, and they write the same `cell_id`s into one file
- # where a reader keyed on `cell_id` sees only the last writer.
- #
- # Observed: a launcher started twice produced three `run_meta` rows and every `cell_id`
- # twice, both marked completed, with `r1M.treatment.rep1` keystroke `p50_ms` reading
- # 73.4 ms in one session and 144.5 ms in the other. Scored last-wins that payload showed a
- # 149.8% regression that does not exist at that size.
- #
- # The marker carries the pid, so a crashed run leaves a marker that names a dead process
- # and the next run says so rather than refusing forever.
- #
+ # Appending is correct across SHARDS, which are deliberate and sequential, and never correct for
+ # two concurrent runs: they contend, so neither measures the machine the other thought it had,
+ # and they write the same `cell_id`s where a reader keyed on `cell_id` sees only the last
+ # writer.
+ # Observed: a launcher started twice produced three `run_meta` rows and every `cell_id` twice,
+ # both completed, with `r1M.treatment.rep1` keystroke `p50_ms` reading 73.4 ms in one session
+ # and 144.5 ms in the other, scored last-wins as a 149.8% regression that does not exist.
+ # The marker carries the pid, so a crashed run leaves a marker naming a dead process and the
+ # next run says so rather than refusing forever.
# ONE FIXED NAME, CREATED EXCLUSIVELY. The name used to be `.running.{session_id}`, and a
- # per-session name cannot be a mutex however carefully it is written: the two contenders
- # race on DIFFERENT paths, so each scanned the directory, each found only the other's
- # absence, and each then created its own marker. Scan-then-write is two syscalls with a
- # window between them, and the window is wide enough to lose. Measured on this guard
- # before the change, two processes released from a barrier against one output directory:
- # 123 of 200 trials admitted BOTH recorders, reproducing the defect-9 signature the guard
- # exists to prevent -- two `run_meta` rows, one `cell_id` completed twice, 73.4 ms against
- # 144.5 ms. Two processes merely launched back to back, with no barrier at all, still
- # collided about 3% of the time.
- #
- # `os.open(..., O_CREAT | O_EXCL)` makes the check and the creation one atomic operation
- # against other opens of the same name, which is the documented lock-file idiom and is
- # available on Unix and Windows alike. `fcntl.flock` is deliberately NOT used: the `fcntl`
- # module does not exist on Windows, and this tool is run by external testers there.
- #
- # TAKEN BEFORE THIS POINT WHEN THE CALLER HAS ONE, AND THAT IS THE NORMAL PATH. `run()`
- # holds the directory from its first millisecond, because by the time the payload is opened
- # a duplicate has already archived the live payload and installed two Unsloth instances on top of the
- # run it is about to be refused in favour of. See `OutDirLock`. A `Recorder` built without
- # one -- the tests, and any caller that only wants a payload -- still takes its own, so the
- # guard cannot be switched off by forgetting to pass it.
- #
+ # per-session name cannot be a mutex: the contenders race on DIFFERENT paths, each finding only
+ # the other's absence. Measured before the change, two processes released from a barrier onto
+ # one directory admitted BOTH recorders in 123 of 200 trials; merely launched back to back they
+ # still collided about 3% of the time.
+ # `os.open(..., O_CREAT | O_EXCL)` makes the check and the creation one atomic operation against
+ # other opens of the same name, the documented lock-file idiom, on Unix and Windows alike.
+ # `fcntl.flock` is deliberately NOT used: `fcntl` does not exist on Windows, and external
+ # testers run this there.
+ # TAKEN BEFORE THIS POINT WHEN THE CALLER HAS ONE, AND THAT IS THE NORMAL PATH: `run()` holds
+ # the directory from its first millisecond, because by the time the payload is opened a
+ # duplicate has already archived the live payload and installed two Unsloth instances. A
+ # `Recorder` built without one still takes its own, so the guard cannot be switched off by
+ # forgetting to pass it.
+ # See `OutDirLock`.
# WHO OWNS THE LOCK IS RECORDED HERE, because it decides who may let go of it. A lock this
- # `Recorder` took is this `Recorder`'s to release when it closes. A lock ADOPTED from the
- # caller outlives the recording -- `run()` closes the recorder and then reads the payload
- # back to render `ab.md` -- so releasing it in `close` would free the directory for the
- # length of the report. See `close`.
+ # `Recorder` took is its to release on close; a lock ADOPTED from the caller outlives the
+ # recording, since `run()` closes the recorder and then reads the payload back to render
+ # `ab.md`, so releasing it in `close` would free the directory for the length of the report.
self._owns_lock = lock is None
if lock is None:
lock = OutDirLock.take(self.path.parent, session_id)
@@ -693,8 +651,7 @@ class Recorder:
row.setdefault("schema", SCHEMA)
row.setdefault("ts_ms", self.now_ms())
row.setdefault("session_id", self.session_id)
- # default = str so a stray Path or dataclass degrades to a string instead of losing the
- # whole row, and the run keeps going.
+ # default = str so a stray Path or dataclass degrades to a string instead of losing the whole row.
self._fh.write(json.dumps(row, default = str) + "\n")
self._fh.flush()
try:
@@ -752,20 +709,13 @@ class Recorder:
except OSError:
pass
# RELEASED, NOT DELETED, and ONLY IF THIS RECORDER TOOK THE LOCK ITSELF.
- #
- # AN ADOPTED LOCK IS NOT THIS OBJECT'S TO DROP. `run()` takes the output directory in its
- # first millisecond and hands it here, then closes the recorder in the `finally` under the
- # cells and goes on to READ THE PAYLOAD BACK -- `_render_ab` and `_summarise` both open
- # `payload.jsonl` after that `finally` and before `run()`'s own outer one. Releasing the
- # adopted lock here freed the directory for exactly that window, and a second invocation
- # arriving in it was admitted: its `prepare_payload` renames the finished run's
- # `payload.jsonl` to `payload-<stamp>.jsonl` before it clones anything, so the reporting
- # step either fails with `FileNotFoundError` on a run whose cells all completed, or -- if
- # the contender has got as far as opening a payload of its own -- reads THAT file and
- # writes an `ab.md` describing another run's rows while still exiting 0.
- #
- # A `Recorder` that took its own lock -- the tests, and any caller that only wants a
- # payload -- has nobody else to release it, so it still does so here.
+ # AN ADOPTED LOCK IS NOT THIS OBJECT'S TO DROP. `run()` hands the directory here, then closes
+ # the recorder in the `finally` and goes on to READ THE PAYLOAD BACK. Releasing the adopted
+ # lock here freed the directory for that window, and a second invocation arriving in it renames
+ # the finished `payload.jsonl` before cloning, so reporting either fails with
+ # `FileNotFoundError` on a run whose cells all completed or writes an `ab.md` describing
+ # another run's rows while exiting 0.
+ # A `Recorder` that took its own lock has nobody else to release it, so it still does so here.
lock = getattr(self, "_lock", None)
if lock is not None and getattr(self, "_owns_lock", True):
lock.release()
diff --git a/tests/studio/studiobench/scene/actions.py b/tests/studio/studiobench/scene/actions.py
index b192b154989..70b0135e7b8 100644
--- a/tests/studio/studiobench/scene/actions.py
+++ b/tests/studio/studiobench/scene/actions.py
@@ -36,24 +36,10 @@ from . import register_action
SETTLE_TIMEOUT_MS = 8000
-#: How long an action that needs the message ACTION BAR waits for it before reporting NOT RUN.
-#:
-#: The bar is mounted `hideWhenRunning`, so Copy, Delete and More do not exist while the thread is
-#: generating. Every film opens these slots after a `send_turn`, spaced by the NOMINAL drain of the
-#: follow-up turn -- FOLLOW_UP_CHARS over the field cadence, 4.59 s. The nominal drain is a floor
-#: rather than an estimate: it assumes the pacer is the binding constraint, and at the 100K rung it
-#: is not. Measured over six 100K cells here, the follow-up streamed for 4.4 to 4.7 s AFTER the
-#: send window closed, so the reply settles within a few hundred milliseconds of the slot on a
-#: quiet machine and a little after it on a loaded one. The CI payload of the run that failed the
-#: liveness gate shows exactly that: one more SSE chunk arrived INSIDE the `message_menu` window
-#: and the reply stopped growing inside it too.
-#:
-#: 1,500 ms, and deliberately NOT `ctx.budget_ms`. The remaining budget at entry is whatever the
-#: previous window's teardown left over -- measured at 51 ms on one of those cells -- so a wait
-#: capped by it is a wait that is not taken in exactly the conditions that need it. It is bounded
-#: so a control that is genuinely absent still reports NOT RUN rather than eating the film: an
-#: action that overruns pushes nothing (every slot has an absolute start) and the overrun is
-#: recorded as `over_budget_ms` on the row.
+#: How long an action needing the message ACTION BAR waits before reporting NOT RUN: 1,500 ms,
+#: deliberately NOT ctx.budget_ms (51 ms left at entry on one measured cell), and bounded so a
+#: genuinely absent control reports NOT RUN rather than eating the film. The wait happens BEFORE
+#: the operation the action times, so `menu_open_ms` is unaffected.
ACTION_BAR_WAIT_MS = 1500
@@ -100,13 +86,10 @@ def _failed(raw: Any) -> str | None:
return None
-# ── the paint floor ─────────────────────────────────────────────────
-#
-# Two rAFs resolve no sooner than two vsync intervals, so ANY timing clocked across a double rAF
-# has a ~33ms floor under it on a 60Hz display. An action that never happened still reports that
-# floor, which reads as a plausible measurement rather than as a failure. Measured per cell and
-# recorded so a reader can subtract it.
+# Two rAFs resolve no sooner than two vsync intervals, so any double-rAF timing has a ~33ms floor
+# on a 60Hz display; measured per cell and recorded so a reader can subtract it.
+# ── the paint floor ─────────────────────────────────────────────────
PAINT_FLOOR_JS = """
async (samples) => {
const values = [];
@@ -129,13 +112,10 @@ def paint_floor_ms(page, samples: int = 9) -> float | None:
return None
+#: The bound that stops a wedged renderer from eating the slot; the wait itself ENDS when nothing
+#: is in flight, and reaching this bound means a lost sample the coverage check fails on.
# ── 1. keystroke to paint ───────────────────────────────────────────
-
-#: How long the keystroke drain will wait for the last paint before giving up on it. Not a "wait
-#: a bit longer" constant: the wait ENDS when nothing is in flight, and this is only the bound that
-#: stops a wedged renderer from eating the slot. Reaching it means a sample was lost, which the
-#: coverage check then fails on rather than quietly publishing the rest.
KEYSTROKE_SETTLE_TIMEOUT_MS = 3000
KEYSTROKE_SETTLE_POLL_MS = 25
@@ -188,9 +168,8 @@ def keystroke(ctx: ActionContext) -> ActionResult:
ctx.page.click(selector)
started = time.monotonic()
- # A real delay between characters. Typing with delay=0 sends the whole burst in one CDP
- # message, which the renderer coalesces into a single input event and a single paint, so the
- # measurement collapses to one sample no matter how many characters were sent.
+ # A real inter-character delay: delay=0 sends the burst in one CDP message, which the renderer
+ # coalesces into a single input event and a single paint.
ctx.page.keyboard.type("a" * count, delay = 60)
_settle_keystrokes(ctx, inst)
got = inst.collect(count)
@@ -210,23 +189,10 @@ def keystroke(ctx: ActionContext) -> ActionResult:
"composer_grew_by": grew,
"composer_text_length": got.get("text_length"),
}
- # EVERY KEYSTROKE ACCOUNTED FOR, not merely a composer that grew.
- #
- # `grew_by` alone was the whole check, and it is satisfied by a textarea whose value is right
- # while the timings describe a subset of the typing. Two ways that happened, and the first
- # INVERTS the metric: a keystroke whose paint had not resolved when the drain ran was simply
- # absent, and the keystroke that has not painted yet is the slowest one -- measured here, a
- # 500 ms keystroke vanished from a reading whose max was 20 ms, on the highest-weight metric in
- # the table. A build that made typing worse read faster, and that reading feeds the null
- # control's noise floor, so it would also tighten the floor every later comparison is judged
- # against. The other way is quieter: a keystroke that never reached the instrument at all is
- # missing from both `samples` and `coalesced`.
- #
- # So the reading stands only when nothing was left in flight and `samples + coalesced` covers
- # every input the instrument saw, and every input the driver commanded arrived. A LOW sample
- # count is not itself a failure -- on a jammed page most keystrokes coalesce behind a slow
- # paint, and that is the finding, not a fault -- which is why this counts coverage rather than
- # demanding a sample per character.
+ # EVERY KEYSTROKE ACCOUNTED FOR, not merely a composer that grew: `grew_by` alone is satisfied
+ # while the timings describe a subset, and the keystroke whose paint had not resolved is the
+ # slowest one (a 500 ms keystroke vanished from a reading whose max was 20 ms). So the reading
+ # stands only when nothing was in flight and samples + coalesced covers every input.
covered = (
seen is not None
and seen >= count
@@ -322,17 +288,14 @@ def _scroll(ctx: ActionContext, label: str) -> ActionResult:
steps = int(ctx.args.get("steps", 14))
step_px = int(ctx.args.get("step_px", 420))
settle_ms = int(ctx.args.get("settle_ms", 200))
- # THE FOLLOW SAMPLER IS SUSPENDED FOR THE DURATION OF THIS GESTURE. This action drags the
- # viewport thousands of pixels off the bottom on purpose, twice, while the reply is still
- # streaming. Samples taken during it record a thread that is not pinned, which is true and is
- # the benchmark's own doing, and averaging them in makes `follows_the_stream` a reading about
- # the film rather than about the app. See the sampler in scene/dom.js.
+ # THE FOLLOW SAMPLER IS SUSPENDED FOR THIS GESTURE: it drags the viewport thousands of pixels off
+ # the bottom while the reply streams, so samples taken during it make `follows_the_stream` a
+ # reading about the film.
_ev(ctx, "() => window.__sb.follow && window.__sb.follow.suspend()")
try:
raw = _ev(ctx, SCROLL_JS, [steps, step_px, settle_ms])
finally:
- # Resumed even when the gesture raised, or one failed scroll silences the sampler for the
- # rest of the cell and the arm reports a follow fraction built from nothing.
+ # Resumed even when the gesture raised, or one failed scroll silences the sampler for the rest of the cell.
_ev(ctx, "() => window.__sb.follow && window.__sb.follow.resume()")
err = _failed(raw)
if err:
@@ -341,10 +304,8 @@ def _scroll(ctx: ActionContext, label: str) -> ActionResult:
return not_run(raw.get("reason", "the scroll did not run"))
commanded = raw["commandedPx"]
travelled = raw["travelledPx"]
- # 90% of commanded. Unsloth replaces assistant-ui's autoscroll with an intent-aware one that
- # snaps a move it reads as programmatic straight back to the bottom: measured on an earlier
- # harness, the gesture landed where it started and the column timed a scroll that did not
- # happen. Travel is the only thing that separates the two.
+ # 90% of commanded: Unsloth's intent-aware autoscroll snaps a move it reads as programmatic
+ # straight back to the bottom, so travel is the only thing separating a real scroll from none.
ok = commanded > 0 and travelled >= 0.9 * commanded
return ActionResult(
ran = True,
@@ -380,27 +341,13 @@ def scroll_after(ctx: ActionContext) -> ActionResult:
return _scroll(ctx, "after")
+# SETTLE ON THE DOM, NOT ON `data-state`: the attribute flips before the content it reveals has
+# mounted, and the gap depends on the collapse mechanism an A/B is comparing. At 100K, reading
+# `pre span` on the attribute frame gave a 41% phantom reduction across arms where a quiet-DOM
+# read gave the same number on both. A null control cannot see a bias it shares, so both the
+# timing and the census terminate on `quietFrames` unchanged frames.
# ── 4. reasoning expand / collapse ──────────────────────────────────
-# SETTLING ON THE DOM, NOT ON A STATE ATTRIBUTE.
-#
-# `data-state="open"` flips when the Collapsible's state changes. It says nothing about whether the
-# content that state reveals has mounted, and how far apart those two frames are depends on the
-# COLLAPSE MECHANISM -- which is exactly the thing an A/B across a collapse change is comparing. So
-# an instant defined by the attribute means a different thing on each arm.
-#
-# Measured, 100K rung, both arms, same corpus. Reading `pre span` on the frame the attribute
-# settles gives 74,917 on the measured-height arm and 44,075 on the grid-rows arm, a 41% apparent
-# reduction. Reading it once the span count has stopped changing gives 74,250 on BOTH, to the span.
-# The 41% was the grid arm being counted before it had finished mounting.
-#
-# Its null control did not catch this and could not have: the null runs one bundle against itself,
-# so the skew is identical on both sides and cancels exactly. A null cannot see a bias it shares.
-#
-# So both the timing and the census now terminate on a QUIET DOM: the observed quantity has to stop
-# changing for `quietFrames` consecutive animation frames before it is read. If it never goes
-# quiet, nothing is returned for it and the reason is recorded, because silence beats a confident
-# wrong answer.
SETTLE_QUIET_FRAMES = 4
REASONING_JS = """
@@ -577,12 +524,8 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:
if not raw.get("ran"):
return not_run(raw.get("reason", "the reasoning toggle did not run"))
- # THE REASON NAMES THE CLAUSE THAT ACTUALLY FAILED.
- #
- # This used to be built from the pane counts alone, so a run whose real failure was a censored
- # timing printed "16 of 16 panes opened and 0 were still open after collapsing" -- a
- # description of SUCCESS -- under the heading EXPECT FAILED. A reader who trusts the message
- # concludes the assertion is broken rather than that the timing is missing.
+ # THE REASON NAMES THE CLAUSE THAT ACTUALLY FAILED: built from pane counts alone it printed a
+ # description of SUCCESS under EXPECT FAILED when the real failure was a censored timing.
failures: list[str] = []
if raw["openCount"] != raw["panes"]:
failures.append(f"only {raw['openCount']} of {raw['panes']} panes opened")
@@ -594,11 +537,9 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:
failures.append(f"close_ms is censored: {raw.get('closeCensoredReason')}")
ok = not failures
- # A CENSORED TIMING IS ABSENT, NOT ZERO, and it is absent LOUDLY. `_action_timings` drops
- # non-numeric values, so a censored cell silently leaves the pooled metric and the survivors
- # are the fast ones -- survivorship bias wearing a mean. The censoring is therefore recorded
- # as its own field so the scoring layer can refuse to pool a metric that is censored at some
- # rungs and not others.
+ # A CENSORED TIMING IS ABSENT, NOT ZERO, and absent loudly: `_action_timings` drops non-numeric
+ # values, so censoring silently leaves the fast survivors behind. Recorded as its own field so
+ # scoring can refuse to pool a metric censored at some rungs and not others.
timings = {}
if raw["openMs"] is not None:
timings["open_ms"] = raw["openMs"]
@@ -609,21 +550,16 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:
ran = True,
expect_ok = ok,
expect = {
- # SCOPED TO WHAT IS MOUNTED, and always was. `reasoningTriggers()` is a
- # querySelectorAll, so on the shipped build it finds every pane in the thread and on a
- # windowed mount it finds the panes in the window. The assertion below stays
- # self-consistent either way (it opens N and expects N open), so the action still
- # passes -- but its COST silently stops being a function of thread length and becomes
- # a function of window size. That is the arm's point, and it is also the reason this
- # timing is not comparable between a windowed arm and a full one without the pane
- # count beside it.
+ # SCOPED TO WHAT IS MOUNTED: `reasoningTriggers()` is a querySelectorAll, so on a windowed mount
+ # its COST stops being a function of thread length and becomes one of window size. The assertion
+ # stays self-consistent either way, so this timing needs the pane count beside it.
+ # A windowed mount is `isWindowed()` in scene/dom.js.
"panes": raw["panes"],
"panes_scope": "mounted",
"open_after_expand": raw["openCount"],
"open_after_collapse": raw["afterClose"],
- # Present ONLY when the span census went quiet. See the note above REASONING_JS: read
- # on the frame the state attribute flips, this number was 41% wrong on one arm and
- # right on the other, and its null control could not see the difference.
+ # Present ONLY when the span census went quiet; read on the state-flip frame this number was 41%
+ # wrong on one arm and right on the other (see the note above REASONING_JS).
"highlight_spans_while_open": raw["spansOpen"],
"highlight_spans_while_open_reason": raw.get("spansOpenReason"),
"settled": raw["spansOpen"] is not None,
@@ -641,14 +577,10 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:
)
-#: THE SAME GESTURE A USER MAKES. `reasoning_toggle` above opens every pane in the thread at once,
-#: which is a deliberate worst case and reads at 2.2 fps at the 100K rung. That number has been
-#: quoted as though it described opening a reasoning pane, and it does not: a user opens ONE, and
-#: almost always the newest one. This action measures that, so the two can be quoted apart.
-#:
-#: It is NOT in the standard film. The scene is a fixed-duration slot schedule, so adding a slot
-#: shifts every window after it and voids comparability against every payload already on disk. Run
-#: it in a purpose-built film until the next corpus or tier bump invalidates those payloads anyway.
+#: THE SAME GESTURE A USER MAKES: `reasoning_toggle` opens every pane at once (2.2 fps at 100K), a
+#: deliberate worst case that has been quoted as though a user did it. NOT in the standard film:
+#: adding a slot shifts every window after it and voids comparability with every payload on disk.
+# The fast film opens this slot 0.4 s after the worst-case drain.
REASONING_ONE_JS = """
async (timeoutMs) => {
const D = window.__sb.dom;
@@ -710,25 +642,19 @@ def reasoning_toggle_one(ctx: ActionContext) -> ActionResult:
ran = True,
expect_ok = ok,
expect = {
- # SCOPED TO WHAT IS MOUNTED, and always was. `reasoningTriggers()` is a
- # querySelectorAll, so on the shipped build it finds every pane in the thread and on a
- # windowed mount it finds the panes in the window. The assertion below stays
- # self-consistent either way (it opens N and expects N open), so the action still
- # passes -- but its COST silently stops being a function of thread length and becomes
- # a function of window size. That is the arm's point, and it is also the reason this
- # timing is not comparable between a windowed arm and a full one without the pane
- # count beside it.
+ # SCOPED TO WHAT IS MOUNTED: `reasoningTriggers()` is a querySelectorAll, so on a windowed mount
+ # its COST becomes a function of window size rather than thread length, and this timing is not
+ # comparable across arms without the pane count beside it.
"panes": raw["panes"],
"panes_scope": "mounted",
- # `panes_opened` is the assertion the two above are context for: this action opens
- # exactly one pane whatever the thread holds, which is the whole point of it existing
- # beside `reasoning_toggle`. Asserted by test_studiobench_reasoning_one_live.
+ # `panes_opened` is the assertion the notes above are context for: this action opens exactly one
+ # pane whatever the thread holds. Asserted by test_studiobench_reasoning_one_live.
"panes_opened": 1,
"open_after_expand": raw["openCount"],
"open_after_collapse": raw["afterClose"],
"highlight_spans_while_open": raw["spansOpen"],
- # The cost driver, so a reading can be normalised rather than compared across threads
- # whose newest reply happens to differ in size.
+ # The cost driver, so a reading can be normalised rather than compared across threads whose
+ # newest reply differs in size.
"highlight_spans_added": raw["spansAdded"],
},
timings = {"open_ms": raw["openMs"], "close_ms": raw["closeMs"]},
@@ -739,126 +665,50 @@ def reasoning_toggle_one(ctx: ActionContext) -> ActionResult:
)
+#: Fixed waits the throwaway turn costs once it starts: 80 ms for the composer, 600 ms to get the
+#: turn going, 400 ms for in-flight chunks, 200 ms for the cleanup delete. RESERVED out of the
+#: drain wait, or the overrun lands in `scroll_after`'s window as a bogus `slot_missed`. Split,
+#: because the 80 ms is spent before the turn is sent.
+# `scroll_after` has a 1,200 ms window on the fast film.
# ── 5. stop generation ──────────────────────────────────────────────
-#: What `stop_generation`'s throwaway turn costs in FIXED waits once it starts, on any machine: 80
-#: ms for the composer to take the text, 600 ms to let the new turn get going so stop is measured
-#: against a live stream rather than a starting one, 400 ms for the chunks already in flight to
-#: land, and 200 ms for the cleanup delete to settle. Polling for the turn to start and for the
-#: stream to stop is on top of it, so this is a floor.
-#:
-#: RESERVED OUT OF THE DRAIN WAIT rather than spent past the deadline. The drain wait below is
-#: bounded by the slot's remaining budget, so a reply that finishes at the last moment of the slot
-#: used to leave nothing at all for the 1.28 s that follows. `stop_generation` closes 300 ms before
-#: `scroll_after` opens on the fast film and 500 ms before it on the quick one, and the runner is
-#: sequential -- every slot has an absolute start, but nothing enters it until the previous action
-#: returns. So the overrun came out of `scroll_after`'s own 1,200 ms window on the fast film and
-#: recorded `slot_missed` there, whose reason blames the machine for reaching the slot late when
-#: what happened is that this action was still cleaning up its scaffolding.
-#:
-#: SPLIT, because the two halves are reserved at different moments. The 80 ms that settles the fill
-#: is spent BEFORE the turn is sent, so it is already gone by the time the turn-start wait is
-#: bounded and a bound that subtracts the whole figure charges it twice.
OWN_TURN_FIXED_AFTER_SEND_MS = 600 + 400 + 200
OWN_TURN_FIXED_MS = 80 + OWN_TURN_FIXED_AFTER_SEND_MS
-#: What the SAME throwaway turn costs BEYOND those sleeps: two polls and the driver round trips
-#: underneath them, neither of which is free.
-#:
-#: MEASURED, because the first version of this reserve was not. It was sized from a page shim whose
-#: `evaluate` is a Python call and whose clock only moves when `wait_for_timeout` is called, so the
-#: waits it recorded were `[100 x29, 80, 600, 400, 200]` and the two polls cost nothing. Driving
-#: the same shipped action against real chromium and a real clock, with a page standing in for the
-#: five calls it makes, the stretch after the drain costs:
-#:
-#: a page that answers instantly 1,394 - 1,451 ms (13 - 17% over the fixed 1,280)
-#: 120 ms to start, 90 ms to stop, 60 ms
-#: to delete -- an unremarkable local app 1,723 - 1,938 ms (35 - 51% over)
-#:
-#: The gap is nine to thirteen CDP round trips (43 - 130 ms of pure driver time), the 50 ms
-#: granularity of both poll loops, and the app's own latency in answering them. Against a 1,280 ms
-#: reserve that put the action 506 - 516 ms past the end of its slot on the fast and quick films,
-#: whose gaps before `scroll_after` opens are 300 ms and 500 ms, so `scroll_after` recorded
-#: `slot_missed` and its reason blamed the machine.
-#:
-#: 700 ms covers the measured 443 - 658 ms with margin and still leaves every film a real drain
-#: wait: the smallest stop slot is 3,000 ms, which keeps 1,020 ms of waiting after the reserve,
-#: and the worst-case drain overhang the fast film has to absorb is 400 ms.
+#: What the same turn costs BEYOND those sleeps: two polls plus the driver round trips. Measured
+#: against real chromium, 1,394-1,938 ms against the fixed 1,280, so 700 ms covers the measured
+#: 443-658 ms and still leaves every film a real drain wait.
#:
-#: SPLIT AT THE MOMENT THE TURN STARTS, because that is where the wait below is bounded and a total
-#: is not a bound. The stop-settle poll, the cleanup delete's own round trips and the driver calls
-#: between them are ALL still ahead at that point, and a bound that reserves only the fixed sleeps
-#: leaves the action exactly `OWN_TURN_FIXED_MS` for a stretch that costs more than that. Measured
-#: against real chromium on the same page as the totals above, with the turn starting on the last
-#: millisecond the bound allows and the fast film's 3,000 ms stop slot:
-#:
-#: a page that answers instantly 60 ms after the sleeps slot + 0.0 s
-#: 90 ms to stop, 60 ms to delete 227 ms after the sleeps slot + 0.15 s
-#: 200 ms to stop, 150 ms to delete 424 ms after the sleeps slot + 0.34 s
-#:
-#: The fast film leaves 300 ms between this slot's deadline and `scroll_after`, so the last row
-#: overran into `scroll_after`'s own window and was recorded there as `slot_missed` -- the same
-#: symptom, one action later, that the reserve above was written to remove. 500 ms covers the
-#: measured 60 - 424 ms; the 200 ms left over is the start poll's own share, which is what the
-#: turn-start wait is allowed to spend.
+#: SPLIT AT THE MOMENT THE TURN STARTS, because that is where the wait below is bounded and a
+#: total is not a bound: the stop-settle poll, the cleanup delete and the driver calls between
+#: them are all still ahead. Measured at 60-424 ms after the sleeps; 500 ms covers it.
+# 90 ms to stop, 60 ms to delete, 227 ms after the sleeps; `STOP_CLEANUP_JS` is unbounded by the slot.
+# Against the fast film's 3,000 ms stop slot.
OWN_TURN_STOP_POLL_MS = 500
OWN_TURN_START_POLL_MS = 200
OWN_TURN_POLL_MS = OWN_TURN_START_POLL_MS + OWN_TURN_STOP_POLL_MS
-#: What the throwaway turn needs in the slot, in total.
-#:
-#: RESERVED OUT OF THE DRAIN WAIT rather than spent past the deadline. The drain wait below is
-#: bounded by the slot's remaining budget, so a reply that finishes at the last moment of the slot
-#: used to leave nothing at all for the ~1.7 s that follows. `stop_generation` closes 300 ms before
-#: `scroll_after` opens on the fast film and 500 ms before it on the quick one, and the runner is
-#: sequential -- every slot has an absolute start, but nothing enters it until the previous action
-#: returns. So the overrun came out of `scroll_after`'s own 1,200 ms window on the fast film and
-#: recorded `slot_missed` there, whose reason blames the machine for reaching the slot late when
-#: what happened is that this action was still cleaning up its scaffolding.
-#:
-#: A RESERVE IS NOT ENOUGH ON ITS OWN, which is why `stop_generation` also re-reads the clock
-#: before it commits. The drain loop tests its deadline at the TOP, so one iteration -- a 100 ms
-#: wait plus a round trip -- lands past it, and no constant chosen here can pay for time that has
-#: already been spent. The reserve decides how much of the slot the drain may have; the check
-#: decides whether what is actually left is enough to start a turn that cannot be abandoned once
-#: it is running.
+#: What the throwaway turn needs in the slot in total, reserved out of the drain wait rather than
+#: spent past the deadline. A reserve is not enough on its own: the drain loop tests its deadline
+#: at the TOP, so `stop_generation` re-reads the clock before committing to a turn.
+#: The fixed waits are 80 ms for the composer to take the text, 600 ms to let the turn get
+#: going, 400 ms for the chunks in flight and 200 ms for the cleanup delete.
+# One iteration is a 100 ms tick.
OWN_TURN_RESERVE_MS = OWN_TURN_FIXED_MS + OWN_TURN_POLL_MS
-#: How long to wait for the throwaway turn to start, in total. The wait for a turn this action will
-#: still MEASURE is bounded by the slot on top of this -- 8 s is 2.7x the whole budget of the fast
-#: film's stop slot, so a turn that was slow to start used to overrun by seconds whatever had been
-#: reserved for it -- but the turn does not stop existing when the slot runs out.
-#:
-#: ENTER IS PRESSED BEFORE THIS WAIT, which is what an earlier version of this comment had wrong
-#: when it called cutting the wait short free. Something IS committed by then: the send is
-#: accepted, the user turn is in the thread, and the reply starts whenever the relay gets round to
-#: it. Returning at the slot bound left exactly the scaffolding the paragraph below refuses to
-#: leave -- measured in chromium on a page that took 1,800 ms to start against the fast film's
-#: 3,000 ms slot, the action returned `not_run` after 1,759 ms and the thread it handed on had two
-#: extra messages in it and a live stream running through the next action's window.
-#:
-#: So the slot bounds how long the turn is WORTH MEASURING, and this bounds how long it is worth
-#: waiting for so it can be taken back: `_reclaim_pending_turn` polls on to this deadline, stops
-#: the turn if it ever becomes stoppable and deletes it, then reports `not_run`. The total spent
-#: waiting for a start is therefore what it was before the slot bound existed, and the slot bound
-#: still does its job on every run where the turn arrives.
-#:
-#: The stop-settle poll and `STOP_CLEANUP_JS` are deliberately NOT bounded by the slot at all: once
-#: Enter is pressed the turn has to be stopped and removed or the thread every later action
-#: measures keeps the scaffolding, and a settle poll cut short would report a stop that worked as
-#: `still_running`.
+#: How long to wait for the throwaway turn to start. Enter is pressed BEFORE this wait, so the
+#: send is already committed: returning at the slot bound left two extra messages and a live
+#: stream running into the next action's window. The slot bounds how long the turn is worth
+#: MEASURING; this bounds how long it is worth waiting for so it can be stopped and deleted.
+# The turn stays in the thread `still_running`.
TURN_START_TIMEOUT_MS = 8000
-#: The composer text the throwaway turn is sent with. Read back as well as written: a composer that
-#: still holds it is a send the app REFUSED, and that is the one case on the timeout path where
-#: nothing was committed and there is nothing to take back.
+#: The composer text the throwaway turn is sent with, read back as well as written: a composer
+#: that still holds it is a send the app REFUSED, the one case where nothing was committed.
OWN_TURN_TEXT = "one more"
-#: Remove the throwaway turn `stop_generation` created, so the thread it leaves behind is the
-#: thread it found. Deletes the assistant turn and then the user turn that prompted it, in that
-#: order, because deleting the user message first can take the reply with it and leave the count
-#: ambiguous. Reports what it managed rather than asserting: a cleanup that half-worked must be
-#: visible in the row, not swallowed.
+#: Remove the throwaway turn: assistant first, then the user turn, because deleting the user
+#: message can take the reply with it and leave the count ambiguous. Reports rather than asserts.
STOP_CLEANUP_JS = """
async (timeoutMs) => {
const D = window.__sb.dom;
@@ -968,9 +818,8 @@ def _reclaim_pending_turn(
if removed is not None:
ctx.page.wait_for_timeout(200)
- # BOTH HALVES, REPORTED SEPARATELY. A turn that was deleted but never stopped and one that was
- # stopped but not deleted leave the film in different states, and a row that says only "cleaned
- # up" cannot be read in the light of either.
+ # BOTH HALVES, REPORTED SEPARATELY: deleted-but-never-stopped and stopped-but-not-deleted leave
+ # the film in different states.
if running is not True:
state = f"never started within the {TURN_START_TIMEOUT_MS}ms it was worth waiting for"
elif stopped:
@@ -997,9 +846,8 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
Queue control at the same position with the same class. Pressing it queues a message and the
stream carries on, and the action reports a fast, precise, entirely wrong number.
"""
- # THE SLOT, ON THE CLOCK THE ACTION ITSELF RUNS ON. Everything below that is bounded is
- # bounded against this rather than against a constant, because a budget spent is not a budget
- # available: `ctx.budget_ms` is what the slot HAD when the runner entered it.
+ # THE SLOT, ON THE CLOCK THE ACTION ITSELF RUNS ON: `ctx.budget_ms` is what the slot HAD when the
+ # runner entered it, and a budget spent is not a budget available.
slot_deadline = time.monotonic() + ctx.budget_ms / 1000.0
def remaining_ms() -> float:
@@ -1010,39 +858,13 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
ctx.page.fill('textarea[aria-label="Message input"]', "")
ctx.page.wait_for_timeout(120)
- # STOP GETS ITS OWN GENERATION. It used to stop whatever the cell was streaming, which
- # permanently truncated the measured reply: at the 10K rung the pacer got 5,304 of 17,737
- # characters away before the socket broke, so the ten actions that follow -- all of them
- # labelled "after the reply is complete" -- and the final census ran against a thread barely
- # a third of the size the rung claims. The seeded-vs-streamed equivalence check then compared
- # that truncated reply against a complete seeded one and reported 20% drift, which read as a
- # finding about seeding and was a finding about this action.
- #
- # Sending its own short turn keeps the measured reply intact and means the action has
- # something to stop at EVERY rung, instead of reporting `not_run` at the small ones where the
- # main stream is already over by the time the slot opens.
- #
- # THE GUARD BELOW IS THE OTHER HALF OF THAT, and without it the paragraph above only held for
- # the default fixture. The own-turn path was entered only when `isRunning()` was FALSE, so the
- # moment anything WAS running this action fell straight through and clicked Stop on it -- the
- # exact behaviour the paragraph above says was removed. The supported way to make that happen
- # is `--stream-tail-chars`, whose entire purpose is a long opening reply: at 96,000 characters
- # the reply streams for 291 s at field cadence against a 243 s standard film, so this slot
- # opens at 28 s with the opening turn still in flight and kills it at about 9,200 characters.
- # The reply-length axis the flag exists to provide is then never exercised, every later action
- # still runs against a settled thread, `ran` is still true, and `--assert-liveness` -- which
- # the flag's own help text sends the caller to -- still passes. A silently truncated fixture
- # reported as a clean run is the most expensive failure shape this harness has.
- #
- # So the reply is given the rest of this slot's budget LESS what the throwaway turn then costs
- # (`OWN_TURN_RESERVE_MS`) to finish on its own, which is all a marginally slow drain needs (the
- # fast film opens this slot 0.4 s after the worst-case drain on the ladder, and 1.0 s of the
- # fast film's 3 s budget is still left over for it). If it has not finished by then, NOTHING IS
- # STOPPED and the row says why. An honest `not_run` costs a column; stopping the reply costs
- # the whole cell, and spending the rest of the slot draining and THEN starting a turn of our
- # own costs the next slot. With the default tail nothing is ever running when this slot opens
- # -- the packing test in fixture/selftest holds every film to that -- so this path is
- # unreachable on an unmodified run and the behaviour there is exactly what it was.
+ # STOP GETS ITS OWN GENERATION. Stopping whatever the cell was streaming permanently truncated
+ # the measured reply (5,304 of 17,737 characters at 10K), so every later action ran against a
+ # third-sized thread. THE GUARD BELOW IS THE OTHER HALF: entering the own-turn path only when
+ # `isRunning()` was false let `--stream-tail-chars` kill the opening reply while liveness passed.
+ # At 96,000 characters the slot opens at 28 s and kills it at about 9,200.
+ # So the reply gets this slot's budget less `OWN_TURN_RESERVE_MS`, and if it has not finished
+ # NOTHING IS STOPPED and the row says why.
if _ev(ctx, "() => window.__sb.dom.isRunning()"):
drain_ms = max(0.0, ctx.budget_ms - OWN_TURN_RESERVE_MS)
settle_deadline = time.monotonic() + drain_ms / 1000.0
@@ -1061,13 +883,9 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
"and the final census measure, so nothing was stopped. Lower --stream-tail-chars "
"or move this slot past the drain"
)
- # AND THE RESERVE IS CHECKED AGAINST THE CLOCK, not assumed off the budget. The loop above
- # tests `settle_deadline` at the top, so the iteration that finds the reply drained has
- # already spent a 100 ms wait and a round trip past it; a drain that lands in the last
- # iteration therefore leaves LESS than the reserve however the reserve is sized. Starting
- # the turn anyway is the same overrun by a smaller amount, and the turn cannot be
- # abandoned halfway -- once Enter is pressed the thread has to be stopped and cleaned up
- # or every later action measures the scaffolding.
+ # AND THE RESERVE IS CHECKED AGAINST THE CLOCK: the loop tests `settle_deadline` at the top, so a
+ # drain landing in the last iteration leaves less than the reserve however it is sized, and the
+ # turn cannot be abandoned once Enter is pressed.
if remaining_ms() < OWN_TURN_RESERVE_MS:
return not_run(
"the cell's own reply drained with only "
@@ -1078,31 +896,18 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
"--stream-tail-chars or move this slot past the drain"
)
- # READ BEFORE ENTER, because it is the only way to tell the turn this action added from the
- # thread it was handed. `_reclaim_pending_turn` deletes only if this number grew.
- #
- # threadTotal, not messageCount, for the reason STOP_CLEANUP_JS gives above and in the same
- # direction: under a windowed mount the window refills as the thread grows, so a send that
- # WORKED reports after == before and the reclaim path then decides there is nothing to remove.
- # The turn it already sent is left in the thread with its stream still running, which is the
- # one outcome this whole path exists to prevent, and it contaminates every later action window
- # and the final census. Identical on the shipped build, where the two are the same number.
+ # READ BEFORE ENTER: the only way to tell the turn this action added from the thread it was
+ # handed, and `_reclaim_pending_turn` deletes only if it grew. threadTotal, not messageCount:
+ # under a windowed mount a send that WORKED reports after == before.
messages_before = _ev(ctx, "() => window.__sb.dom.threadTotal()")
ctx.page.fill('textarea[aria-label="Message input"]', OWN_TURN_TEXT)
ctx.page.wait_for_timeout(80)
ctx.page.keyboard.press("Enter")
own_generation = True
sent_at = time.monotonic()
- # HOW LONG THE TURN IS WORTH MEASURING. Bounded by the SLOT as well as by
- # `TURN_START_TIMEOUT_MS`: 8 s is 2.7x the whole stop slot on the fast and quick films, so a
- # turn slow to start overran by seconds no matter what had been reserved for it.
- #
- # WHAT IS RESERVED OUT OF IT is the rest of the turn, not only its sleeps. A turn starting on
- # the last millisecond this allows still has the stop-settle poll, the delete and the driver
- # calls between them ahead of it, and reserving `OWN_TURN_FIXED_MS` left 424 ms of that unpaid
- # against real chromium -- 344 ms past a 3,000 ms slot with 300 ms before `scroll_after` opens,
- # recorded there as a missed slot. `OWN_TURN_FIXED_MS` also counts the 80 ms above, which is
- # already spent. See `OWN_TURN_STOP_POLL_MS`.
+ # HOW LONG THE TURN IS WORTH MEASURING, bounded by the SLOT as well as by `TURN_START_TIMEOUT_MS`
+ # (8 s is 2.7x the whole stop slot). What is reserved out of it is the rest of the turn, not only
+ # its sleeps: reserving `OWN_TURN_FIXED_MS` left 424 ms unpaid against real chromium.
start_wait_ms = max(
0.0,
min(
@@ -1111,8 +916,7 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
),
)
deadline = time.monotonic() + start_wait_ms / 1000.0
- # HOW LONG IT IS WORTH WAITING FOR, which is a different question and is NOT bounded by the
- # slot. See `_reclaim_pending_turn`.
+ # HOW LONG IT IS WORTH WAITING FOR, a different question, and NOT bounded by the slot. See `_reclaim_pending_turn`.
reclaim_deadline = sent_at + TURN_START_TIMEOUT_MS / 1000.0
started_late = (
f"nothing was generating and a new turn did not start within {start_wait_ms:.0f}ms"
@@ -1148,11 +952,9 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
chars_after = _ev(ctx, "() => window.__sb.dom.assistantChars()")
ok = stopped_ms is not None
- # LEAVE THE THREAD AS WE FOUND IT. The throwaway turn is scaffolding, not content: left in
- # place it adds an assistant message and a reasoning pane that the rest of the film, the final
- # census and the seeded-versus-streamed comparison all then measure. That showed up
- # immediately as "streamed 5 assistant messages vs seeded 4", a drift introduced entirely by
- # this action while it was busy fixing a different one.
+ # LEAVE THE THREAD AS WE FOUND IT: left in place the throwaway turn adds an assistant message and
+ # a reasoning pane that the rest of the film, the census and the seeded-versus-streamed
+ # comparison all measure.
removed = None
if own_generation:
removed = _ev(ctx, STOP_CLEANUP_JS, SETTLE_TIMEOUT_MS)
@@ -1164,16 +966,15 @@ def stop_generation(ctx: ActionContext) -> ActionResult:
"chars_before": chars_before,
"chars_after": chars_after,
"still_running": not ok,
- # Which reply was stopped. A reader comparing `chars_added_after_stop` across
- # rungs needs to know whether this was a throwaway turn or the cell's own.
+ # Which reply was stopped: `chars_added_after_stop` reads differently for a throwaway turn than
+ # for the cell's own.
"own_generation": own_generation,
- # Whether the scaffolding was removed again. Reported rather than asserted: a
- # cleanup that failed leaves an extra turn in the thread, and every census
- # after this point needs to be readable in that light.
+ # Whether the scaffolding was removed. Reported rather than asserted: a failed cleanup leaves an
+ # extra turn every later census must be read against.
"scaffold_removed": (None if removed is None else bool(removed.get("removed"))),
"scaffold_note": (None if removed is None else removed.get("reason")),
- # A stop that worked leaves the text where it was, give or take the chunks
- # already in flight. A large jump means the stream ran on.
+ # A stop that worked leaves the text where it was, give or take the chunks already in flight; a
+ # large jump means the stream ran on.
"chars_added_after_stop": (
None if chars_after is None or chars_before is None else chars_after - chars_before
),
@@ -1287,8 +1088,8 @@ def model_change(ctx: ActionContext) -> ActionResult:
return not_run("no model selector trigger on the page")
before = _ev(ctx, "() => window.__sb.dom.currentModelLabel()")
started = time.monotonic()
- # Click the LABEL, not the trigger's right edge: a `span[data-eject-hit]` sits there and
- # clicking it ejects the model instead of opening the picker.
+ # Click the LABEL, not the trigger's right edge: a `span[data-eject-hit]` sits there and ejects
+ # the model instead of opening the picker.
trigger.click(position = {"x": 8, "y": 8})
opened_ms = None
deadline = started + SETTLE_TIMEOUT_MS / 1000
@@ -1411,10 +1212,8 @@ def copy_markdown(ctx: ActionContext) -> ActionResult:
triggers a file download rather than a clipboard write. The action bar has `autohide`, so it
is hovered first or the button is not in the tree to click.
"""
- # Hover AND WAIT, on the same reasoning as `message_menu`: this bar is `hideWhenRunning`, so a
- # slot that opens with the follow-up's last chunks still arriving finds no Copy button at all.
- # The fixed 150 ms this replaced was a hover settle, not a wait for the reply, and it returned
- # the same "no Copy button" whether the control was missing or merely not mounted yet.
+ # Hover AND WAIT, on the same reasoning as `message_menu`: the bar is `hideWhenRunning`, so a slot
+ # opening with the follow-up's last chunks still arriving finds no Copy button.
found = _ev(
ctx,
"""
@@ -1451,9 +1250,8 @@ def copy_markdown(ctx: ActionContext) -> ActionResult:
return not_run("no Copy button on the last assistant message")
ctx.page.wait_for_timeout(200)
elapsed = (time.monotonic() - started) * 1000
- # Read back from the clipboard. Headless Chromium grants clipboard-read only with the
- # permission, which the browser factory requests; if it is not granted the action still RAN,
- # and the assertion says it could not be proved rather than that it failed.
+ # Read back from the clipboard: headless Chromium grants clipboard-read only with the permission
+ # the browser factory requests, and without it the action still RAN but could not be proved.
clip = None
reason = None
try:
@@ -1511,14 +1309,9 @@ def select_text(ctx: ActionContext) -> ActionResult:
if not raw.get("ran"):
return not_run(raw.get("reason", "selection did not run"))
visible = raw.get("visibleChars") or 0
- # NON-EMPTY, and the coverage fraction reported as evidence rather than gated on.
- #
- # "Half the visible characters" looked like a reasonable bar and is not one: innerText
- # collapses runs of whitespace and skips nested scrollers, and Selection.toString applies its
- # own normalisation, so the two counts are not the same quantity even when the selection
- # covers the whole message. Measured on a real reply: 22 selected against 80 "visible", with
- # the selection plainly spanning the entire message. The action's claim is that text in this
- # message got selected; the fraction belongs in the evidence, not in the verdict.
+ # NON-EMPTY, with the coverage fraction as evidence rather than a gate: innerText collapses
+ # whitespace and skips nested scrollers while Selection.toString normalises differently, so the
+ # two counts are not the same quantity.
ok = raw["chars"] > 0
return ActionResult(
ran = True,
@@ -1563,43 +1356,17 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:
err = _failed(raw)
if err:
return not_run(err)
- # A SENTINEL ON THE CLIPBOARD BEFORE THE COPY, so "the copy happened" is OBSERVED rather than
- # assumed from the keystroke having been sent.
- #
- # THE FAILURE THIS EXISTS FOR, measured. Playwright's WebKit never performs a clipboard copy on
- # Control+C. The action pressed the key, slept 250ms for the copy to land, and reported the
- # sleep as `copy_ms`. In every WebKit payload on this machine that number is 258.5 ms at the 1K
- # rung, 258.8 ms at 10K and 263.9 ms at 100K: a hundredfold change in the quantity under study
- # moved the "measurement" by two percent, because it was measuring `wait_for_timeout(250)`.
- # Chromium reads about 1,538 ms at 100K for the same action. Forty-three rows across eleven
- # payloads reported a sleep as a result.
- #
- # A measurement that silently reports a sleep is the worst failure mode in this harness, so the
- # test is on the CLIPBOARD and not on the engine name: an engine that starts working, or a new
- # one, is admitted automatically, and an engine that stops working is refused automatically.
- #
- # WHAT THE CHECK NEEDS is a KNOWN PRE-COPY VALUE, and the sentinel is only the best way of
- # getting one. Writing it can fail on its own: `writeText` needs transient user activation or
- # the `clipboard-write` permission and throws `NotAllowedError` without either, while
- # `readText` is gated separately -- so the two are not granted or refused together, and
- # runtime/browser.py asks for the pair on Chromium and for neither on the other engines.
- #
- # Clearing the sentinel on that failure and carrying on is what re-admitted the exact defect
- # above: with no pre-copy value, `clip == sentinel` can never fire, and a Control+C that did
- # nothing leaves the clipboard holding whatever `copy_markdown` put there earlier in the same
- # film -- a plausible, non-empty string that is read back as a fresh copy of the whole thread,
- # with the 250 ms settle beside it as `copy_ms`.
- #
- # So a failed write falls back to SNAPSHOTTING what the clipboard already holds, which answers
- # the same question ("did this keystroke change the clipboard"), and when even that cannot be
- # established the action is NOT RUN. The residual, stated: if the clipboard already held
- # character-for-character what the copy would produce, an honest copy is refused as a
- # no-op. That direction is the safe one, it needs the write permission to be missing first,
- # and the film never copies the same thread twice.
+ # A SENTINEL ON THE CLIPBOARD BEFORE THE COPY, so the copy is OBSERVED rather than assumed from
+ # the keystroke. Playwright's WebKit never copies on Control+C: the action reported its own
+ # 250 ms sleep as `copy_ms` across forty-three rows. The test is on the CLIPBOARD, not the engine
+ # name. A failed `writeText` falls back to SNAPSHOTTING what the clipboard already holds; when
+ # even that fails the action is NOT RUN, because clearing the sentinel re-admits the defect.
+ # Chromium reads about 1,538 ms at 100K for the same action.
+ # Residual: an honest copy of character-identical content reads as a no-op.
sentinel = f"__sb_clipboard_sentinel_{int(time.monotonic() * 1000)}__"
sentinel_written = False
- #: What the clipboard held before Control+C, and where that value came from. `None` means no
- #: pre-copy value could be established at all, which is refused below.
+ #: What the clipboard held before Control+C and where that value came from; None means no
+ #: pre-copy value could be established, which is refused below.
pre_copy: Optional[str] = None
pre_copy_source = "sentinel"
pre_copy_reason = None
@@ -1630,19 +1397,10 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:
ctx.page.keyboard.press("Control+C")
ctx.page.wait_for_timeout(250)
copy_ms = (time.monotonic() - copy_started) * 1000
- # THE CLIPBOARD, NOT THE SELECTION.
- #
- # This action used to score itself on `Selection.toString().length`, and that is not what a
- # user gets. The two came apart the moment an arm existed that unmounts messages AND handles
- # the copy event: the selection can only ever cover mounted nodes, while a copy handler that
- # reads the message store puts the whole conversation on the clipboard. Scored on the
- # selection, a build that had FIXED the data loss would still be reported as losing data --
- # the alarm would stay lit for a reason that no longer existed, which is the fastest way to
- # get an alarm switched off.
- #
- # Read back before the selection is cleared, and the failure to read it is reported rather
- # than treated as an empty clipboard: headless Chromium grants clipboard-read only with the
- # permission the browser factory requests.
+ # THE CLIPBOARD, NOT THE SELECTION: a selection covers only mounted nodes while a copy handler
+ # reading the message store puts the whole conversation on the clipboard, so scoring on the
+ # selection would report a build that FIXED the data loss as still losing it. Read back before
+ # the selection is cleared, and a failed read is reported rather than read as an empty clipboard.
clip = None
clip_reason = None
try:
@@ -1652,9 +1410,9 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:
clipboard_chars = len(clip) if isinstance(clip, str) else None
ctx.page.evaluate("() => window.getSelection().removeAllRanges()")
- # NO CONFIRMED COPY, NO TIMING. Reported NOT RUN rather than as a number, because a reader who
- # sees `copy_ms` has no way to tell a real copy from a keystroke that went nowhere, and the
- # engines that fail this do so silently and consistently enough to look like data.
+ # NO CONFIRMED COPY, NO TIMING: a reader seeing `copy_ms` cannot tell a real copy from a
+ # keystroke that went nowhere, and the engines that fail do so consistently enough to look
+ # like data.
if pre_copy is not None and clip == pre_copy:
held = (
"it still holds the sentinel written before the keystroke"
@@ -1687,22 +1445,16 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:
expect_ok = ok,
expect = {
"selected_chars": raw["chars"],
- # WHAT ACTUALLY REACHED THE CLIPBOARD. The user-facing quantity, and the one the
- # truncation alarm is scored on. A windowed mount cannot SELECT what is not in the
- # DOM, but it can still COPY it, if the app's copy handler reads the message store --
- # which is the fix, and which this is the only reading able to see.
+ # WHAT ACTUALLY REACHED THE CLIPBOARD, the user-facing quantity the truncation alarm is scored
+ # on: a windowed mount cannot SELECT what is not in the DOM but can still COPY it from the store.
"clipboard_chars": clipboard_chars,
"clipboard_readable": clip_reason is None,
"clipboard_note": clip_reason,
- # WHICH pre-copy value the change was confirmed against: the sentinel this action
- # wrote, or a snapshot of what the clipboard already held when the write was refused.
- # A snapshot is the weaker of the two -- see the note above the write -- and a reader
- # comparing rows has no other way to see which one a timing rests on.
+ # WHICH pre-copy value the change was confirmed against: the sentinel this action wrote, or a
+ # snapshot taken when the write was refused (the weaker of the two).
"copy_confirmed_against": pre_copy_source,
- # And the DOM coverage beside it, so the two can be told apart. Where
- # `mounted_fraction` is well below 1 and `clipboard_chars` is nonetheless whole, the
- # copy-from-store path is working; where both are short, the conversation is being
- # lost. That is a user-visible defect and must not be filed as a measurement problem.
+ # The DOM coverage beside it: `mounted_fraction` well below 1 with whole `clipboard_chars` means
+ # the copy-from-store path works; both short means the conversation is being lost.
"messages_total": total,
"messages_mounted": mounted,
"mounted_fraction": (
@@ -1716,15 +1468,9 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:
"copy_ms": round(copy_ms, 1),
"total_ms": round((time.monotonic() - started) * 1000, 1),
},
- # `expect_ok` stays `chars > 0` deliberately. Within ONE run there is nothing to compare
- # against: the selection is taken over the viewport's DOM, so if the DOM is windowed the
- # selection and every DOM-derived reference shrink together and agree with each other.
- # An absolute floor would need calibrating per rung and per platform, and would still be a
- # guess. The counts below are the real check, and they are paired against the other arm.
- #
- # `clipboard_chars` FIRST, because it is the one a user would notice. `selected_chars` is
- # kept beside it as the mechanism: if the clipboard held and the selection fell, the copy
- # handler is reading the store; if both fell, the conversation is being lost.
+ # `expect_ok` stays `chars > 0`: within one run the selection and every DOM-derived reference
+ # shrink together, and an absolute floor would need per-rung, per-platform calibration.
+ # `clipboard_chars` comes first as the user-visible number, `selected_chars` as the mechanism.
counts = {
"clipboard_chars": clipboard_chars,
"selected_chars": raw["chars"],
@@ -1752,10 +1498,9 @@ def send_turn(ctx: ActionContext) -> ActionResult:
"""
pacer = ctx.args.get("_pacer")
queue = ctx.args.get("_stream_queue") or []
- # A SHARED MUTABLE cursor, not a scalar in `args`. The scene runner builds each action's args
- # as `{**base_args, **slot.args}`, so a scalar written back here lands in a dict that is
- # discarded when the action returns: the second send re-sent the first turn, reported
- # `turn_index: 1` twice, and failed because that message was already in the thread.
+ # A SHARED MUTABLE cursor, not a scalar in `args`: the runner rebuilds each action's args, so a
+ # scalar written back is discarded and the second send re-sent the first turn.
+ # Slot args merge as `{**base_args, **slot.args}`.
cursor = ctx.args.get("_stream_cursor")
if not isinstance(cursor, dict):
return not_run("no shared stream cursor was passed to the action")
@@ -1765,21 +1510,16 @@ def send_turn(ctx: ActionContext) -> ActionResult:
if index >= len(queue):
return not_run(f"the stream queue is exhausted ({len(queue)} turns planned)")
if _ev(ctx, "() => window.__sb.dom.isRunning()"):
- # Sending while a reply is in flight queues the message instead of starting a stream, and
- # the action would report a fast, precise number about a message that is merely parked.
+ # Sending while a reply is in flight queues the message instead of starting a stream, and the
+ # action would report a precise number about a message that is merely parked.
return not_run("a reply was still streaming, so this send would have been queued")
unit = queue[index]
cursor["i"] = index + 1
- # NO `pacer.reset()` HERE. It used to clear the pacer's stats before loading the next turn, and
- # `CellRunner` records `last_stats()` -- so the opening reply's `StreamStats` were discarded by
- # the first follow-up and the first follow-up's by the second. An opening reply that
- # disconnected or delivered 4,624 of its 10,000 characters was then erased by a later turn that
- # DID finish, and the cell was marked complete and scored against an undersized thread, because
- # the only other liveness signal is the UI no longer running and the later turn satisfies it.
- # Every turn is tagged, so keeping them all costs one small record each and lets
- # `check_planned_streams` verify the cell streamed what it planned. `CellRunner` still resets
- # once per cell, which is the boundary that reset is for.
+ # NO `pacer.reset()` HERE: `CellRunner` records `last_stats()`, so resetting discarded the
+ # opening reply's StreamStats and let a later completed turn mask a disconnected one. Every turn
+ # is tagged instead, so `check_planned_streams` can verify what the cell streamed.
+ # One turn delivered 4,624 of its 10,000 characters.
tag = f"{ctx.args.get('cell_id', 'cell')}#turn{index + 1}"
pacer.load(
unit["reasoning"],
@@ -1793,10 +1533,8 @@ def send_turn(ctx: ActionContext) -> ActionResult:
return not_run("no composer on the page")
ctx.page.fill(selector, f"studiobench follow-up {index + 1}")
ctx.page.wait_for_timeout(80)
- # THE THREAD'S LENGTH, not the mounted count. "The thread grew" is a statement about the
- # conversation; a windowed mount answers it about the viewport, and the answer it gives is
- # `after == before` for a send that worked perfectly, because the new pair arrives at the
- # bottom while two messages leave the top of the window.
+ # THE THREAD'S LENGTH, not the mounted count: under a windowed mount a send that worked perfectly
+ # reports `after == before`, since the new pair arrives as two messages leave the top.
before = _ev(ctx, "() => window.__sb.dom.threadTotal()")
mounted_before = _ev(ctx, "() => window.__sb.dom.messageCount()")
started = time.monotonic()
@@ -1811,8 +1549,8 @@ def send_turn(ctx: ActionContext) -> ActionResult:
ctx.page.wait_for_timeout(50)
after = _ev(ctx, "() => window.__sb.dom.threadTotal()")
mounted_after = _ev(ctx, "() => window.__sb.dom.messageCount()")
- # A POSITIVE consequence: the turn actually started streaming AND the thread grew. A send that
- # silently failed leaves both unchanged and would otherwise read as an instant send.
+ # A POSITIVE consequence: the turn actually started streaming AND the thread grew, so a silently
+ # failed send cannot read as an instant one.
ok = (
first_ms is not None
and isinstance(after, int)
@@ -1825,16 +1563,15 @@ def send_turn(ctx: ActionContext) -> ActionResult:
expect = {
"messages_before": before,
"messages_after": after,
- # What was MOUNTED either side, kept beside the thread totals. On the shipped build
- # these two pairs are identical; where they differ, the difference is the size of the
- # window and it is the reading the whole virtualization arm exists to produce.
+ # What was MOUNTED either side, kept beside the thread totals: identical on the shipped build,
+ # and where they differ the difference is the window size the virtualization arm measures.
"mounted_before": mounted_before,
"mounted_after": mounted_after,
"turn_index": index + 1,
"queued_turns": len(queue),
"streamed_chars": len(unit["reasoning"]) + len(unit["content"]),
- # The tag this turn's stream carries, so the cell can check the turn against the
- # pacer's own record of it rather than re-deriving the naming rule from the outside.
+ # The tag this turn's stream carries, so the cell checks it against the pacer's own record rather
+ # than re-deriving the naming rule.
"pacer_tag": tag,
"unit_kind": unit.get("kind"),
},
@@ -1843,9 +1580,9 @@ def send_turn(ctx: ActionContext) -> ActionResult:
)
-#: What the page can tell us when the attachments button cannot be found. Reads geometry, style,
-#: hit-testing and the surrounding chrome, so the three explanations that look identical from a
-#: `not_run` string look different here.
+#: What the page can tell us when the attachments button cannot be found: geometry, style,
+#: hit-testing and surrounding chrome, so three explanations that look identical from a `not_run`
+#: string look different here.
IMAGE_BUTTON_DIAGNOSTIC = """() => {
const sel = 'button[aria-label="Tools and attachments"]';
const all = [...document.querySelectorAll(sel)];
@@ -1889,30 +1626,20 @@ def image_upload(ctx: ActionContext) -> ActionResult:
png = ctx.args.get("image_path")
if not png:
return not_run("no image path was supplied to the action")
- # `:visible`, and the FIRST visible match.
- #
- # Two reasons, both measured. `document.querySelector` returns the first match in DOCUMENT
- # ORDER and the composer exists more than once: the welcome-screen instance stays in the tree
- # behind the docked one, and compare mode has two threads. The hidden duplicate comes first,
- # so a plain query handed back a button that can never be clicked and the action reported "not
- # visible" about a control that a direct probe found at 36x36 pixels, fully opaque, on screen.
- #
- # And Playwright's actionability wait does not give up quickly on an unclickable element: at
- # its 30s default it blocked a 9s slot for more than three times the slot's budget, to report
- # that nothing happened. Everything here is bounded by what is left of the slot.
+ # `:visible`, and the FIRST visible match: `document.querySelector` returns document order and
+ # the composer exists more than once (the welcome-screen instance stays in the tree, compare mode
+ # has two threads), so a plain query handed back an unclickable button while a direct probe found
+ # the real one. Playwright's actionability wait also blocks for its 30s default, so everything
+ # here is bounded by what is left of the slot.
locator = ctx.page.locator('button[aria-label="Tools and attachments"]:visible').first
try:
plus = locator.element_handle(timeout = 2000)
except Exception: # noqa: BLE001
plus = None
if plus is None:
- # WHY, not just THAT. A bare "not visible" is the opaque zero of the action layer: it
- # conflates a button that is absent, a button that is present and covered, and a locator
- # that disagrees with the page, and it has already cost three wrong hypotheses. A direct
- # probe found the control at 36x36, fully opaque and hit-testable, on a fresh chat and
- # after a settings round trip and under a 20,000-character composer fill, so whatever
- # this is, it is none of those. Carrying the state into the row means the NEXT run
- # answers it instead of the next investigation.
+ # WHY, not just THAT: a bare "not visible" conflates a button that is absent, one that is covered
+ # and a locator that disagrees with the page, and has already cost three wrong hypotheses.
+ # Carrying the probe state into the row means the next run answers it.
return not_run(
"no visible attachments button on the composer: "
+ json.dumps(_ev(ctx, IMAGE_BUTTON_DIAGNOSTIC) or {})
@@ -1956,20 +1683,12 @@ def image_upload(ctx: ActionContext) -> ActionResult:
)
+#: The end of the thread as a plain string that can be looked for after the rebuild. NOT the
+#: seeder's `last_marker`: `send_turn` appends turns mid-film, so the seeded marker sits several
+#: messages from the end while `end_present` requires it within two rows. `trim()` then a prefix,
+#: never a whitespace-collapsing normalisation, since readiness.PROBE_JS matches RAW text.
# ── 13. thread reopen ───────────────────────────────────────────────
-#: The end of the thread, as a plain string that can be looked for again after the rebuild.
-#:
-#: NOT the seeder's `last_marker`, even though that is the string the readiness gate uses at the
-#: start of the cell. `send_turn` appends turns to the thread MID-FILM, twice in every scene, so by
-#: the time this slot opens the seeded marker sits several messages from the end -- and
-#: `end_present` requires the marker within two rows of the end, so it could never be satisfied.
-#: Reading the last user turn out of the DOM immediately before leaving gives the marker for the
-#: thread as it actually stands at this point in the film.
-#:
-#: `trim()` and then a prefix, never a whitespace-collapsing normalisation: readiness.PROBE_JS
-#: matches with `textContent.includes(marker)` against RAW text, and trimming plus slicing from the
-#: front leaves a genuine substring of that raw text while collapsing internal runs would not.
_LAST_USER_TEXT_JS = """
() => {
const rows = Array.from(document.querySelectorAll('[data-role="user"]'));
@@ -1980,15 +1699,10 @@ _LAST_USER_TEXT_JS = """
}
"""
-#: How long the rebuilt thread is given to reach the readiness gate's definition of ready, in
-#: seconds: what is left of the SLOT, floored and capped.
-#:
-#: Bounded by the slot because the condition below is now one a broken arm can fail to satisfy
-#: forever -- a thread that never stops mutating never settles -- and the film's own last slot,
-#: `delete_message`, is the one place an overrun has nowhere to go (see scene/schedule.py, which
-#: records a sweep where exactly that lost delete on every base-arm cell). Floored at 10s so a
-#: tight budget still gives a 1M-token thread a fair chance, and capped at the flat minute the
-#: previous "declared its total" loop waited, so nothing waits longer than it used to.
+#: How long the rebuilt thread is given to reach the readiness gate, in seconds: what is left of
+#: the SLOT, floored at 10s so a 1M-token thread still gets a fair chance and capped at the flat
+#: minute the previous loop waited. Slot-bounded because a broken arm can fail the condition
+#: forever and `delete_message`, the film's last slot, has nowhere to absorb an overrun.
_REOPEN_READY_FLOOR_S = 10.0
_REOPEN_READY_CEILING_S = 60.0
@@ -2005,52 +1719,34 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:
thread_id = ctx.args.get("thread_id")
if not thread_id:
return not_run("no thread id was supplied to the action")
- # The THREAD's length, not the mounted count: a windowed arm that reopens with a different
- # scroll anchor mounts a different number of rows, and the exact-equality assertion at the end
- # of this action would then fail for a reason that has nothing to do with the rebuild.
+ # The THREAD's length, not the mounted count: a windowed arm reopening at a different scroll
+ # anchor mounts a different number of rows and would fail the exact-equality assertion.
before = _ev(ctx, "() => window.__sb.dom.threadTotal()")
mounted_before = _ev(ctx, "() => window.__sb.dom.messageCount()")
- # AN INTEGER, not merely truthy. `_ev` hands back `{"__error": ...}` when the page throws, and
- # a dict is truthy: it would reach the comparisons below and the readiness gate's own
- # `expected_messages` as a length, and raise there instead of being reported here.
+ # AN INTEGER, not merely truthy: `_ev` hands back `{"__error": ...}` when the page throws, and a
+ # truthy dict would reach the readiness gate as a length and raise there.
if not isinstance(before, int) or before <= 0:
return not_run("the thread has no messages to rebuild")
- # WHICH READINESS MODE THIS ARM IS HELD TO, decided from the mount that is on screen now and
- # therefore before anything is torn down. A thread whose declared total is larger than what it
- # has mounted IS a windowed mount -- that is `isWindowed()` in scene/dom.js -- and it is the
- # same distinction runtime/readiness.py draws between its two modes. Read from the page rather
- # than taken from a session flag so both arms are judged by one rule.
+ # WHICH READINESS MODE THIS ARM IS HELD TO, decided from the mount on screen before anything is
+ # torn down: a declared total larger than the mounted count IS a windowed mount. Read from the
+ # page rather than a session flag so both arms are judged by one rule.
mode = (
MODE_WINDOWED if isinstance(mounted_before, int) and mounted_before < before else MODE_FULL
)
marker = _ev(ctx, _LAST_USER_TEXT_JS)
if not isinstance(marker, str) or not marker:
- # Refused HERE, before a single thing has been touched, so a thread this action cannot
- # verify is also a thread it has not disturbed. Without a string that identifies the end of
- # the conversation there is no way to tell a rebuilt thread from a half-rebuilt one, which
- # is the entire question the wait below exists to answer.
+ # Refused HERE, before anything is touched, so a thread this action cannot verify is also one it
+ # has not disturbed: without a string identifying the end of the conversation there is no way to
+ # tell a rebuilt thread from a half-rebuilt one.
return not_run(
"the last user turn carried no text to identify the end of the thread with, so a "
"finished rebuild could not have been told from a partial one"
)
started = time.monotonic()
- # NO FALLBACK ON THE WAY OUT. Workspace task #102 and the defect it left behind.
- #
- # That task taught this action to REFUSE a substituted navigation, because a goto is a FULL
- # DOCUMENT NAVIGATION -- the SPA is torn down and reparsed, the bundle re-executes, the runtime
- # rehydrates -- while the click is a client-side route change that rebuilds one React subtree.
- # They are not the same operation and they do not cost the same thing; read as though it were
- # the click, the substitution produced thread_reopen at 6.0 fps, a number about a page load
- # quoted as a number about a thread.
- #
- # But it refused AFTER THE FACT, by inspecting `path` once `page.goto` had already run. The row
- # was then honest and the rest of the film was collateral damage: the scene carried on from an
- # empty new chat, and `delete_message` -- the last slot of every film, three slots later --
- # found no messages and went unexercised for a reason that had nothing to do with deleting.
- #
- # So the substitution is declined BEFORE it happens. Nothing is clicked, nothing is navigated,
- # the thread is still on screen, and the slots after this one still have the thread they were
- # written for.
+ # NO FALLBACK ON THE WAY OUT. A goto is a FULL DOCUMENT NAVIGATION while the click is a
+ # client-side subtree rebuild, and read as the click the substitution produced thread_reopen at
+ # 6.0 fps. Refusing AFTER `page.goto` had run left the scene on an empty new chat, so the
+ # substitution is now declined BEFORE it happens.
leave = _click_or_navigate(
ctx,
'button[aria-label="New chat"]',
@@ -2069,19 +1765,11 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:
"page navigation; a document reload is not a thread rebuild, so the action was not run "
f"and no navigation was performed ({leave.reason})"
)
- # BOTH CLOCKS START AT THE CLICK THAT WORKED, not at the first attempt on it.
- #
- # `_click_or_navigate` tries `handle.click` first, and Playwright's hit-target check retries
- # for its whole 2,000 ms timeout against a control whose centre does not hit-test to it. The
- # New chat button IS such a control by design -- `opacity-0 pointer-events-none` until its
- # header group is hovered -- so that timeout was paid on EVERY run and landed inside
- # `close_ms`, which sweep/floor_table.py harvests and quotes against the other arm. A metric
- # that is two seconds of harness retry plus the unmount is the same defect as reporting a
- # settle delay as `copy_ms`, one action along.
- #
- # The retry is not discarded, it is MOVED: `left_click_retry_ms` and `reopen_click_retry_ms`
- # below carry it as the harness's own cost, which is what it is. `started` still governs the
- # budget arithmetic, because the retry really did consume the slot's wall clock.
+ # BOTH CLOCKS START AT THE CLICK THAT WORKED. `_click_or_navigate` tries `handle.click` first,
+ # and Playwright's hit-target check retries for its full 2,000 ms against the New chat button,
+ # which is `opacity-0 pointer-events-none` until its header group is hovered, so that timeout
+ # landed inside the `close_ms` floor_table quotes. The retry is MOVED to the retry fields.
+ # Moved to `left_click_retry_ms` and `reopen_click_retry_ms`.
click_left_at = leave.started_at if leave.started_at is not None else started
# Unmount FIRST, or "already back" is indistinguishable from "never left".
closed_ms = None
@@ -2095,11 +1783,9 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:
return not_run("the thread never unmounted, so the rebuild could not be timed")
reopen_started = time.monotonic()
- # THE FALLBACK IS ALLOWED ON THE WAY BACK, and refused on the way out, for one reason: what it
- # does to the scene. From an empty new chat, a navigation to the thread's own URL is what puts
- # the thread back on screen for the slots that follow, so as a REPAIR it is the right tool --
- # where on the way out it was the thing that broke the scene. It is still not a measurement of
- # a rebuild, which is what the `not_run` below says, and no timing is reported for it.
+ # THE FALLBACK IS ALLOWED ON THE WAY BACK: from an empty new chat, navigating to the thread's own
+ # URL restores the scene for the slots that follow. Still not a measurement of a rebuild, so no
+ # timing is reported for it.
back = _click_or_navigate(
ctx, f'[data-thread-id="{thread_id}"]', f"{ctx.args['base_url']}/chat?thread={thread_id}"
)
@@ -2117,33 +1803,11 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:
"the thread's sidebar row could not be clicked and a full page navigation was "
f"substituted, so the rebuild was never timed ({back.reason})"
)
- # WHEN IS THE REBUILD OVER?
- #
- # NOT when `threadTotal()` reaches the length it had. On a windowed arm `threadTotal()` reads
- # `aria-setsize`, which is the STORE'S CLAIM about how long the conversation is, and the very
- # first reopened row publishes it: the condition was satisfied while three of eighteen messages
- # were mounted, with no bottom window, no final assistant content and no syntax highlighting
- # yet. The action then reported that as `reopen_ms`, took its census and its parity digest off
- # a half-built DOM, and still passed its own assertion because `after == before` was a
- # statement about the same declared total. A declaration is not a rebuild.
- #
- # WHAT `reopen_ms` MEASURES NOW: from the click on the thread's sidebar row until the reopened
- # thread satisfies runtime/readiness.py's gate -- the composer is present, the END of the
- # conversation is mounted (the last user turn, found by the marker captured above), and the
- # mount has SETTLED, meaning two samples STABLE_GAP_MS apart agree on the mounted count, the
- # element count and the laid-out height. Plus, in `windowed` mode, that the thread declares the
- # right total on every row and is anchored at the end.
- #
- # THAT SAME FUNCTION, not a second definition of "ready" written here. Two disagreeing
- # definitions in one harness is a defect of its own, and this is the definition the cell was
- # already admitted under: the gate that let the cell start is the gate that decides the rebuild
- # is finished, so the number is the cost of getting back to the state the cell began in.
- #
- # COMPARABILITY: every condition gated in `full` mode is one a fully mounted thread reaches by
- # construction, and each arm is judged in the mode its own mount is in, exactly as the cell's
- # opening gate judged it. The extra `windowed` conditions are a contract that arm already had
- # to meet to be scored at all. What this costs is a floor of one STABLE_GAP_MS, paid equally by
- # both arms, because "settled" is a claim about two samples and cannot be made from one.
+ # WHEN IS THE REBUILD OVER? Not when `threadTotal()` reaches its old length: on a windowed arm
+ # that reads `aria-setsize`, the STORE'S CLAIM, which the first reopened row publishes while
+ # three of eighteen messages are mounted. `reopen_ms` now runs from the sidebar click until
+ # runtime/readiness.py's own gate passes. That same function, not a second definition of ready,
+ # and each arm is judged in its own mode; the cost is one STABLE_GAP_MS paid equally by both.
left_ms = ctx.budget_ms - (time.monotonic() - started) * 1000
timeout_s = min(_REOPEN_READY_CEILING_S, max(_REOPEN_READY_FLOOR_S, left_ms / 1000))
reopen_ms = None
@@ -2159,13 +1823,12 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:
reopen_ms = (time.monotonic() - click_back_at) * 1000
readiness = ready.as_dict()
except ThreadNotReady as exc:
- # A REAL FINDING ABOUT THE ARM, and it keeps `ran = True` so the row carries which
- # conditions were still outstanding. `timings["reopen_ms"]` stays null, which the scoring
- # layer reads as "ran but recorded no reopen_ms" rather than as a fast rebuild.
+ # A REAL FINDING ABOUT THE ARM: `ran` stays True with the outstanding conditions on the row, and
+ # `reopen_ms` stays null so scoring does not read it as a fast rebuild.
readiness = exc.detail
except Exception as exc: # noqa: BLE001
- # The probe itself failed (a closed page, an uninstalled `window.__sb`). Nothing was
- # learned about the rebuild, so nothing is claimed about it.
+ # The probe itself failed (a closed page, an uninstalled `window.__sb`), so nothing is claimed
+ # about the rebuild.
ctx.log(f" the readiness probe failed during the reopen: {type(exc).__name__}: {exc}")
return not_run(
f"the reopened thread could not be probed for readiness: {type(exc).__name__}: {exc}"
@@ -2185,21 +1848,15 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:
"highlight_spans_after": spans,
"left_via": leave.path,
"reopened_via": back.path,
- # THE HARNESS'S OWN COST, named as such and kept OUT of the two timings above. This is
- # how long `_click_or_navigate` spent on attempts that failed before the one that
- # worked -- almost always Playwright's 2,000 ms hit-target retry against a control it
- # cannot hit at the centre. It belongs in the payload (a run where this grows is a run
- # whose controls have moved) and it does not belong in a rebuild timing.
+ # THE HARNESS'S OWN COST, kept OUT of the two timings above: how long `_click_or_navigate` spent
+ # on attempts that failed, almost always Playwright's 2,000 ms hit-target retry.
"left_click_retry_ms": round((click_left_at - started) * 1000, 1),
"reopen_click_retry_ms": round((click_back_at - reopen_started) * 1000, 1),
- # WHICH GATE THE TIMING WAS TAKEN AGAINST, beside the timing itself. Two arms in
- # different readiness modes are answering slightly different questions, and a reader
- # comparing their `reopen_ms` has no other way to see that.
+ # WHICH GATE THE TIMING WAS TAKEN AGAINST: two arms in different readiness modes are answering
+ # slightly different questions.
"reopen_ready_mode": mode,
- # The gate's own verdict: every condition, the last probe reading and how many samples
- # it took. When the rebuild did not finish, this is what says which condition was still
- # outstanding -- "mounted 3 of 18, aria-setsize 18" is the difference between a slow
- # app and an arm that publishes a total it has not built.
+ # The gate's own verdict: "mounted 3 of 18, aria-setsize 18" is the difference between a slow app
+ # and an arm that publishes a total it has not built.
"reopen_readiness": readiness,
},
timings = {
@@ -2229,18 +1886,10 @@ class Transition:
ok: bool
path: str # "click", "navigate" or "failed"
reason: str = ""
- #: The monotonic instant the interaction that SUCCEEDED was issued, so a caller timing "from
- #: the click" can start its clock at the click rather than at the first attempt. `None` when
- #: nothing succeeded.
- #:
- #: THE FAILED ATTEMPT IS NOT PART OF THE GESTURE. Playwright's hit-target check reads
- #: `elementFromPoint` at the control's centre and retries until the timeout, so a control that
- #: is laid out but not hit-testable at its centre -- the sidebar's `New chat`, which ships
- #: `opacity-0 pointer-events-none` until its group is hovered, and which therefore falls
- #: through to the group underneath on every hit test -- costs the full 2,000 ms before the
- #: reveal path below succeeds. Timed from the first attempt, `thread_reopen.close_ms` was
- #: two seconds of Playwright retry with the unmount somewhere inside it, on every run, and
- #: sweep/floor_table.py quotes it as a metric.
+ #: The monotonic instant the interaction that SUCCEEDED was issued, so a caller can time from the
+ #: click rather than the first attempt; None when nothing succeeded. The failed attempt is not
+ #: part of the gesture: Playwright's hit-target check retries for the full 2,000 ms against the
+ #: sidebar's `New chat`, which made `thread_reopen.close_ms` two seconds of retry.
started_at: Optional[float] = None
@property
@@ -2248,9 +1897,8 @@ class Transition:
return self.path == "navigate"
-#: Where on an element to look for a point a user could actually hit. Fractions of the box, centre
-#: first (so a normal control behaves exactly as before), then the lower band, which is what stays
-#: uncovered when a sticky header overlaps the top of a control.
+#: Where on an element to look for a point a user could hit: fractions of the box, centre first,
+#: then the lower band that stays uncovered when a sticky header overlaps the top.
_HIT_POINTS = (
(0.5, 0.5),
(0.5, 0.75),
@@ -2262,13 +1910,9 @@ _HIT_POINTS = (
(0.5, 0.25),
)
-#: EVERY match, not the first. `document.querySelector` returns the first element in DOCUMENT
-#: ORDER, and this app renders "New chat" twice -- once in the sidebar header and once on the chat
-#: page -- exactly as it renders the composer twice. When the sidebar is collapsed its button is
-#: still in the tree at zero size, comes first, and is the one a plain query hands back: the probe
-#: then reports "no point on the control hit-tests to it" about a button that is 34x34 and fully
-#: clickable a few hundred pixels away. This is the same trap `image_upload` documents at length,
-#: and it cost this action a second time.
+#: EVERY match, not the first: this app renders "New chat" twice, and a collapsed sidebar leaves
+#: its zero-size button first in document order, so a plain query reported "no point hit-tests to
+#: it" about a fully clickable button elsewhere. The same trap `image_upload` documents.
_HIT_TEST_JS = """
([selector, points]) => {
for (const el of document.querySelectorAll(selector)) {
@@ -2297,18 +1941,14 @@ def _reachable_point(ctx: ActionContext, selector: str) -> tuple[float, float] |
return None
-#: How long the reveal transition needs, and how many times to look. `.sidebar-header-action`
-#: transitions opacity over 150ms. One sample after a fixed sleep is a race that this test suite
+#: `.sidebar-header-action` transitions opacity over 150ms, and one sample after a fixed sleep
#: lost roughly one run in three on a loaded machine, so the hit test is retried instead.
_REVEAL_SETTLE_MS = 120
_REVEAL_ATTEMPTS = 5
-#: The control's own box, whatever its computed style says. An earlier version only offered a
-#: hover point for a control that LOOKED hidden, which made the reveal conditional on a style read
-#: taken at exactly the wrong moment: mid-transition `getComputedStyle` reports the old opacity,
-#: and after a failed Playwright click the element can read as already revealed while not yet
-#: being hit-testable. Since this only runs when the control is already known to be unreachable,
-#: there is nothing to gain by being clever about whether it deserves a hover.
+#: The control's own box, whatever its computed style says: mid-transition `getComputedStyle`
+#: reports the old opacity, and after a failed click the element can read as revealed while not
+#: yet hit-testable. This only runs when the control is already known to be unreachable.
_HOVER_TARGET_JS = """
(selector) => {
for (const el of document.querySelectorAll(selector)) {
@@ -2398,19 +2038,11 @@ def _click_or_navigate(
return Transition(ok = True, path = "click", started_at = attempt_at)
except Exception as exc: # noqa: BLE001
click_error = f"{selector} was not clickable: {type(exc).__name__}"
- # THE CENTRE IS COVERED, BUT THE CONTROL IS NOT. Try the rest of it, the way a user would.
- #
- # Playwright clicks the centre of an element and refuses when something else hit-tests
- # there. The sidebar's sticky group label overlaps the top of the New chat button, so the
- # centre is taken and the click times out -- but most of the button is uncovered and a
- # person clicks it every day without noticing. Giving up at the centre and calling the
- # control unreachable is the harness being more fragile than a user, and it cost this
- # action on every CI run.
- #
- # So the element's own box is hit-tested at a spread of points and the first one that
- # resolves to the control is clicked with a real mouse event. If NO point on the control
- # is reachable then it genuinely cannot be clicked, which is a finding worth reporting
- # rather than a reason to substitute a page load.
+ # THE CENTRE IS COVERED, BUT THE CONTROL IS NOT. Playwright clicks the centre and refuses when
+ # something else hit-tests there; the sidebar's sticky group label overlaps the top of New chat
+ # while most of the button is clickable. So the box is hit-tested at a spread of points and the
+ # first that resolves to the control is clicked with a real mouse event; no reachable point is a
+ # finding worth reporting rather than a reason to substitute a page load.
point = _reachable_point(ctx, selector) or _reveal_by_hover(ctx, selector)
if point is not None:
try:
@@ -2523,18 +2155,10 @@ async (opts) => {
"""
-#: How long an action that needs a message's ACTION BAR will wait for the reply to finish.
-#:
-#: Unsloth hides the action bar while a message is generating, so "no More button on the last
-#: assistant message" and "no Delete button (a running message hides it)" are the same fact
-#: reported by two actions. The film schedules `message_menu` about four seconds after a
-#: `send_turn` whose reply runs for roughly fourteen, so the menu was being asked for on a message
-#: that was still being written, and the action reported NOT RUN on every CI run.
-#:
-#: Waiting is what a USER does, and it is the only thing that makes the action measurable at all:
-#: there is no menu to open until the reply lands. Bounded by the slot's own budget so a reply
-#: that never finishes cannot swallow the rest of the film, and the wait happens BEFORE the
-#: operation the action times, so `menu_open_ms` is unaffected.
+#: How long an action that needs a message's ACTION BAR waits for the reply to finish. Unsloth
+#: hides the bar while a message generates, and the film schedules `message_menu` about four
+#: seconds after a `send_turn` whose reply runs for roughly fourteen, so the action reported NOT
+#: RUN on every CI run. Bounded by the slot's budget, and waited BEFORE the timed operation.
_ACTION_BAR_WAIT_FRACTION = 0.6
_ACTION_BAR_POLL_MS = 100
@@ -2556,14 +2180,10 @@ def _wait_for_the_reply_to_land(ctx: ActionContext) -> bool:
@register_action(name = "message_menu", default_budget_ms = 12000)
def message_menu(ctx: ActionContext) -> ActionResult:
- # TWO WAITS, AND THEY COVER DIFFERENT THINGS. This one waits for the STREAM to stop, bounded by
- # a fraction of the slot's own budget, because a follow-up reply runs for seconds and Unsloth
- # unmounts the whole action bar for its whole duration -- far longer than the page-side wait
- # below is allowed to sit. `waitForButtonMs` then waits for the BAR TO MOUNT, which happens a
- # few hundred milliseconds after the reply settles, and which this one cannot see because
- # `isRunning()` is already false by then. Dropping either leaves a real NOT RUN on the table:
- # without the first, a fourteen second reply outlasts a 1.5 s page-side wait; without the
- # second, the slot asks for a control that has not been mounted yet.
+ # TWO WAITS, COVERING DIFFERENT THINGS: this one waits for the STREAM to stop, bounded by a
+ # fraction of the slot, while `waitForButtonMs` waits for the BAR TO MOUNT a few hundred ms
+ # later, which this one cannot see because `isRunning()` is already false.
+ # See `OWN_TURN_STOP_POLL_MS`.
if not _wait_for_the_reply_to_land(ctx):
return not_run(
"a reply was still generating when the slot's budget ran out, and Unsloth hides the "
@@ -2581,9 +2201,8 @@ def message_menu(ctx: ActionContext) -> ActionResult:
return not_run(raw.get("reason", "the menu action did not run"))
waited_ms = raw.get("waitedMs")
if waited_ms:
- # Logged rather than swallowed. A cell that had to wait says so, because a wait means the
- # film opened this slot before the follow-up had drained, and that is a fact about the
- # PACKING which is worth seeing even though the action ran.
+ # Logged rather than swallowed: a cell that had to wait says so, because the wait is a fact about
+ # the film's PACKING.
ctx.log(f" message_menu waited {waited_ms}ms for the action bar to be mounted")
# Opened AND closed AND a non-zero item count. Any one of the three alone can be satisfied by
# a menu that never rendered its items.
@@ -2593,13 +2212,12 @@ def message_menu(ctx: ActionContext) -> ActionResult:
expect_ok = ok,
expect = {
"items_while_open": raw["items"],
- # Radix puts the body on the modal layer while the menu is up, which is the
- # fan-out under suspicion. This proves the open really took that path.
+ # Radix puts the body on the modal layer while the menu is up, which is the fan-out under
+ # suspicion, so this proves the open really took that path.
"body_pointer_events_open": raw["bodyPointerEvents"],
"body_pointer_events_closed": raw["bodyPointerEventsAfterClose"],
- # In the payload, not only in the log: a run whose cells all waited is a run whose
- # film opens this slot too early, and that is not visible from a row that only says
- # `ran`.
+ # In the payload, not only in the log: a run whose cells all waited is a run whose film opens
+ # this slot too early.
"action_bar_wait_ms": waited_ms,
},
timings = {
@@ -2682,9 +2300,8 @@ def delete_message(ctx: ActionContext) -> ActionResult:
return not_run(err)
if not raw.get("ran"):
return not_run(raw.get("reason", "the delete action did not run"))
- # The THREAD TOTAL dropped. A delete that detached the node but left the thread the same
- # length is a different bug -- or, on a windowed mount, a node the virtualizer recycled -- and
- # must not read as a successful delete.
+ # The THREAD TOTAL dropped: a delete that detached the node but left the thread the same length
+ # is a different bug, or on a windowed mount a node the virtualizer recycled.
ok = raw["ms"] is not None and raw["after"] < raw["before"]
return ActionResult(
ran = True,
diff --git a/tests/studio/studiobench/scene/dom.js b/tests/studio/studiobench/scene/dom.js
index 7803f7b9ca2..3f41be6aa5a 100644
--- a/tests/studio/studiobench/scene/dom.js
+++ b/tests/studio/studiobench/scene/dom.js
@@ -1,27 +1,17 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
-// The selector adapter for the REAL Unsloth chat UI.
-//
-// The salvaged action JS from playwright_heavy_thread.py calls `window.__heavyThread`, an API the
-// smoke fixture exported. The shipping app exports nothing of the kind, so the actions are ported
-// onto this: the same method names, backed by the app's own selectors. One file to edit when a
-// class changes, instead of fifteen.
-//
-// The chat thread has essentially NO test ids -- exactly one, `composer-tool-status`, in the whole
-// of components/assistant-ui and features/chat. So the contract is class hooks (`aui-*`,
-// `unsloth-*`), `data-role`, `data-slot` and accessible names, all read out of the shipped TSX.
-//
-// Three selectors carry a trap and are handled here rather than in each action:
-//
-// - The composer exists TWICE in compare mode and once on the welcome screen vs once docked, so
-// everything scopes to the first thread root.
-// - The Stop button is REPLACED by a Queue button when the composer has text, because
-// `queueDisabled` depends on `composerText.trim().length > 0`. An action that presses stop
-// with text in the box presses queue and measures nothing.
-// - Radix keeps a collapsed Collapsible's content element mounted for its animation, so
-// "the reasoning pane is open" cannot be read from the content element's presence. It is read
-// from `data-state` on the root, which is what Radix actually toggles.
+// The selector adapter for the REAL Unsloth chat UI: the salvaged actions call the smoke
+// fixture's `window.__heavyThread` API and are ported onto this instead. Same method names,
+// the app's own selectors, one file to edit when a class changes.
+// Salvaged from playwright_heavy_thread.py.
+// The chat thread has essentially no test ids (exactly one, `composer-tool-status`), so the
+// contract is class hooks (`aui-*`, `unsloth-*`), `data-role`, `data-slot` and accessible
+// names read out of the shipped TSX.
+// Three selectors carry a trap, handled here rather than per action: the composer exists TWICE
+// in compare mode (so everything scopes to the first thread root); the Stop button is REPLACED
+// by Queue when the composer has text, since `queueDisabled` follows `composerText`, so
+// pressing stop with text measures nothing; and Radix keeps a collapsed Collapsible's content
+// mounted for its animation, so "open" is read from `data-state` on the root.
(() => {
if (window.__sb && window.__sb.dom) return;
@@ -31,8 +21,8 @@
const qa = (sel, root) => Array.from((root || document).querySelectorAll(sel));
// Accessible name for the app's TooltipIconButton, which renders the label in a visually
- // hidden span rather than an aria-label. `getByRole(name:)` on the driver side would do this,
- // but the actions run inside one page.evaluate so they need it here.
+ // hidden span rather than an aria-label. `getByRole(name:)` would do this driver-side, but
+ // the actions run inside one page.evaluate.
const nameOf = (el) => {
if (!el) return "";
const aria = el.getAttribute("aria-label");
@@ -43,11 +33,10 @@
};
// The two hooks the app publishes a part's progress through: assistant-ui's `data-status` on a
- // text part (markdown-text.tsx) and `aria-busy` on the reasoning content (reasoning.tsx). Named
- // once because the streaming scan and the control on it must never drift apart.
- // Every control ComposerRightControls can put in the run-state slot, in no significant order.
- // "Stop research" / "Stopping research" are included because they are the same slot, even though
- // nothing in this benchmark can reach a research run today.
+ // text part (markdown-text.tsx) and `aria-busy` on the reasoning content (reasoning.tsx).
+ // Named once so the streaming scan and its control cannot drift apart.
+ // Every control ComposerRightControls can put in the run-state slot. The research pair is
+ // included because it is the same slot, even though nothing here can reach a research run.
const RUN_STATE_CONTROLS = [
"Stop generating",
"Stop queued message",
@@ -87,133 +76,84 @@
queueButton() {
return q('button[aria-label="Queue message"]');
},
- // THE DISPATCHED HALF OF A QUEUE RUN. `ComposerRightControls` renders this once the queued
- // entry has been dispatched, under `isQueueRunning && !thread.isRunning` (thread.tsx), so the
- // thread reports itself NOT running and neither `stopButton()` nor `queueButton()` matches.
- // Read on its own, that transient came out as an ordinary idle composer: `streaming` and
- // `queued_idle` both false, exactly as a settled Send arm reads. An arm caught here against a
- // settled one then had a differing composer with no run-state difference to explain it, which
- // is the shape the comparison layer treats as a rendering regression.
+ // THE DISPATCHED HALF OF A QUEUE RUN: `ComposerRightControls` renders this under
+ // `isQueueRunning && !thread.isRunning`, so the thread reports itself NOT running and neither
+ // `stopButton()` nor `queueButton()` matches. Read on its own that transient looked like a
+ // settled Send arm, giving a differing composer with no run-state difference to explain it:
+ // the shape the comparison layer treats as a rendering regression.
stopQueuedButton() {
return q('button[aria-label="Stop queued message"]');
},
isRunning() {
return Boolean(D.stopButton() || D.queueButton());
},
- // WHICH RUN-STATE CONTROL THE COMPOSER IS RENDERING, as a token.
- //
- // `ComposerRightControls` renders exactly one of these at a time and which one is a function
- // of the run state: Send when nothing is happening, Stop while a reply is being written, Queue
- // while one is queued or while text sits in the box mid-reply, and the research pair while a
- // research run is going. They are DIFFERENT SUBTREES, and the composer dock is inside
- // `.aui-thread-root`, so `digest_scaffold` carries whichever one is up.
- //
- // Two arms showing different tokens differ in the scaffold FOR THAT REASON, with no rendering
- // difference between them. `isRunning()` is too coarse to say so: it is true for Stop AND for
- // Queue, so a queued-idle arm and a streaming arm agree on it while rendering different
- // controls. The token is the thing the scaffold actually contains.
+ // WHICH RUN-STATE CONTROL THE COMPOSER IS RENDERING, as a token. `ComposerRightControls`
+ // renders exactly one at a time as a function of the run state, and they are DIFFERENT
+ // SUBTREES inside `.aui-thread-root`, so `digest_scaffold` carries whichever is up and two
+ // arms showing different tokens differ in the scaffold for that reason. `isRunning()` is too
+ // coarse to say so, being true for Stop AND Queue.
runStateControl() {
for (const name of RUN_STATE_CONTROLS) {
if (byName("button", name)) return name;
}
return "";
},
- // THE PROMPT QUEUE'S OWN SURFACE. `PromptQueueStack` renders the waiting prompts inside the
- // composer root with the accessible name "Prompt queue, <n> of <m>", and that is the only
- // place the queue states its own existence in the DOM. Scoped to the thread root so a queue
- // running on ANOTHER thread, which the sidebar also renders, is not read as this one's.
+ // THE PROMPT QUEUE'S OWN SURFACE: `PromptQueueStack` renders waiting prompts inside the
+ // composer root with the accessible name "Prompt queue, <n> of <m>", the only place the queue
+ // states its existence in the DOM. Scoped to the thread root so another thread's queue is not
+ // read as this one's.
promptQueue() {
return q('[aria-label^="Prompt queue,"]', D.threadRoot());
},
- // A REPLY IS ACTUALLY BEING WRITTEN, which is NOT the question `isRunning()` answers.
- //
- // `isRunning()` has to accept the Queue button: with text in the composer a running thread
- // renders that button and NO Stop button (`queueDisabled` depends on the composer having a
- // queueable prompt), so an action that waited for Stop would wait out its budget on a live
- // stream. Every caller that asks "may I send now" wants exactly that broad reading.
- //
- // It is the wrong reading for a positive control on the streaming probe, because
- // `ComposerRightControls` renders "Queue message" in a SECOND place: under
- // `isQueueRunning && !thread.isRunning`, while a queued prompt waits to be dispatched and
- // nothing at all is generating. `isRunning()` cannot tell that queued-idle interval from a
- // live stream whose `data-status` hooks have gone quiet, and those two need opposite
- // treatment -- one is an ordinary settled thread, the other is an instrument that has stopped
- // working.
- //
- // The queued-idle button always comes with the queue surface: `syncPromptQueueUI` marks the
- // entry `dispatched` from the active item, and `getPromptQueueUIItemsForRun` drops only
- // DISPATCHED items, so an undispatched active item -- the one that renders "Queue message"
- // rather than "Stop queued message" -- is always in the list the stack renders.
- //
- // WHAT THIS GIVES UP, pinned in scene/selftest/test_studiobench_queued_idle_live.py: while a
- // queue run holds further prompts AND a reply is streaming AND the composer has text, the
- // queue surface is up and the only control is the Queue button, so this reads false and the
- // control is not armed for that capture. Under-claiming, and cheap: probe blindness is a
- // renamed selector, which is global, so every other capture in the run still catches it.
- // A RESEARCH RUN IS A GENERATION THIS PREDICATE CANNOT SEE, and deliberately so for now.
- // `ComposerRightControls` renders "Stop research" / "Stopping research" under
- // `isResearchActive`, which neither `stopButton()` nor `queueButton()` matches, and a research
- // report renders through `MarkdownPreview` rather than the assistant text part, so
- // `streamingMessages()` finds no `data-status` for it either. All THREE are blind to it
- // together.
- //
- // Not patched, because nothing here can start one: `ResearchMessage` is gated on
- // `message.metadata` and the seeder writes `None` for every message, and no action, corpus
- // unit or scene touches the composer's research toggle. Fixing one of the three would leave
- // two that still disagree with it. If a research scene is ever added, move all three in the
- // same change, and add the report's own busy hook to `STATUS_HOOK` at the same time.
+ // A REPLY IS ACTUALLY BEING WRITTEN, which is NOT what `isRunning()` answers. `isRunning()`
+ // must accept the Queue button, since with text in the composer a running thread renders no
+ // Stop button, and every "may I send now" caller wants that broad reading. It is wrong for a
+ // positive control on the streaming probe, because "Queue message" is also rendered under
+ // `isQueueRunning && !thread.isRunning` while nothing is generating, and that queued-idle
+ // interval needs the opposite treatment from a live stream whose hooks have gone quiet. The
+ // queued-idle button always comes with the queue surface, since
+ // `getPromptQueueUIItemsForRun` drops only DISPATCHED items. WHAT THIS GIVES UP (pinned in
+ // test_studiobench_queued_idle_live.py): with a queue run, a streaming reply and text in the
+ // composer this reads false and the control is not armed. Under-claiming, and cheap, since
+ // probe blindness is a renamed selector and therefore global.
+ // `syncPromptQueueUI` marks dispatched items.
+ // A RESEARCH RUN IS A GENERATION THIS PREDICATE CANNOT SEE, deliberately: "Stop research"
+ // matches neither `stopButton()` nor `queueButton()`, and a research report renders through
+ // `MarkdownPreview` rather than the assistant text part, so `streamingMessages()` finds no
+ // `data-status` either. Not patched, because nothing here can start one: `ResearchMessage` is
+ // gated on `message.metadata` and the seeder writes `None`. If a research scene is added, move
+ // all three in one change and add the report's busy hook to `STATUS_HOOK`.
+ // The state is `isResearchActive`.
generating() {
if (D.stopButton()) return true;
if (!D.queueButton()) return false;
return !D.promptQueue();
},
- // WHICH MESSAGES ARE STILL BEING WRITTEN, read from the app's own published state rather than
- // guessed at from `isRunning()` plus "it is probably the last one".
- //
- // assistant-ui gives every text part a status and Unsloth serialises it: markdown-text.tsx
- // renders `<div data-status={status.type}>` around the Streamdown tree, so a part that is
- // still arriving reads `data-status="running"` and a finished one reads `"complete"`. The
- // reasoning pane publishes the same fact as `aria-busy` on `[data-slot="reasoning-content"]`
- // (reasoning.tsx), which is a separate part of the same message and can be running while the
- // answer is not, or the other way round. Either one means this message's DOM is mid-flight.
- //
- // Both are the APP's statements about its own state. Nothing here reads a timer, a character
- // count or "the last assistant message", all of which are the benchmark inventing a fact the
- // page already publishes -- and inventing it is how you end up attributing a stream to the
- // wrong message on the arm that renders faster.
+ // WHICH MESSAGES ARE STILL BEING WRITTEN, read from the app's own published state:
+ // markdown-text.tsx renders `<div data-status={status.type}>`, and the reasoning pane
+ // publishes the same fact as `aria-busy` on `[data-slot="reasoning-content"]`, a separate part
+ // that can be running while the answer is not. Both are the APP's statements about its own
+ // state; reading a timer, a character count or "the last assistant message" is how you
+ // attribute a stream to the wrong message on the arm that renders faster.
streamingMessages() {
return qa("[data-role]").filter(
(m) => m.querySelector('[data-status="running"], [aria-busy="true"]') !== null,
);
},
- // ── IS THE STREAMING PROBE BLIND, OR IS THERE SIMPLY NOTHING FOR IT TO SEE ──────────
- //
- // `streamingMessages()` returning nothing has three completely different causes, and the
- // positive control on it is only worth having if they are told apart.
- //
- // THE HOOK IS GONE a build renamed or dropped `data-status`. It is one line in
- // markdown-text.tsx and it is rendered for `complete` parts as
- // well as `running` ones, so on a working build EVERY assistant
- // message that has rendered a part carries it, and on a blinded
- // build none does.
- // THE ROW IS NOT MOUNTED a windowed arm scrolled away from the tail has unmounted the
- // message it is writing into. That is what windowing is for.
- // THE ROW HAS NO PARTS YET between the send being accepted and the reply's first part
- // arriving, the assistant message is mounted with zero content
- // parts and thread.tsx renders "Generating..." in its place. It
- // publishes no status because it has nothing to publish yet, and
- // `send_turn` returns the instant `isRunning()` flips, so a
- // capture lands here twice a film.
- //
- // Scoped to ASSISTANT messages throughout: a user message never publishes `data-status` even
- // on a perfectly working build, because only the assistant parts are rendered through
- // `MarkdownText` (thread.tsx `ASSISTANT_PART_COMPONENTS`). A window holding only user rows
- // would otherwise read as a build with no hook.
+ // IS THE STREAMING PROBE BLIND, OR IS THERE NOTHING TO SEE? `streamingMessages()` returning
+ // nothing has three causes worth telling apart. THE HOOK IS GONE: a build renamed
+ // `data-status`, which is rendered for `complete` parts too, so on a working build every
+ // assistant message that rendered a part carries it. THE ROW IS NOT MOUNTED: a windowed arm
+ // scrolled away from the tail, which is what windowing is for. THE ROW HAS NO PARTS YET:
+ // between the send being accepted and the first part arriving, thread.tsx renders
+ // "Generating...", and `send_turn` returns the instant `isRunning()` flips, so a capture lands
+ // here twice a film. Scoped to ASSISTANT messages, since only assistant parts render through
+ // `MarkdownText`.
statusHookPresent() {
return D.assistantMessages().some((m) => m.querySelector(STATUS_HOOK) !== null);
},
- // Whether the message a reply would be written INTO is publishing parts this probe can read.
- // False means it has none yet, which is the third case above and not a broken instrument.
+ // Whether the message a reply would be written INTO is publishing parts this probe can read;
+ // false means it has none yet, the third case above and not a broken instrument.
lastAssistantPublishesStatus() {
const last = D.lastAssistantMessage();
return Boolean(last && last.querySelector(STATUS_HOOK));
@@ -224,25 +164,16 @@
messageCount() {
return qa("[data-role]").length;
},
- // HOW LONG THE THREAD IS, as opposed to how much of it is mounted.
- //
- // On the shipped build these are the same number and this returns exactly what
- // messageCount() does, so nothing about an existing run changes. They stop being the same
- // number the moment an arm mounts a window, and at that point every before/after assertion in
- // actions.py -- send_turn's "the thread grew", delete's "the count dropped", thread_reopen's
- // "it came back with the same messages" -- is asking about the THREAD and being answered
- // about the window. Under a windowed mount those three all read the wrong answer in the same
- // direction: the window refills as fast as it empties, so a delete that worked reports
- // `after == before` and fails, and a send that worked reports the same and fails.
- //
- // aria-setsize is where a windowed list is already required to publish this: WAI-ARIA says a
- // list whose items are not all in the DOM must carry aria-setsize and aria-posinset. So this
- // is not a private channel invented for the benchmark, it is the accessible name for the
- // quantity, and an arm that does not provide it is one a screen reader cannot navigate.
+ // HOW LONG THE THREAD IS, as opposed to how much is mounted. Identical to messageCount() on
+ // the shipped build, and different the moment an arm mounts a window, at which point
+ // send_turn's "the thread grew", delete's "the count dropped" and thread_reopen's "same
+ // messages" ask about the THREAD and are answered about the window, all in the same direction,
+ // because the window refills as fast as it empties. aria-setsize is where WAI-ARIA already
+ // requires a windowed list to publish this, so it is the accessible name for the quantity
+ // rather than a private channel.
threadTotal() {
- // On the message, or on the row wrapper a virtualizer positions it in. See the same walk in
- // runtime/readiness.py: the ordinal belongs on the element that is a member of the set, and
- // for a windowed list that is the positioned row rather than the message inside it.
+ // On the message, or on the row wrapper a virtualizer positions it in. Same walk as
+ // runtime/readiness.py: the ordinal belongs on the element that is a member of the set.
const first = q("[data-role]");
const owner = first ? first.closest("[aria-setsize]") : q("[aria-setsize]");
if (owner) {
@@ -251,8 +182,8 @@
}
return qa("[data-role]").length;
},
- // True when the thread is publishing a total that is larger than what it has mounted, i.e.
- // this really is a windowed mount and not merely a short thread.
+ // True when the thread publishes a total larger than what it has mounted, i.e. a windowed
+ // mount and not merely a short thread.
isWindowed() {
return D.threadTotal() > qa("[data-role]").length;
},
@@ -264,19 +195,16 @@
return all.length ? all[all.length - 1] : null;
},
- // ── following the stream ─────────────────────────────────────
- //
- // The jump-to-bottom control, which thread.tsx renders permanently and hides with
- // `invisible` when the intent-aware autoscroll reports itself at the bottom. Reading the
- // app's own state rather than recomputing it means the harness and the app cannot disagree
- // about whether the thread is pinned, which is the whole question here.
+ // The jump-to-bottom control, which thread.tsx renders permanently and hides with `invisible`
+ // when the intent-aware autoscroll reports itself at the bottom. Reading the app's own state
+ // means the harness and the app cannot disagree about whether the thread is pinned.
jumpToBottomButton() {
return q(".aui-thread-scroll-to-bottom");
},
appSaysAtBottom() {
const jump = D.jumpToBottomButton();
- // `null`, NOT `false`, when the control is absent. A build that does not render it has not
- // told us it is scrolled up; it has told us nothing, and the two must not be summed.
+ // `null`, NOT `false`, when the control is absent: a build that does not render it has told
+ // us nothing, and the two must not be summed.
return jump ? jump.classList.contains("invisible") : null;
},
distanceFromBottom() {
@@ -285,7 +213,6 @@
return Math.round(vp.scrollHeight - vp.clientHeight - vp.scrollTop);
},
- // ── reasoning ────────────────────────────────────────────────
reasoningRoots() {
return qa('[data-slot="reasoning-root"]');
},
@@ -297,25 +224,19 @@
// content mounted for the animation, so a presence check reads every pane as open.
return qa('[data-slot="reasoning-root"][data-state="open"]').length;
},
- // STILL MOUNTED IS NOT THE SAME AS STILL OPEN, and closing needs the other one.
- //
- // `reasoningOpenCount` flips on the click, because `data-state` is the state. The CHILDREN
- // outlive it on both collapse mechanisms and by design: Radix's `Presence` suspends the
- // unmount until `animationend` of `animate-collapsible-up`, and the grid arm's
- // `UnmeasuredCollapsibleContent` renders `present && children` until the `grid-template-rows`
- // `transitionend` or its 250 ms backstop. For that whole window every pane is closed, every
- // span it contributed is still in the document, and a census asked whether it has stopped
- // moving answers yes -- because it has not started.
- //
- // So a collapse is settled when the content is GONE, which is what this counts. Both arms are
- // covered by one selector: Radix removes the element, the grid arm leaves it behind with
- // `hidden`, and neither is a mounted pane.
+ // STILL MOUNTED IS NOT STILL OPEN. `reasoningOpenCount` flips on the click, but the CHILDREN
+ // outlive it on both collapse mechanisms by design: Radix's `Presence` suspends the unmount
+ // until `animationend`, and the grid arm renders `present && children` until `transitionend`
+ // or its 250 ms backstop. For that window every pane is closed while every span it contributed
+ // is still in the document, and a census asked whether it has stopped moving answers yes
+ // because it has not started. So a collapse is settled when the content is GONE, which one
+ // selector covers on both arms.
+ // The grid arm is `UnmeasuredCollapsibleContent`.
reasoningContentMounted() {
return qa('[data-slot="reasoning-content"]').filter((el) => !el.hasAttribute("hidden"))
.length;
},
- // ── action bar ───────────────────────────────────────────────
actionBar(message) {
const m = message || D.lastAssistantMessage();
if (!m) return null;
@@ -330,9 +251,9 @@
const m = message || D.lastAssistantMessage();
return m ? byName("button", name, m) : null;
},
- // Hover the last assistant message, which is what mounts its action bar. `autohide` on the
- // bar unmounts it on every message that is not hovered, so a control read without this is
- // read out of a tree it was never in.
+ // Hover the last assistant message, which is what mounts its action bar: `autohide` unmounts
+ // it on every message that is not hovered, so a control read without this is read out of a
+ // tree it was never in.
hoverLastAssistantMessage() {
const m = D.lastAssistantMessage();
if (m) {
@@ -343,35 +264,18 @@
return m;
},
- // WAIT for one of the action bar's controls, up to `waitMs`, instead of sampling for it once.
- //
- // WHY A WAIT AND NOT A SAMPLE, which is the whole point of this method. The assistant action
- // bar is mounted with `hideWhenRunning` (thread.tsx), so while the thread is generating there
- // is no Copy, no Delete and no More ANYWHERE in the tree -- not hidden, not disabled, absent.
- // Every action that needs one of them is therefore scheduled after a `send_turn`, on the
- // arithmetic that the follow-up turn drains in FOLLOW_UP_CHARS / the field cadence. That
- // arithmetic is the NOMINAL drain: it assumes the pacer's cadence is the binding constraint,
- // and at the 100K rung it is not -- the renderer is, so the reply arrives about 25% later
- // than the cadence says, and the slot can open with the last few chunks still in flight.
- //
- // Measured on the CI runner, from the payload of the run that failed the liveness gate: the
- // `message_menu` window opened at 32,000ms, took ONE more SSE chunk inside itself, and the
- // reply stopped growing 71 characters later, inside the same window. The reply settled about
- // a third of a second after the instant the action sampled the DOM. A single sample turns
- // that third of a second into `NOT RUN -- no More button on the last assistant message`,
- // which reads like a missing control and is really a clock being read too early.
- //
- // So the control is waited for. The wait is bounded, it is reported, and it happens BEFORE
- // any measurement clock starts, so a cell that had to wait is not a cell whose open latency
- // was inflated by the waiting. `fixture/selftest/test_studiobench_rung_plan.py` already
- // asserts this behaviour in prose -- "a slot may legitimately open a little before the
- // follow-up finishes and wait inside its own budget" -- and this is the code that makes the
- // sentence true.
- //
- // Polling per paint, not per millisecond, and scoped: `actionButton` searches inside the last
- // assistant message, so this is O(that message) rather than O(the thread), and it stops the
- // instant the control appears (0 iterations on every cell where the reply has settled, which
- // is the normal case).
+ // WAIT for one of the action bar's controls, up to `waitMs`, instead of sampling once. The bar
+ // is mounted with `hideWhenRunning`, so while the thread generates there is no Copy, Delete or
+ // More anywhere: not hidden, absent. Every action needing one is scheduled after a `send_turn`
+ // on the NOMINAL drain arithmetic, which assumes the pacer is the binding constraint; at 100K
+ // the renderer is, so the reply arrives about 25% later. On the CI run that failed the
+ // liveness gate the `message_menu` window opened at 32,000ms, took one more SSE chunk inside
+ // itself, and the reply stopped growing 71 characters later inside the same window: a single
+ // sample turns that third of a second into `NOT RUN -- no More button`. The wait is bounded,
+ // reported, and happens BEFORE any measurement clock starts. Polling per paint and scoped to
+ // the last assistant message, so it is O(that message) and stops as soon as the control
+ // appears.
+ // The drain arithmetic is FOLLOW_UP_CHARS over the field cadence; see test_studiobench_rung_plan.py.
async waitForActionButton(name, waitMs, everyMs) {
const started = performance.now();
const budget = Math.max(0, Number(waitMs) || 0);
@@ -383,17 +287,16 @@
let el = D.actionButton(name);
while (!el && performance.now() - started < budget) {
await nextPaint();
- // Re-hovered every pass: the bar unmounts again whenever the message re-renders, which
- // during a stream is on every chunk.
+ // Re-hovered every pass: the bar unmounts again whenever the message re-renders, which during
+ // a stream is on every chunk.
D.hoverLastAssistantMessage();
el = D.actionButton(name);
}
return {
el,
waitedMs: Math.round((performance.now() - started) * 10) / 10,
- // Recorded whether the control was found or not. A miss with `running: true` is the
- // reply not having settled; a miss with `running: false` is a control that is genuinely
- // not there, and those are different bugs.
+ // Recorded whether the control was found or not: a miss with `running: true` is the reply not
+ // having settled, a miss with `running: false` is a control that is genuinely not there.
running: D.isRunning(),
};
},
@@ -405,7 +308,6 @@
return menu ? qa(".aui-action-bar-more-item", menu).length : 0;
},
- // ── settings ─────────────────────────────────────────────────
settingsTrigger() {
return q('button[aria-label="Settings"]');
},
@@ -421,7 +323,6 @@
return q('[data-testid="settings-tab-' + id + '"]');
},
- // ── model picker ─────────────────────────────────────────────
modelTrigger() {
return q("button.unsloth-model-selector-trigger");
},
@@ -439,7 +340,6 @@
return t ? (t.textContent || "").trim() : null;
},
- // ── composer plus menu / attachments ─────────────────────────
plusButton() {
return q('button[aria-label="Tools and attachments"]') || q('[data-tour="chat-plus-menu"]');
},
@@ -451,7 +351,6 @@
);
},
- // ── sidebar / threads ────────────────────────────────────────
threadRows() {
return qa('[data-testid="recent-thread"]');
},
@@ -462,12 +361,10 @@
return q('button[aria-label="New chat"].sidebar-header-action') || q('button[aria-label="New chat"]');
},
- // ── code blocks ──────────────────────────────────────────────
codeCopyButtons() {
return qa('button[title="Copy code"]');
},
- // ── census ───────────────────────────────────────────────────
counts() {
const started = performance.now();
const out = {
@@ -477,19 +374,16 @@
reasoning_panes: qa('[data-slot="reasoning-root"]').length,
reasoning_open: qa('[data-slot="reasoning-root"][data-state="open"]').length,
code_blocks: qa("pre").length,
- // Shiki spans. THE span density check: the field capture ran 90,262 characters against
- // 16,186 spans, i.e. 5.6 characters per span, and a fixture that does not reproduce that
- // is not measuring the same highlighter load per character.
+ // Shiki spans. THE span density check: the field capture ran 90,262 characters against 16,186
+ // spans, 5.6 characters per span, and a fixture that does not reproduce that is not measuring
+ // the same highlighter load per character.
highlight_spans: qa("pre span").length,
- // WHERE the spans live, not just how many. A collapsed reasoning pane UNMOUNTS its
- // children, so a thread with the same text can carry wildly different DOM depending on
- // how that text got there. Without this split, "seeded has 20% fewer spans" is a number
- // with three possible explanations and no way to choose between them.
- // Tool components. TWO markers, because there are two renderers: a known tool gets a
- // `tool-group-root`, and anything else gets the generic `tool-fallback-root` ("Used
- // tool"). Counting only the first read ZERO on a thread that visibly contained tool
- // blocks -- the second wrong selector in a row for this one component, and both times the
- // reading was a confident zero rather than an error.
+ // WHERE the spans live, not just how many: a collapsed reasoning pane UNMOUNTS its children,
+ // so a thread with the same text can carry wildly different DOM. Without the split, "seeded
+ // has 20% fewer spans" has three possible explanations.
+ // Tool components, TWO markers because there are two renderers: a known tool gets a
+ // `tool-group-root` and anything else a generic `tool-fallback-root`. Counting only the first
+ // read ZERO on a thread that visibly contained tool blocks.
tool_groups: qa('[data-slot="tool-group-root"]').length
+ qa('[data-slot="tool-fallback-root"]').length,
tool_groups_open: qa('[data-slot="tool-group-content"]').length
@@ -499,29 +393,19 @@
content_spans:
qa("pre span").length - qa('[data-slot="reasoning-root"] pre span').length,
content_code_blocks: qa("pre").length - qa('[data-slot="reasoning-root"] pre').length,
- // Carried in the census so the peak occupancy and the character count come from the
- // SAME reading. Two separate reads, taken either side of a destructive action, disagree.
+ // Carried in the census so the peak occupancy and the character count come from the SAME
+ // reading; two reads either side of a destructive action disagree.
assistant_chars: D.assistantChars(),
viewport_scroll_height: (D.viewport() || {}).scrollHeight || null,
viewport_client_height: (D.viewport() || {}).clientHeight || null,
- // ── DOES THE THREAD STILL FOLLOW THE STREAM? ──────────────
- //
- // Three readings, taken with every census, so the answer exists for every window of the
- // film rather than being reconstructed afterwards from timings.
- //
- // This is here because of a specific way a virtualized arm can produce a beautiful and
- // meaningless frame rate. If the thread stops following, the message being streamed
- // drifts out of the viewport; a windowed list then UNMOUNTS it, and the streaming cost
- // collapses to almost nothing. That is not a measurement of virtualization, it is a
- // measurement of not rendering the thing being measured, and it flatters the arm in
- // exactly the direction this campaign keeps having to catch. An fps number from such a
- // run must never be readable without this beside it.
- //
- // `app_at_bottom` is the app's OWN state, not arithmetic: thread.tsx renders the
- // scroll-to-bottom control permanently and hides it with `invisible` exactly when
- // use-intent-aware-autoscroll considers itself at the bottom. `distance_from_bottom` is
- // kept alongside because a virtualizer working from estimated row heights can sit a few
- // pixels off while the app is quite correctly pinned.
+ // DOES THE THREAD STILL FOLLOW THE STREAM? Three readings taken with every census, so the
+ // answer exists for every window rather than being reconstructed from timings. If the thread
+ // stops following, the streamed message drifts out of the viewport, a windowed list UNMOUNTS
+ // it, and the streaming cost collapses to almost nothing: a beautiful frame rate that measures
+ // not rendering the thing being measured. `app_at_bottom` is the app's OWN state (thread.tsx
+ // hides the scroll-to-bottom control with `invisible` exactly when use-intent-aware-autoscroll
+ // considers itself at the bottom); `distance_from_bottom` sits alongside because a virtualizer
+ // working from estimated row heights can be a few pixels off while correctly pinned.
viewport_scroll_top: (D.viewport() || {}).scrollTop || null,
distance_from_bottom: D.distanceFromBottom(),
app_at_bottom: D.appSaysAtBottom(),
@@ -530,8 +414,8 @@
return out;
},
- // Characters of assistant text currently in the DOM. Used for the seeded-vs-streamed
- // equivalence check and for chars-per-span.
+ // Characters of assistant text currently in the DOM, for the seeded-vs-streamed equivalence
+ // check and chars-per-span.
assistantChars() {
let n = 0;
for (const m of qa('[data-role="assistant"]')) n += (m.textContent || "").length;
@@ -541,38 +425,21 @@
window.__sb.dom = D;
- // ── the follow-the-stream sampler ──────────────────────────────────────────────────────────
- //
- // WHY THIS IS NOT DONE FROM THE DRIVER. The question is "while a reply was streaming, was the
- // thread pinned to the bottom", and the stream runs during the gap windows -- which are
- // measured windows whose whole purpose is to observe the page doing nothing but stream. A
- // `page.evaluate` per sample would put a CDP round trip and a forced style read inside exactly
- // those windows, four times a second, which is the same class of mistake as the census that
- // used to run inside the action windows.
- //
- // So it samples in the page and is READ ONCE PER CELL, outside every window. A 250ms timer,
- // four ticks a second: frames.js documents a 1ms timer as costing nothing at ~150 ticks a
- // second, so this is two orders of magnitude below that. It does no layout it has not already
- // caused -- `classList.contains` is free, and `distanceFromBottom` is only read when a run is
- // in progress.
- //
- // WHAT IT IS FOR. A thread that stops following unmounts the message being streamed, and the
- // streaming cost then collapses because the renderer is no longer rendering the thing under
- // measurement. That produces a superb frame rate and means nothing. `pinned_fraction` is the
- // reading that makes an fps number from a windowed arm readable at all.
- // THE COUNTERS SURVIVE A NAVIGATION, via sessionStorage.
- //
- // They did not, and the symptom was a confident "NOT MEASURED" on the arm that behaved best.
- // The film ends with `thread_reopen`, which falls back to `page.goto` when the New chat control
- // is covered; a full document navigation destroys the JS context and re-runs the init scripts,
- // so a sampler that lived only in memory came back at zero and the whole cell's streaming
- // phase was reported as never sampled. The treatment arm, whose thread_reopen did not run at
- // all, kept its counters and looked like the only arm with data.
- //
- // sessionStorage is per-origin and per-tab and outlives a same-origin navigation, which is
- // exactly the lifetime wanted: one cell, one page, however many documents it passes through.
- // Saved on pagehide rather than on every tick, so the cost is one serialisation per navigation
- // rather than four a second.
+ // WHY THIS IS NOT DONE FROM THE DRIVER: the stream runs during the gap windows, whose whole
+ // purpose is to observe the page doing nothing but stream, and a `page.evaluate` per sample
+ // would put a CDP round trip and a forced style read inside them four times a second. So it
+ // samples in the page and is READ ONCE PER CELL, outside every window: a 250ms timer, two
+ // orders of magnitude below the 1ms timer frames.js documents as free, doing no layout it has
+ // not already caused. `pinned_fraction` is what makes an fps number from a windowed arm
+ // readable at all.
+ // That timer runs at ~150 ticks a second.
+ // THE COUNTERS SURVIVE A NAVIGATION, via sessionStorage. They did not, and the symptom was a
+ // confident "NOT MEASURED" on the arm that behaved best: the film ends with `thread_reopen`,
+ // whose `page.goto` fallback destroys the JS context and re-runs the init scripts, so an
+ // in-memory sampler came back at zero while the treatment arm, whose thread_reopen did not
+ // run, kept its counters and looked like the only arm with data. sessionStorage is per-origin
+ // and per-tab and outlives a same-origin navigation, exactly the lifetime wanted. Saved on
+ // pagehide rather than per tick.
const FOLLOW_KEY = "__sb_follow_v1";
const restore = () => {
try {
@@ -593,74 +460,50 @@
yanked_back_samples: 0,
stream_samples: 0,
reattachments: 0,
- // Set once the thread is seen to fall behind while a run is in progress, and never cleared.
- // A thread that drifts away and is later yanked back to the bottom has still failed the
- // contract, and an end-of-cell reading would show it pinned.
+ // Set once the thread is seen to fall behind while a run is in progress, and never cleared: a
+ // thread that drifts away and is later yanked back has still failed the contract, and an
+ // end-of-cell reading would show it pinned.
ever_fell_behind: false,
}, restore() || {});
window.addEventListener("pagehide", () => {
try {
window.sessionStorage.setItem(FOLLOW_KEY, JSON.stringify(F));
} catch (e) {
- // A full sessionStorage is not worth losing the page over; the reading degrades to
- // "not measured", which is the honest outcome and is already handled.
+ // A full sessionStorage is not worth losing the page over; the reading degrades to "not
+ // measured", which is already handled.
}
});
const FOLLOW_TICK_MS = 250;
- // How far from the bottom still counts as following. Generous on purpose: a virtualizer
- // working from estimated row heights corrects them as real rows are measured, so it can sit a
- // little short of the exact bottom while behaving perfectly. 64px is under two lines of text,
- // so it cannot hide a thread that has stopped following.
+ // How far from the bottom still counts as following. Generous on purpose, since a virtualizer
+ // working from estimated heights can sit short of the exact bottom while behaving perfectly;
+ // 64px is under two lines, so it cannot hide a thread that has stopped following.
const FOLLOW_TOLERANCE_PX = 64;
- // TWO PHASES, BECAUSE THE INTENT CONTRACT HAS TWO HALVES.
- //
- // The contract (plans/proud-wiggling-falcon.md): autoscroll follows a stream, AND a user who
- // has scrolled up is never yanked down. Those are opposite requirements about the same
- // scrollTop, so one number cannot score both -- and the first version of this sampler tried,
- // producing 47% to 50% pinned on BOTH arms with an identical 6,615px worst drift. That figure
- // is the film: `scroll_during_generation` drags the viewport thousands of pixels up, twice,
- // while the reply is still arriving, and the app then CORRECTLY declines to drag it back. The
- // sampler was recording the app honouring the second half of the contract and scoring it as a
- // failure of the first.
- //
- // ATTACHED (before the harness scrolls anywhere): the thread must stay pinned as content
- // arrives. This is "does it follow", and it is what the gate scores.
- // DETACHED (after the harness has deliberately scrolled): the thread must NOT come back to the
- // bottom on its own. Any sample that finds it pinned again while still streaming is the app
- // yanking the user down, which is the other half of the contract and is recorded as its own
- // finding rather than mixed into the fraction.
+ // TWO PHASES, BECAUSE THE INTENT CONTRACT HAS TWO HALVES (plans/proud-wiggling-falcon.md):
+ // autoscroll follows a stream, AND a user who has scrolled up is never yanked down. One number
+ // cannot score both: the first version reported 47-50% pinned on both arms with an identical
+ // 6,615px worst drift, which is the film, since `scroll_during_generation` drags the viewport
+ // thousands of pixels up twice and the app correctly declines to drag it back, so the sampler
+ // scored the second half of the contract as a failure of the first. ATTACHED (before the
+ // harness scrolls) is "does it follow" and is what the gate scores; DETACHED (after a
+ // deliberate scroll) is "it must not come back on its own", recorded as its own finding.
let detached = false;
let suspended = 0;
- // WHICH RUN THE USER SCROLLED AWAY FROM, and it is the difference between a yank and a return.
- //
- // `resume()` below clears `detached` only when the gesture itself ended at the bottom. On the
- // real films it never does: `SCROLL_JS` jumps to the bottom and then steps 14 x 420px away from
- // it, so at any rung whose thread is taller than 5,880px the gesture ends thousands of pixels
- // up and `detached` is latched for the rest of the cell. The film then starts TWO MORE RUNS of
- // its own -- `stop_generation` and `send_turn` both submit a turn -- and the app pins to the
- // bottom for them, which session.py already documents as intended behaviour rather than a
- // violation. Every one of those samples was landing in the detached branch, counted as a yank,
- // and excluded from `attached_fraction_of_stream`.
- //
- // Measured, at head, across every 100K payload in outputs/: attached_fraction 0.07 to 0.15 with
- // reattachments 0, on the BASE arm as well as the treatment and on pure null controls, so the
- // `follows_the_stream` gate failed every 100K cell of every run including two copies of the
- // shipped build. It passed only on the 1K smoke film, where the thread is short enough that the
- // gesture's reversal happens to land back at the bottom -- i.e. the verdict was a property of
- // the thread's height, not of the app.
- //
- // So a run the user STARTED is a fresh expression of intent to be at the end, exactly as
- // returning to the bottom is. Re-attachment is granted only when a run that began AFTER the
- // gesture is also OBSERVED at the bottom: an app that declines to pin stays detached and is
- // scored as before, and a pin during the SAME run the user scrolled away from is still a yank.
+ // WHICH RUN THE USER SCROLLED AWAY FROM, the difference between a yank and a return.
+ // `resume()` clears `detached` only when the gesture ended at the bottom, which on the real
+ // films it never does: `SCROLL_JS` steps 14 x 420px away, so above 5,880px `detached` latches
+ // for the rest of the cell, and the film then starts two more runs whose intended pinning was
+ // counted as a yank. Measured at head across every 100K payload: attached_fraction 0.07 to
+ // 0.15 with zero reattachments, on the BASE arm and on pure null controls, so
+ // `follows_the_stream` failed every 100K cell of every run and passed only on the 1K film. So
+ // a run the user STARTED is a fresh expression of intent to be at the end: re-attachment is
+ // granted only when a run that began AFTER the gesture is also observed at the bottom.
let runSeq = 0;
let wasRunning = false;
let detachedAtRun = 0;
setInterval(() => {
F.samples += 1;
// Read BEFORE the suspended early-return, or a run that starts and ends inside a deliberate
- // gesture is never seen and the run after it is mistaken for the one the user scrolled away
- // from.
+ // gesture is never seen and the run after it is mistaken for the one scrolled away from.
const running = D.isRunning();
if (!running) wasRunning = false;
else if (!wasRunning) { wasRunning = true; runSeq += 1; }
@@ -668,7 +511,7 @@
if (!running) return;
const app = D.appSaysAtBottom();
const distance = D.distanceFromBottom();
- // "At the bottom" by the app's own answer, with the geometry standing in only when the build
+ // "At the bottom" by the app's own answer, with geometry standing in only when the build
// renders no jump control. `app === false` is the app saying no and is never overridden.
const atBottom =
app === true || (app === null && distance !== null && distance <= FOLLOW_TOLERANCE_PX);
@@ -705,50 +548,36 @@
window.__sb.follow = {
// Called by any action that moves the viewport on purpose. Nested-safe, because more than one
// scope may legitimately be open at once.
- // Called by any action that moves the viewport on purpose. Nested-safe. The FIRST suspend
- // also latches `detached`: from that moment the user has expressed an intent to be somewhere
- // other than the bottom, and everything after it is scored against the second half of the
- // contract instead of the first.
- // `detachedAtRun` is stamped on every suspend rather than only the first, so a second gesture
- // during a later run detaches from THAT run: without it, scrolling away during run 3 would be
- // compared against run 1 and the very next at-bottom sample would re-attach.
+ // The FIRST suspend also latches `detached`: from then the user has expressed an intent to be
+ // somewhere other than the bottom, and everything after is scored against the second half of
+ // the contract. `detachedAtRun` is stamped on every suspend rather than only the first, so a
+ // second gesture during a later run detaches from THAT run.
suspend() { suspended += 1; detached = true; detachedAtRun = runSeq; },
resume() {
suspended = Math.max(0, suspended - 1);
- // RE-ATTACH IF THE GESTURE LEFT US AT THE END, and this is not a nicety.
- //
- // `detached` used to latch on the first suspend and never clear, so from the harness's
- // first deliberate scroll onwards every remaining sample went to the detached branch and
- // `running_samples` stopped growing. In the shipped film that scroll happens 1.5s into an
- // 18s opening stream and is followed by two more streamed turns, so the follow verdict was
- // computed from the first ~3s and covered 13% of the streaming time -- while reading, and
- // being reported as, "the thread follows the stream". Measured: running_samples 11,
- // detached_samples 72.
- //
- // The contract is about INTENT, and intent is re-expressed by coming back: a user who
- // scrolls up is detached until they return to the end, at which point they are following
- // again. That is the same re-attachment Unsloth's own intent-aware autoscroll implements.
- // Only checked here, on the way out of a deliberate gesture, so the app silently pulling
- // the viewport down on its own is still scored as a yank rather than laundered into a
- // re-attachment.
+ // RE-ATTACH IF THE GESTURE LEFT US AT THE END, and this is not a nicety. `detached` used to
+ // latch on the first suspend and never clear, so from the harness's first deliberate scroll
+ // every sample went to the detached branch: in the shipped film that scroll is 1.5s into an 18s
+ // opening stream, so the verdict came from the first ~3s and covered 13% of the streaming time
+ // while reporting "the thread follows the stream" (running_samples 11, detached_samples 72).
+ // The contract is about INTENT, and intent is re-expressed by coming back, exactly as
+ // Unsloth's own intent-aware autoscroll implements. Only checked on the way out of a
+ // deliberate gesture, so the app pulling the viewport down on its own is still a yank.
if (suspended === 0 && detached) {
const app = D.appSaysAtBottom();
const distance = D.distanceFromBottom();
- // EITHER answer is enough, and the geometry is not merely a fallback for a build with no
- // jump control. The control's `invisible` class is updated from a scroll LISTENER, and
- // scroll events are dispatched asynchronously, so a gesture that has just returned the
- // viewport to the end can reach this line while the class still says otherwise. Reading
- // the class alone loses the re-attachment to a race and quietly restores the old
- // never-reattach behaviour on exactly the fast gestures the harness performs.
- // `distanceFromBottom()` is computed from scrollTop and cannot be stale.
+ // EITHER answer is enough, and the geometry is not merely a fallback: the control's
+ // `invisible` class is updated from a scroll LISTENER and scroll events are dispatched
+ // asynchronously, so a gesture that has just returned the viewport to the end can reach this
+ // line while the class still says otherwise. `distanceFromBottom()` cannot be stale.
+ // `distanceFromBottom()` is computed from scrollTop.
if (app === true || (distance !== null && distance <= FOLLOW_TOLERANCE_PX)) {
detached = false;
F.reattachments += 1;
} else {
- // Still away from the end. Re-stamp against the run in flight NOW, not the one that was
- // in flight when the gesture began: a gesture long enough to span a run boundary would
- // otherwise look like a scroll away from the previous run and be re-attached by the very
- // first sample of the current one.
+ // Still away from the end. Re-stamp against the run in flight NOW, not the one in flight when
+ // the gesture began: a gesture spanning a run boundary would otherwise be re-attached by the
+ // very first sample of the current run.
detachedAtRun = runSeq;
}
}
@@ -766,16 +595,14 @@
yanked_back_samples: F.yanked_back_samples,
// The second half of the intent contract, as its own verdict.
yanked_after_scroll: F.yanked_back_samples > 0,
- // null, not 1.0, when nothing was ever sampled mid-run. A cell whose stream finished
- // before the first tick has not demonstrated that the thread follows; it has
- // demonstrated nothing, and 1.0 would read as a pass.
+ // null, not 1.0, when nothing was sampled mid-run: a cell whose stream finished before the
+ // first tick has demonstrated nothing, and 1.0 would read as a pass.
pinned_fraction: measured > 0 ? F.running_pinned / measured : null,
pinned_fraction_reason:
measured > 0 ? null : "no sample was taken while a reply was streaming",
- // HOW MUCH OF THE STREAM THIS VERDICT ACTUALLY COVERS. `pinned_fraction` is computed
- // over the attached phases only, so without this it can read 1.0 on a cell where the
- // thread was attached for three seconds of an eighteen-second stream. Reported beside it
- // so the one cannot be quoted without the other.
+ // HOW MUCH OF THE STREAM THIS VERDICT COVERS: `pinned_fraction` is computed over the attached
+ // phases only, so without this it can read 1.0 on a cell attached for three seconds of an
+ // eighteen-second stream.
stream_samples: F.stream_samples,
attached_fraction_of_stream:
F.stream_samples > 0 ? F.running_samples / F.stream_samples : null,
diff --git a/tests/studio/studiobench/scene/parity.js b/tests/studio/studiobench/scene/parity.js
index b1ce91a7b7d..486ead53cf7 100644
--- a/tests/studio/studiobench/scene/parity.js
+++ b/tests/studio/studiobench/scene/parity.js
@@ -1,114 +1,38 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
// UI PARITY: a structural signature of the rendered thread, taken at the close of every action
// window on BOTH arms of an A/B.
-//
-// WHY THIS AND NOT SCREENSHOTS. The screenshot parity pair (scripts PR 217) is the right artefact
-// for a human reader of a pull request: it shows the surface, side by side, and a person checks it
-// against `expect`. What it cannot do is breadth. Each pair costs two isolated installs, shows one
-// surface, and needs somebody to open the composite. Eighteen PRs across eighteen action surfaces
-// is 324 composites and nobody will look at them.
-//
-// This runs inside a comparison the tool is ALREADY doing. studiobench drives both arms through
-// the same eighteen scripted actions against a byte-identical seeded thread inside one session, so
-// the two DOMs are already meant to agree. Digesting them costs one `evaluate` per slot and turns
-// "the UI is unchanged" from a claim into a per-action assertion. It is the cheap wide net; the
-// screenshot pair stays the deep, human-readable one for the surfaces that matter most.
-//
-// ── WHAT THIS DIGEST CAN AND CANNOT SEE ──────────────────────────────────────────────────────
-//
-// Read this before quoting a passing parity check at anybody. The digest is a serialisation of the
-// DOM, so its sensitivity is exactly the DOM and nothing else.
-//
-// IT SEES: tag structure and nesting, element order, every attribute name, every attribute value
-// that is not on the volatile list below (so `class`, `data-slot`, `data-state`, `data-role`,
-// `aria-hidden`, `hidden`, `disabled`, `role`, `title`, `alt`, `placeholder`), and all text content
-// after time normalisation. Added elements, removed elements, reordered siblings, retitled buttons
-// and flipped states all move the digest.
-//
-// IT DOES NOT SEE, and a green parity check asserts NOTHING about any of these:
-// - CSS coming from a stylesheet. A class name is compared as a STRING. If `.aui-md-p` changes
-// its margin in a `.css` file, every class list is identical and the digest is identical while
-// the page reflows. Only the bounded computed-style probe below sees any of this, and it sees
-// three properties on at most a few dozen elements.
-// - Computed layout: geometry, sizes, positions, scroll extents, whether anything overflows or
-// is clipped, whether two elements overlap.
-// - Colour, typography, borders, shadows, opacity, transforms, animation.
-// - Raster content: images, canvas, video, SVG paint. An `<img>` is compared by its attributes,
-// and its `src` is normalised (see below), so a DIFFERENT PICTURE at the same normalised URL
-// is invisible.
-// - Anything outside the thread root and the overlay selectors: the sidebar, the header, toasts.
-// - Anything the DOM does not carry at all: focus ring appearance, caret position, text
-// selection, scroll position (deliberately normalised), `<input>` live values typed by a user
-// (the DOM attribute is not the property), shadow DOM, and any state kept only in JS.
-//
-// THAT LIST HAS NOW BEEN MEASURED RATHER THAN JUST ASSERTED, and the result is worse than the
-// wording above suggests. A concurrent campaign measuring the sidebar drag ran this digest against
-// a real, visible change and got 0 of 34 differing pairs -- and its null control also returned 0 of
-// 34, so the instrument was not discriminating in either direction on that change. Three
-// purpose-built captures (sidebar-inclusive structure, sidebar inline style, custom-property reach)
-// each found the same change 34 of 34 with the null at zero. The two blind spots that mattered were
-// the two named above: the sidebar is outside the root, and computed layout is never read.
-//
-// So a PARITY OK verdict from this file means NO THREAD-STRUCTURE CHANGE WAS DETECTED. It does not
-// mean the UI is unchanged, and the two are far enough apart that the second should never be
-// written down on the strength of the first.
-//
-// Extending this into a real visual diff is the screenshot pair's job, not this one's. The point
-// of the digest is breadth at near-zero cost per action, and its limits are stated here so nobody
-// reads "18 actions, 0 differences" as "the UI is pixel-identical". It is not that claim.
-//
-// ── AND A NOTE ON ANY SCAN THAT CAN RETURN ZERO ──────────────────────────────────────────────
-//
-// `styleProbe` below walks a hand-written selector list. If Unsloth renames a class the list goes
-// quiet, matches nothing, and its digest becomes the hash of an empty string -- identical on both
-// arms, reported as a MATCH. A scan of nothing must never be reported as agreement, so
-// `compare_styles` refuses a zero-element probe instead of matching it, and `elements` travels
-// with every reading so the count can be checked rather than assumed.
-//
-// The same team hit the general form of this and it is worth recording: their CSSOM scan returned
-// a clean zero because CSS nesting gives every `CSSStyleRule` a truthy but empty `cssRules`, so
-// code that treats a truthy `cssRules` as "this is a grouping rule, recurse" silently skips every
-// declaration in the document. They caught it only because they had gated on a positive control.
-// Nothing in this file walks the CSSOM today, so that specific bug is not present here -- but any
-// scan added later that can legitimately return zero needs a positive control, and a zero without
-// one should not be believed.
-//
-// ── WHAT IS NORMALISED AWAY, and why each one has to be ──────────────────────────────────────
-//
-// A digest that trips on things that legally differ between two runs of the SAME build is worse
-// than no digest: it trains you to ignore it. Every entry here was OBSERVED to differ in a
-// base-vs-base null control; nothing is normalised on suspicion.
-//
-// generated ids Radix mints `radix-:r1a:` per mount, and aria-controls/labelledby point at
-// them. They differ between two runs of one build.
-// rendered durations the action bar prints things like `295ms`. PR 217 hit exactly this on
-// unslothai/unsloth#9054: a 295 vs 310 difference that is wall clock, not
-// content. Any number immediately followed by a time unit is collapsed.
-// relative times "2 minutes ago" moves on its own.
-// scroll state `--aui-scroll-stabilizer` and transform offsets encode where the thread
-// happens to be scrolled, which the film varies deliberately.
-// record identifiers thread ids, message ids and attachment ids are minted by the BACKEND when
-// the fixture seeds the thread. The two arms of an A/B are two separate
-// Unsloth installs with two separate databases, so these can never agree and
-// carry no information about the frontend.
-// absolute URLs `src`/`href` carry the arm's own origin, and the two arms are on two
-// ports by construction. `http://127.0.0.1:5830/x` and `...:5831/x` are the
-// same asset. Origins are stripped; the PATH is kept and still compared.
-// blob and data URLs an uploaded image is addressed by a per-mount `blob:` UUID.
-//
-// WHAT IS DELIBERATELY KEPT. Tag structure, `data-slot`, `data-state`, `data-role`, `aria-hidden`,
-// `hidden`, `disabled`, the class list, and all text content. `data-state` in particular: whether
-// a reasoning pane reads open or closed IS the UI, and a perf change that quietly flips it is the
-// exact defect this is meant to catch.
+// Cheap breadth, unlike the screenshot parity pair: one `evaluate` per slot inside a comparison
+// the tool already runs, versus 324 composites nobody will open.
+// IT SEES: tag structure and nesting, element order, attribute names, non-volatile attribute
+// values (class, data-slot, data-state, data-role, aria-hidden, hidden, disabled, role, title,
+// alt, placeholder) and all text after time normalisation.
+// IT DOES NOT SEE: stylesheet CSS (a class name compares as a string), computed layout and
+// geometry, colour, typography and animation, raster content behind a normalised src, anything
+// outside the thread root, and anything the DOM does not carry (focus, caret, selection, scroll
+// position, live input values, shadow DOM, JS state).
+// Measured, not asserted: against a real visible sidebar-drag change this digest found 0 of 34
+// differing pairs with its null control also at 0, while three purpose-built captures found it
+// 34 of 34. The blind spots that mattered were the sidebar being outside the root and computed
+// layout never being read.
+// So a PARITY OK verdict means NO THREAD-STRUCTURE CHANGE WAS DETECTED. It does not mean the UI is unchanged.
+// Any scan that can return zero needs a positive control: `styleProbe` walks a hand-written
+// selector list, so a renamed class makes it hash the empty string identically on both arms.
+// `compare_styles` refuses a zero-element probe, and `elements` travels with every reading.
+// Everything normalised below was OBSERVED to differ in a base-vs-base null control, nothing on
+// suspicion: Radix generated ids, rendered durations (unslothai/unsloth#9054), relative times,
+// scroll state, backend-minted record ids, absolute URLs (two ports; the path is kept), and
+// blob/data URLs.
+// A 295 vs 310 difference that is wall clock, not a build change.
+// DELIBERATELY KEPT: tag structure, data-slot, data-state, data-role, aria-hidden, hidden,
+// disabled, the class list and all text. A perf change that quietly flips a reasoning pane's
+// data-state is the exact defect this catches.
(() => {
if (window.__sb && window.__sb.parity) return;
window.__sb = window.__sb || {};
- // Attributes whose VALUE is volatile between two runs of the same build. The attribute's
- // presence is still recorded; only the value is dropped.
+ // Attributes whose VALUE is volatile between two runs of the same build; the presence is still recorded.
const VOLATILE_ATTRS = new Set([
"id", "for", "aria-controls", "aria-labelledby", "aria-describedby", "aria-activedescendant",
"style", "data-radix-scroll-area-viewport", "data-testid-instance",
@@ -120,14 +44,12 @@
// Attributes that carry no rendered meaning at all.
const IGNORED_ATTRS = new Set(["data-react-checksum", "data-reactroot"]);
- // VIRTUALIZATION BOOKKEEPING. Dropped -- name and value -- from the VISIBLE-REGION digest only,
- // and passed in by that caller rather than applied here, because the two comparisons want
- // different things from these two attributes. The whole argument is at the call site, in
- // `parityVisible.capture()`.
+ // VIRTUALIZATION BOOKKEEPING, dropped name and value from the VISIBLE-REGION digest only and
+ // passed in by that caller; the whole argument is at `parityVisible.capture()`.
const VIRTUALIZATION_ATTRS = new Set(["aria-posinset", "aria-setsize"]);
- // Attributes whose value is a URL. The origin is stripped and the path kept, so a genuinely
- // different asset still moves the digest while the arm's own port does not.
+ // URL-valued attributes: the origin is stripped and the path kept, so a genuinely different asset
+ // still moves the digest while the arm's own port does not.
const URL_ATTRS = new Set(["src", "href", "srcset", "action", "poster", "data-src", "formaction"]);
// A UUID, or a long hex run. Both are how this app spells "an id the server just made up".
@@ -155,8 +77,7 @@
.replace(/\s+/g, " ")
.trim();
- // FNV-1a, 32 bit, expressed as unsigned hex. Not cryptographic: this is a change detector, and
- // the pair being compared is two runs of one page rather than an adversary.
+ // FNV-1a, 32 bit, unsigned hex. Not cryptographic: a change detector over two runs of one page.
const hash = (str) => {
let h = 0x811c9dc5;
for (let i = 0; i < str.length; i++) {
@@ -172,29 +93,19 @@
return normText(raw);
};
- // `dropAttrs`, when given, is a Set of attribute names this digest does not compare AT ALL --
- // neither presence nor value, unlike VOLATILE_ATTRS which keeps the presence. Optional and off
- // by default: every existing caller (the whole-thread digest, the per-message rows, the overlays
- // and the node-driven unit tests) passes nothing and gets exactly what it got before.
- //
- // `elide`, when given, is a Set of ELEMENTS whose subtree this digest does not serialise. It is
- // not a way of ignoring them: a marker carrying the tag and the element's `data-role` is written
- // in their place, so the element's PRESENCE, its position among its siblings and its role are
- // still compared and a message that vanished still moves the digest. Only the content inside it
- // is withheld. The one caller is the streamed-message elision in `capture()`; the whole argument
- // for it is stated there. Optional and off by default, so every existing caller passes nothing
- // and gets byte-for-byte what it got before.
+ // `dropAttrs` is a Set of attributes this digest does not compare AT ALL, unlike VOLATILE_ATTRS
+ // which keeps the presence. `elide` is a Set of ELEMENTS whose subtree is not serialised: a marker
+ // carrying the tag and `data-role` goes in their place, so presence, position and role still
+ // compare and a vanished message still moves the digest. Both off by default.
const signature = (root, dropAttrs, elide) => {
if (!root) return "";
const parts = [];
const walk = (el, depth) => {
- // A depth cap is a TRUNCATION, and a truncated signature that reads like a complete one is
- // exactly the silent false negative this instrument exists to rule out. The marker is left
- // in the signature so anything deeper is visible as "not walked" rather than as "absent".
+ // A depth cap is a TRUNCATION, so the marker is left in the signature and anything deeper reads
+ // as "not walked" rather than as absent.
if (depth > 40) { parts.push("<!depth-cap>"); return; }
if (elide && elide.has(el)) {
- // Named, not silent. A reader of the raw signature sees exactly which element was held
- // back and why, rather than a hole they have to infer from a length.
+ // Named, not silent: a reader of the raw signature sees which element was held back and why.
parts.push(
"<!in-flight " + el.tagName.toLowerCase() +
" role=" + ((el.getAttribute && el.getAttribute("data-role")) || "?") + ">"
@@ -217,21 +128,11 @@
}
}
parts.push(">");
- // ADJACENT TEXT NODES ARE JOINED BEFORE THEY ARE NORMALISED, and this is the difference
- // between a normaliser that works and one that looks like it works.
- //
- // React renders `Thought for {n} seconds` as THREE text nodes -- "Thought for ", "3",
- // " seconds" -- because the interpolation is its own child. Normalising each node on its
- // own, `normText("3")` sees a bare digit with no unit after it, the rendered-duration rule
- // cannot match, and a wall-clock number reaches the digest. Two runs of a BYTE-IDENTICAL
- // build then disagree by one character, which is exactly what a null control here measured:
- // `msg22(assistant):17334->17334c`, same length, `3` against `2`, hunted down to this line
- // by `sweep/parity_null_control.py --hunt`. It was the single largest source of false
- // alarms in that control, and every one of them read as "this pull request changed the UI".
- //
- // The run is joined RAW and normalised once, so what gets normalised is the text as the
- // page renders it rather than the accident of where React split it. Element boundaries
- // still break a run, so this cannot weld two separate labels into one string.
+ // ADJACENT TEXT NODES ARE JOINED BEFORE THEY ARE NORMALISED. React renders `Thought for {n}
+ // seconds` as three text nodes, so per-node normalisation sees a bare digit, the rendered-duration
+ // rule cannot match, and a wall-clock number reaches the digest -- the largest single source of
+ // false alarms in the null control. Joined raw and normalised once.
+ // Found by `sweep/parity_null_control.py --hunt`.
let run = "";
const flush = () => {
if (!run) return;
@@ -254,19 +155,12 @@
return parts.join("");
};
- // The bounded computed-style probe. THREE properties on at most `STYLE_CAP` elements, and its
- // digest is reported SEPARATELY from the structural one rather than folded into it.
- //
- // Separate on purpose. `getComputedStyle` reads post-stylesheet, post-animation state, so it is
- // the only part of this file that can see a `.css` change at all -- and for the same reason it
- // is the part most likely to be caught mid-transition and differ between two runs of one build.
- // Folding it into `digest` would put the whole instrument's credibility on its least stable
- // reading. Kept apart, a structural mismatch stays a hard signal and a style mismatch is an
- // advisory the caller can weigh.
- //
- // `display` and `visibility` because they are how CSS makes something disappear without touching
- // the DOM. `pointer-events` because an overlay that stops or starts swallowing clicks is a real,
- // shipped class of defect here and leaves no structural trace at all.
+ // The bounded computed-style probe: three properties on at most `STYLE_CAP` elements, digested
+ // SEPARATELY from the structural signature. `getComputedStyle` is the only part of this file that
+ // can see a `.css` change, and the most likely to be caught mid-transition, so folding it in
+ // would stake the instrument's credibility on its least stable reading. `display` and
+ // `visibility` are how CSS hides something without touching the DOM; `pointer-events` because an
+ // overlay that swallows clicks leaves no structural trace.
const STYLE_CAP = 64;
const STYLE_PROPS = ["display", "visibility", "pointer-events"];
const STYLE_SELECTORS = [
@@ -294,89 +188,50 @@
}
}
const sig = parts.join("");
- // `capped` travels with the reading. A style digest taken over the first 64 of 300 elements
- // is a partial reading, and a partial reading that does not say so reads as a complete one.
+ // `capped` travels with the reading: a style digest over the first 64 of 300 elements is partial,
+ // and a partial reading that does not say so reads as a complete one.
return { digest: hash(sig), chars: sig.length, elements: n, capped, props: STYLE_PROPS, sig };
};
const D = () => (window.__sb.dom || {});
- // ── VISIBLE-REGION PARITY ────────────────────────────────────────────────────────────────────
- //
- // THE POLICY THIS SERVES. All changes must preserve UI and UX idempotency, with three
- // exemptions: a difference may be accepted deliberately when performance improves dramatically;
- // a difference that exists only OFF SCREEN is fine by definition, because rendering only what is
- // visible is an accepted technique rather than a parity violation; and a select-all need not
- // select all, PROVIDED the copy stays complete. Nothing in this file can see the third -- the
- // clipboard is scored behaviourally, in analysis/behaviour.py -- so a selection that shrank is
- // not a finding here.
- //
- // The whole-document digest above cannot express the second exemption. It compares everything
- // that is in the DOM, so ANY deferred off-screen work fails it by construction -- virtualization,
- // deferred syntax highlighting, content-visibility, lazy images. Refusing such a pair as
- // NOT_APPLICABLE was the right instinct and it withholds a verdict. This supplies the verdict.
- //
- // THE CLAIM IT SUPPORTS, exactly: every message that was visible in the viewport at any point
- // during the action is present on both arms and identical between them, and every difference
- // lies outside the viewport.
- //
- // ── TWO BOUNDARY DECISIONS, MADE EXPLICITLY ──────────────────────────────────────────────────
- //
- // These are where a visible-region check goes wrong quietly, so they are decided here in the
- // open rather than left implicit in whatever the code happens to do.
- //
- // 1. PARTIAL INTERSECTION COUNTS AS VISIBLE, AND THE ELEMENT IS DIGESTED IN FULL.
- // A message one pixel into the viewport is visible to the user. The alternative -- digesting
- // only the part inside the viewport -- is not definable on a DOM subtree without reading
- // geometry per node, and reading geometry is the one thing this must not do (see below). So
- // the whole element is compared. The error this admits is a FALSE ALARM: a difference in the
- // off-screen tail of a partly-visible message is reported as a visible difference. The error
- // it refuses to admit is a false pass. Given a parity gate, that is the right way round.
- //
- // 2. ANYTHING VISIBLE AT ANY POINT DURING THE ACTION IS COMPARED, not just at the end.
- // An action that scrolls makes messages visible and then hides them again; a single sample at
- // the close of the window would compare the DOM the scroll happened to land on and silently
- // ignore everything the user actually saw on the way. So the observer is installed BEFORE the
- // window opens and the compared set is the UNION of everything that ever intersected. The
- // per-message digest is still the one taken at the close, which is a real limitation and is
- // named as such in `now_visible` versus `ever_visible`: a message that was visible mid-action
- // and has since changed is compared in its final state.
- //
- // ── WHY INTERSECTIONOBSERVER AND NOT GEOMETRY ────────────────────────────────────────────────
- //
- // `getBoundingClientRect()` / `getClientRects()` on content inside a `content-visibility` locked
- // subtree makes Chromium render that subtree in order to answer, so a geometry-based visibility
- // probe unlocks exactly what it came to observe and then reports that nothing was skipped. That
- // is measured, not theoretical: one session reported 0 off-screen unrendered roots while the
- // event counter recorded 22 simultaneously in the skipped state. See the content-visibility trap
- // section of CONTRIBUTING-perf.md.
- //
- // IntersectionObserver is the correct instrument and not merely a safe one: it is the same
- // mechanism Blink's own relevance machinery uses to decide whether a `content-visibility: auto`
- // subtree is skipped, so observing with it neither forces rendering nor perturbs the decision.
- // NOTHING in this section may call a geometry method on a candidate element.
+ // THE POLICY: changes must preserve UI and UX idempotency, with three exemptions -- a dramatic
+ // performance win, a difference that exists only OFF SCREEN, and a select-all that need not
+ // select all provided the copy stays complete. The third is scored in analysis/behaviour.py.
+ // The whole-document digest cannot express the off-screen exemption: it compares everything in
+ // the DOM, so any deferred off-screen work fails it by construction. Refusing such a pair as
+ // NOT_APPLICABLE withholds a verdict; this supplies one.
+ // THE CLAIM: every message visible in the viewport at any point during the action is present on
+ // both arms and identical between them, and every difference lies outside the viewport.
+ // 1. PARTIAL INTERSECTION COUNTS AS VISIBLE and the element is digested IN FULL: digesting only
+ // the visible part would need per-node geometry, which this must not read. The admitted error is
+ // a FALSE ALARM, never a false pass.
+ // 2. ANYTHING VISIBLE AT ANY POINT IS COMPARED, not just at the end: the observer is installed
+ // BEFORE the window opens and the compared set is the UNION of everything that intersected. The
+ // per-message digest is still the one taken at the close, named `now_visible` vs `ever_visible`.
+ // GEOMETRY IS FORBIDDEN HERE: `getBoundingClientRect()` on content inside a `content-visibility`
+ // locked subtree makes Chromium render it, so a geometry probe unlocks exactly what it came to
+ // observe -- one session reported 0 off-screen unrendered roots while the counter recorded 22
+ // skipped. IntersectionObserver is what Blink's own relevance machinery uses.
+ // `getClientRects()` too.
const VIS = {
obs: null, mut: null, ever: new Set(), watching: false,
- // Nodes already counted in `unplaced`, so a row offered and refused several times before it
- // mounts does not inflate the diagnostic. NOT a "do not look at this again" set: placement is
- // re-read on every offer, because a recycled row changes position. See `observeOne`.
+ // Nodes already counted in `unplaced`, so a row refused several times before it mounts does not
+ // inflate the diagnostic. NOT a "do not look at this again" set: placement is re-read on every
+ // offer, because a recycled row changes position.
unplacedSeen: new WeakSet(),
- // Rows that were observed and could NOT be placed in the thread at all: no `aria-posinset`
- // and not among the messages the DOM currently holds. Such a row is stamped with no ordinal,
- // so it is silently absent from `ever_visible`, and a silence that cannot be counted is the
- // failure this whole file is written against. Reported with the capture.
+ // Rows observed that could NOT be placed in the thread at all: no `aria-posinset` and not among
+ // the messages the DOM holds. Such a row is stamped with no ordinal and so is silently absent
+ // from `ever_visible`, which is why it is reported with the capture.
unplaced: 0,
// The batch-scoped position index. See `positionIndex`.
index: null,
};
const ordinalOf = (el, position) => {
- // The message's position in the THREAD, not in the mounted list. This is what makes a windowed
- // arm comparable with a fully mounted one at all: mounted index 0 is message 10 on one arm and
- // message 1 on the other, so a per-index comparison compares different messages and reports
- // every row as changed. `aria-posinset` is the windowed arm's own claim about position; a
- // fully mounted thread does not publish it and does not need to, because there the DOM order
- // IS the thread order, which is what `position` carries.
+ // The message's position in the THREAD, not in the mounted list: mounted index 0 is message 10 on
+ // one arm and message 1 on the other, so a per-index comparison reports every row as changed.
+ // `aria-posinset` is the windowed arm's own claim; a fully mounted thread's DOM order IS it.
const owner = el.closest ? el.closest("[aria-posinset]") : null;
if (owner) {
const n = Number(owner.getAttribute("aria-posinset"));
@@ -385,41 +240,20 @@
return position;
};
- // THE POSITION INDEX, AND WHY THE FALLBACK ORDINAL IS NOT A COUNTER.
- //
- // A row publishing no `aria-posinset` is stamped with its position among the thread's messages
- // in the DOM AS IT STANDS. That has to be resolved when the row is OBSERVED rather than when an
- // entry is delivered, which is the reason the ordinal is stamped onto the node in the first
- // place: by delivery time the row may have been unmounted and `closest()` would answer nothing.
- //
- // A LIFETIME COUNT OF OBSERVED NODES IS NOT THAT POSITION, and the gap between them was a
- // measured failure rather than a tidiness point. `thread_reopen` makes a fully mounted arm
- // remove and recreate every message row inside one document. Those rebuilt rows legitimately
- // publish no `aria-posinset` -- only a windowed arm publishes one -- so a counter that had
- // already seen the thread's N rows stamped the rebuilt ones N+1..2N, while the windowed arm on
- // the other side of the A/B stamped its real 1..N. `compare_visible` then saw two disjoint
- // visible sets and reported "the two arms put DIFFERENT MESSAGES on screen" for a rebuild that
- // was identical, failing thread_reopen on every base-versus-windowed pair.
- //
- // THE COST, WHICH IS WHY A COUNTER WAS TEMPTING. `observeAdded` runs inside the MEASURED action
- // window, so an O(document) lookup per mutation is workspace task #102 exactly: the instrument
- // charging its own walk to the action, on a DOM whose size is the quantity under investigation.
- // Three things keep that walk off the per-mutation path:
- //
- // 1. `aria-posinset` is read FIRST, so a windowed arm -- the only kind that mounts rows by the
- // hundred as it scrolls -- never builds an index at all.
- // 2. The index is built at most ONCE PER MUTATION BATCH, and only by a batch that actually
- // mounted a message element. A stream is text churn inside rows that are already mounted:
- // it adds no elements, so it builds nothing, which is what
- // `test_the_top_up_is_proportional_to_the_mutation_not_to_the_document` pins.
- // 3. It is read from the live DOM inside the callback, i.e. after every mutation in the batch
- // has been applied, so ONE read describes the batch's settled state rather than some
- // intermediate one. A rebuild that removes N rows and adds N rows in one commit is read
- // once, and reads N rows rather than 2N.
- //
- // What is left paying for it is an arm publishing no ordinals that mounts rows mid-action:
- // send_turn's one or two new messages, and thread_reopen's single rebuild. That is a handful of
- // document reads per action window, in exchange for the ordinals being right.
+ // THE POSITION INDEX, AND WHY THE FALLBACK ORDINAL IS NOT A COUNTER. A row publishing no
+ // `aria-posinset` is stamped with its position among the thread's messages in the DOM as it
+ // stands, resolved when the row is OBSERVED, because by delivery time it may be unmounted.
+ // A LIFETIME COUNT OF OBSERVED NODES IS NOT THAT POSITION: `thread_reopen` recreates every message
+ // row inside one document, and rebuilt rows legitimately publish no ordinal, so a counter stamped
+ // them N+1..2N against the windowed arm's real 1..N and `compare_visible` reported two disjoint
+ // visible sets for an identical rebuild.
+ // THE COST, which is why a counter was tempting: `observeAdded` runs inside the MEASURED window,
+ // so an O(document) lookup per mutation is the trap. Three things keep that walk off the
+ // per-mutation path: `aria-posinset` is read FIRST, so a windowed arm never builds an index; the
+ // index is built at most once per mutation batch and only by a batch that mounted a message
+ // element; and it is read from the live DOM inside the callback, so a rebuild reads N not 2N.
+ // Workspace task #102. The per-mutation bound is held by
+ // test_the_top_up_is_proportional_to_the_mutation_not_to_the_document.
const positionIndex = () => {
if (VIS.index) return VIS.index;
const nodes = (D().messages && D().messages()) || [];
@@ -429,100 +263,64 @@
return map;
};
- //: `position` is the row's 1-based position among the thread's messages when the caller already
- //: knows it -- the full scan below holds the whole ordered list and needs no index. Left
- //: undefined, the position is resolved from the index, and only when no ordinal was published.
+ // `position` is the row's 1-based position among the thread's messages when the caller already
+ // knows it; left undefined it is resolved from the index, and only when no ordinal was published.
const observeOne = (el, position) => {
if (!VIS.obs) return;
let ord = ordinalOf(el, typeof position === "number" ? position : null);
if (ord === null) ord = positionIndex().get(el) || 0;
- // The ordinal is stamped on the node, because by the time an entry is delivered the node may
- // have been unmounted and `closest()` would return nothing.
+ // The ordinal is stamped on the node: by the time an entry is delivered the node may be unmounted
+ // and `closest()` would return nothing.
if (ord > 0) {
- // RE-READ RATHER THAN WRITE ONCE. A placed node used to be marked `seen` and skipped for the
- // rest of the window, which assumed a row's position in the thread cannot change while the
- // node lives. A virtualizer that RECYCLES rows breaks that assumption on purpose: it hands
- // the same node to another item and renumbers it. The stamp then stayed at the old ordinal
- // for the rest of the run, so the row's content was digested under a position it no longer
- // held, and the message it now showed was never reported visible at all.
- //
- // Re-reading is bounded work. The full scan runs once; every other call arrives from
- // `observeAdded`, which walks only what a mutation added, and `positionIndex` is built at
- // most once per batch.
+ // RE-READ RATHER THAN WRITE ONCE: a virtualizer that RECYCLES rows renumbers them, so a `seen`
+ // stamp stayed at the old ordinal, the row's content was digested under a position it no longer
+ // held, and the message it now showed was never reported visible. Re-reading is bounded: the full
+ // scan runs once and every other call walks only what a mutation added.
const had = el.__sbOrdinal;
el.__sbOrdinal = ord;
if (typeof had === "number" && had !== ord) {
- // A RENUMBERED ROW THAT NEVER STOPPED INTERSECTING REPORTS NOTHING. IntersectionObserver
- // delivers on a CHANGE of intersection, and this node's intersection did not change --
- // only its identity did -- so `ever` would never learn the new ordinal. Re-registering the
- // target makes the observer deliver an initial entry for it, which is the documented
- // behaviour of `observe()` and the only way to ask "is this thing on screen right now"
- // without calling a geometry method, which this section may not do.
+ // A RENUMBERED ROW THAT NEVER STOPPED INTERSECTING REPORTS NOTHING: IntersectionObserver delivers
+ // on a CHANGE of intersection and only this node's identity changed. Re-registering the target
+ // makes the observer deliver an initial entry, the only way to ask "is this on screen now"
+ // without calling a geometry method.
VIS.obs.unobserve(el);
}
} else {
- // NO ORDINAL RATHER THAN A MADE-UP ONE. A row that publishes nothing and is not in the
- // thread's message list has no position this instrument can honestly claim, and a guessed
- // one lands in `ever_visible` as a message the other arm never showed.
- //
- // NOT MARKED `seen`, WHICH IS THE WHOLE POINT. Marking it here is what made an unplaced row
- // able to hide a VISIBLE one. A row is unplaceable when it is observed while detached --
- // added and removed inside one task, so no paint could have shown it -- and a virtualizer
- // that RECYCLES DOM nodes hands that same node back on a later batch, mounted and about to
- // be shown. With the node already in `seen` the early return fired, it was never stamped,
- // and every intersection it went on to report was dropped for want of an ordinal: visible
- // on this arm, absent from `ever_visible`, and `compare_visible` free to call MATCH on the
- // strength of the rows that did place. Left out of `seen` it is simply placed on the batch
- // that mounts it, which is the batch that can place it.
- //
- // Counted once per node all the same. `unplacedSeen` is a separate WeakSet so a node that
- // is offered and refused several times before it mounts does not inflate the diagnostic
- // into a number nobody can interpret.
+ // NO ORDINAL RATHER THAN A MADE-UP ONE: a guessed position lands in `ever_visible` as a message
+ // the other arm never showed. NOT marked `seen`, which is the whole point -- a row observed while
+ // detached is handed back by a recycling virtualizer, and with the node already in `seen` every
+ // later intersection was dropped and `compare_visible` was free to call MATCH.
if (!VIS.unplacedSeen.has(el)) {
VIS.unplacedSeen.add(el);
VIS.unplaced += 1;
}
}
- // Idempotent by specification: `observe()` on a target already being observed returns without
- // adding a second registration, so re-offering an unplaced row costs nothing.
+ // Idempotent by specification: `observe()` on a target already being observed adds no second
+ // registration, so re-offering an unplaced row costs nothing.
VIS.obs.observe(el);
};
- //: The FULL scan. O(the document), so it runs exactly once, when the observer is installed --
- //: which happens before the measured window opens.
+ // The FULL scan. O(the document), so it runs exactly once, when the observer is installed, which
+ // is before the measured window opens.
const observeAll = () => {
if (!VIS.obs) return;
const nodes = (D().messages && D().messages()) || [];
for (let i = 0; i < nodes.length; i++) observeOne(nodes[i], i + 1);
};
- // THE TOP-UP, AND WHY IT IS NOT A RESCAN.
- //
- // A windowed list mounts rows as it scrolls, so rows appearing after the observer was installed
- // have to be picked up or a windowed arm is only ever asked about what it happened to have
- // mounted at the start. The obvious way to do that is to re-run the full scan from a
- // MutationObserver -- and the MutationObserver runs DURING the measured action window, so a
- // full `querySelectorAll` per mutation batch would charge an O(document) walk to the action,
- // once per batch, on a 64,000-element DOM, growing with exactly the quantity under
- // investigation. That is workspace task #102 verbatim: the census and the parity digest running
- // inside the window, which reported delete_message at 14.3 fps when it costs 49.0.
- //
- // So this walks only what was ADDED. `addedNodes` is small by construction -- a virtualizer
- // mounts one or two rows per scroll step -- and the cost is proportional to the mutation rather
- // than to the document. The one exception is stated in full at `positionIndex` above: an arm
- // that publishes no ordinals needs the thread's current message list to place a row it just
- // mounted, and that list is read ONCE for the whole batch and never once per row.
+ // THE TOP-UP, AND WHY IT IS NOT A RESCAN. A windowed list mounts rows as it scrolls, but
+ // re-running the full scan from a MutationObserver would charge an O(document) walk to the
+ // measured action once per batch on a 64,000-element DOM, which reported delete_message at 14.3
+ // fps where it costs 49.0. So this walks only what was ADDED, and the one document read an
+ // ordinal-less arm needs is taken once for the whole batch.
const observeAdded = (records) => {
- // ONE INDEX PER BATCH AT MOST, and none at all for a batch that mounts nothing needing one.
- // Dropped again on the way out so the next batch cannot be answered from a stale list: rows
- // mount and unmount between batches, and a position read from the previous DOM is precisely
- // the wrong answer -- the same class of mistake as the lifetime counter this replaced.
+ // ONE INDEX PER BATCH AT MOST, dropped on the way out so the next batch cannot be answered from a
+ // stale list: a position read from the previous DOM is precisely the wrong answer, the same
+ // class of mistake as the lifetime counter this replaced.
VIS.index = null;
for (const rec of records) {
- // A RENUMBERED ROW ARRIVES AS ITS OWN TARGET, not in anybody's `addedNodes`. The ordinal
- // lives on `[aria-posinset]`, which `ordinalOf` reaches with `closest()`, so the row that
- // has to be re-placed is the target's message descendant, or the target itself when the
- // attribute is on the message.
+ // A RENUMBERED ROW ARRIVES AS ITS OWN TARGET, not in anybody's `addedNodes`, so the row to
+ // re-place is the target's message descendant, or the target itself when the attribute is on it.
if (rec.type === "attributes") {
const t = rec.target;
if (!t || t.nodeType !== 1) continue;
@@ -539,8 +337,8 @@
if (node.hasAttribute && node.hasAttribute("data-role")) {
observeOne(node);
}
- // A row wrapper arrives with the message inside it, so the added node is the ancestor.
- // Bounded by the added subtree, never by the document.
+ // A row wrapper arrives with the message inside it, so the added node is the ancestor. Bounded by
+ // the added subtree, never by the document.
if (node.querySelectorAll) {
for (const inner of node.querySelectorAll("[data-role]")) {
observeOne(inner);
@@ -569,17 +367,11 @@
}
}, { root: vp, threshold: 0 });
observeAll();
- // childList, plus ONE attribute by name, and no character-data records. A row ENTERING the
- // DOM is most of what needs observing, and a row leaving it has already contributed to
- // `ever`. Text changing inside a mounted row -- which is what a stream is -- must not
- // reach this callback at all, and with `attributeFilter` naming a single attribute it
- // cannot: a stream mutates text, not `aria-posinset`.
- //
- // `aria-posinset` is here because it is the one attribute whose change means the row is a
- // DIFFERENT MESSAGE. A virtualizer that recycles a still-connected row renumbers it in
- // place, which is not a childList mutation, so without this the renumbering is invisible
- // and the node keeps a stamp that now names the wrong message. Filtered to that name the
- // callback fires when a windowed arm re-uses a row and at no other time.
+ // childList plus ONE attribute by name and no character-data records: text changing inside a
+ // mounted row -- which is what a stream is -- must not reach this callback. `aria-posinset` is
+ // here because its change means the row is a DIFFERENT MESSAGE, which is not a childList
+ // mutation when a virtualizer renumbers a still-connected row in place.
+ // `ordinalOf` reaches `[aria-posinset]` with `closest()`.
VIS.mut = new MutationObserver(observeAdded);
VIS.mut.observe(vp, {
childList: true,
@@ -599,22 +391,13 @@
if (!VIS.watching) {
return { visible_attempted: false, reason: "the visibility observer was never installed" };
}
- // WAIT FOR A FRAME FIRST, and this is not defensive padding.
- //
- // IntersectionObserver computes intersections as a step of the rendering lifecycle, so
- // with no frame between `observe()` and the read there is nothing to deliver and
- // `takeRecords()` returns an empty list. An action that scrolls produces frames and hides
- // this completely; a QUIET action -- open a menu, change a setting, type a character --
- // may not, and the capture then reports that the viewport showed nothing at all. The
- // analysis would correctly refuse such a pair as NOT_COMPARABLE, so it is not a false
- // pass, but it would silently strip visible-region coverage from exactly the actions that
- // are cheapest to get right. Observed directly: at rest, with no scroll, the observer
- // reported an empty set while message 1 filled the viewport.
+ // WAIT FOR A FRAME FIRST: IntersectionObserver computes intersections as a step of the rendering
+ // lifecycle, so with no frame between `observe()` and the read `takeRecords()` returns nothing.
+ // Observed directly, the observer reported an empty set while message 1 filled the viewport.
await new Promise((resolve) => {
requestAnimationFrame(() => requestAnimationFrame(resolve));
});
- // Then deliver anything still queued. Tearing down before this drops the last entries of
- // the action -- the ones most likely to describe what it just put on screen.
+ // Then deliver anything still queued: tearing down first drops the last entries of the action.
const records = VIS.obs.takeRecords();
for (const entry of records) {
if (entry.isIntersecting && typeof entry.target.__sbOrdinal === "number") {
@@ -623,25 +406,15 @@
}
const dom = D();
const nodes = (dom.messages && dom.messages()) || [];
- // THE SAME NODE SET THE POSITIVE CONTROL BELOW COUNTS. The rows used to re-walk the
- // `data-status` / `aria-busy` selectors inline, so "the rows read the same hooks as the
- // control" was true only while two copies happened to agree. Read from one call and the
- // control cannot fire while a row still claims to be in flight, or the reverse.
+ // THE SAME NODE SET THE POSITIVE CONTROL BELOW COUNTS: read from one call, so the control cannot
+ // fire while a row still claims to be in flight, or the reverse.
const live = new Set((dom.streamingMessages && dom.streamingMessages()) || []);
const byOrdinal = {};
let mounted_ever_visible = 0;
- // TWO MOUNTED ROWS CANNOT BE ONE MESSAGE, and until this counter existed nothing said so.
- // The assignment below is keyed by ordinal, so a second row reusing one SILENTLY REPLACED
- // the first row's digest, and `VIS.ever` is a Set of numbers so it collapsed them too.
- // Whether the ghost row was caught was pure DOM order: reproduced in a browser, the same
- // ghost inserted BEFORE the real row returns MATCH and AFTER it returns DIFFER.
- //
- // COUNTED DIRECTLY, NOT DERIVED. `unmounted_at_capture` looks like it should notice, but
- // the extra row and the vacancy cancel in exactly the renumber case, so it reads a clean 0
- // over a live collision.
- //
- // The residual: a collision that had resolved to one row by the time `capture()` ran is
- // not detectable here. Seeing it needs the clash recorded where the ordinal is stamped.
+ // TWO MOUNTED ROWS CANNOT BE ONE MESSAGE: the assignment below is keyed by ordinal, so a second
+ // row reusing one silently replaced the first row's digest and `VIS.ever` collapsed them too.
+ // Counted directly, not derived: `unmounted_at_capture` reads a clean 0 over a live collision
+ // because the extra row and the vacancy cancel.
let ordinal_collisions = 0;
const collided = [];
for (let i = 0; i < nodes.length; i++) {
@@ -649,31 +422,12 @@
const ord = typeof el.__sbOrdinal === "number" ? el.__sbOrdinal : ordinalOf(el, i + 1);
if (!VIS.ever.has(ord)) continue;
mounted_ever_visible += 1;
- // WITHOUT THE VIRTUALIZATION BOOKKEEPING, and the asymmetry is the reason.
- //
- // runtime/readiness.py deliberately accepts `aria-posinset` / `aria-setsize` either on
- // the `[data-role]` message or on an ancestor row wrapper -- it walks with `closest()`,
- // because the ordinal belongs on whichever element is the member of the set. An arm that
- // takes the first option therefore carries two attributes on EVERY message that the
- // fully mounted arm publishes on none, so the per-message digests differed on every
- // message while the rendered content was identical, and visible-region parity reported a
- // wall of differences for a DOM shape the gate explicitly permits. That makes auto-mode
- // parity unusable for the very arm it exists to score.
- //
- // NOT normalised in the whole-document structural digest, which is a separate decision
- // and a deliberate one. That digest is only ever applied to pairs where NEITHER arm is
- // windowing (see `decide_modes`), and between two fully mounted arms an ordinal that
- // appears, disappears or changes IS a change worth seeing. `signature` is shared with
- // the thread digest, the per-message rows, the overlays and the node-driven unit tests,
- // so the exclusion is passed in HERE by the one caller that wants it rather than baked
- // into the function every caller uses.
- //
- // What this gives up: a windowed arm publishing a WRONG ordinal on a message is no
- // longer visible in this digest. It is not unchecked -- runtime/readiness.py gates on
- // the ordinals being real positions (at least 1, distinct, no larger than the declared
- // set size, reaching the seeded total) and `probe_thread_completeness` walks them for
- // holes -- and those are the checks that can say what a right ordinal would be, which a
- // digest comparison against an arm that publishes none never could.
+ // WITHOUT THE VIRTUALIZATION BOOKKEEPING: runtime/readiness.py accepts `aria-posinset` /
+ // `aria-setsize` on the message or on an ancestor row, so an arm taking the first option carries
+ // two attributes on every message the fully mounted arm publishes on none, and visible-region
+ // parity reported a wall of differences for a DOM shape the gate permits. NOT normalised in the
+ // whole-document digest, which only runs where neither arm windows, so the exclusion is passed in
+ // by this caller. A windowed arm publishing a WRONG ordinal is covered by the readiness gate.
const sig = signature(el, VIRTUALIZATION_ATTRS);
if (Object.prototype.hasOwnProperty.call(byOrdinal, String(ord))) {
ordinal_collisions += 1;
@@ -683,41 +437,25 @@
role: el.getAttribute("data-role") || "?",
digest: hash(sig),
chars: sig.length,
- // Still being written at capture time, so its digest names a point in a stream rather
- // than a rendering. Carried here for the same reason as in the structural capture and
- // handled the same way by `compare_visible`: residue, never a difference.
+ // Still being written at capture time, so its digest names a point in a stream rather than a
+ // rendering: residue, never a difference.
in_flight: live.has(el),
};
}
const ever = [...VIS.ever].sort((a, b) => a - b);
- // THE SAME POSITIVE CONTROL THE STRUCTURAL CAPTURE CARRIES, because this payload is scored
- // by `compare_visible` on its own and never consults the structural one. The per-row
- // `in_flight` above is taken from the SAME `streamingMessages()` call, so it goes quiet in
- // the same way and at the same moment, and the arms of an A/B are two DIFFERENT builds --
- // `--ab REF` installs the treatment under its own `UNSLOTH_STUDIO_HOME` -- so a renamed
- // hook on the treatment only is the asymmetric case rather than a symmetric one that
- // cancels out. Both rows then read
- // `in_flight: false`, one arm's reply is mid-tail and the other's has finished, and the
- // loop in `compare_visible` scores that as a rendering difference: the wall-clock false
- // alarm this whole change exists to remove.
- //
- // READ GLOBALLY, not over the visible rows. "Is a reply being written" and "is the message
- // it is being written into on screen" are different questions, and a reply streaming below
- // the fold is an ordinary state that must not refuse anything.
+ // THE SAME POSITIVE CONTROL THE STRUCTURAL CAPTURE CARRIES, because this payload is scored by
+ // `compare_visible` alone. The per-row `in_flight` comes from the SAME `streamingMessages()`
+ // call, and the two arms are two different builds, so a hook renamed on the treatment only is
+ // asymmetric rather than cancelling. READ GLOBALLY, not over the visible rows: a reply streaming
+ // below the fold is ordinary and must not refuse anything.
+ // `--ab REF` installs the treatment under its own `UNSLOTH_STUDIO_HOME`.
const generating = dom.generating
? Boolean(dom.generating())
: Boolean(dom.isRunning && dom.isRunning());
- // WHY THE PROBE IS QUIET, which is three questions and not one. See
- // `dom.statusHookPresent` / `dom.lastAssistantPublishesStatus`.
- //
- // the last assistant message IS publishing parts and none of them says running
- // -> the hook's VOCABULARY changed. Blind, unless this arm is windowing, in which case
- // the message being written may not be the last one mounted and nothing here can
- // tell the two apart.
- // the last assistant message publishes NOTHING
- // -> it has no parts yet, which is ordinary -- unless no assistant message anywhere
- // publishes anything, and then the hook itself is gone. Blind on any arm, windowed
- // or not, because a settled message would still be carrying it.
+ // WHY THE PROBE IS QUIET is three questions, not one: parts published with none saying running
+ // means the hook's VOCABULARY changed, blind unless this arm is windowing; the last assistant
+ // message publishing nothing is ordinary unless no assistant message anywhere publishes, and
+ // then the hook itself is gone.
const lastPublishes = dom.lastAssistantPublishesStatus
? Boolean(dom.lastAssistantPublishesStatus())
: false;
@@ -727,30 +465,27 @@
return {
visible_attempted: true,
streaming: generating,
- // Carried so a reader can tell the two zero-in-flight readings apart in the record, not
- // only in the verdict: a stream this capture could not see, versus a hook that is gone.
+ // Carried so a reader can tell the two zero-in-flight readings apart in the record: a stream this
+ // capture could not see, versus a hook that is gone.
status_hook_present: anyPublishes,
- // `hookGone` and not merely "nothing is running": see `dom.statusHookPresent`. A windowed
- // arm scrolled away from the tail can UNMOUNT the message being written, and
- // `streamingMessages()` can only scan mounted DOM, so `live.size` is zero on a build
- // whose hooks are perfectly intact. That is the ordinary state this mode exists to
- // score, and refusing it would discard the settled rows that WERE on screen.
+ // `hookGone` and not merely "nothing is running": a windowed arm scrolled away from the tail can
+ // UNMOUNT the message being written, and `streamingMessages()` scans only mounted DOM, so
+ // `live.size` is zero on a build whose hooks are intact.
+ // See `dom.statusHookPresent`.
+ // And `dom.lastAssistantPublishesStatus`.
in_flight_unplaced: Boolean(generating && live.size === 0 && probeBlind),
- // Every ordinal the viewport ever showed during the window, INCLUDING any that have
- // since been unmounted. The gap between this and `messages` is the honest measure of
- // what a windowed arm could not be asked about at capture time.
+ // Every ordinal the viewport ever showed, including any since unmounted; the gap between this and
+ // `messages` measures what a windowed arm could not be asked about at capture time.
ever_visible: ever,
ever_visible_count: ever.length,
mounted_ever_visible,
unmounted_at_capture: ever.length - mounted_ever_visible,
- // Rows observed that this instrument could not place in the thread: no published
- // ordinal, and absent from the message list when they were observed. They carry no
- // ordinal and so appear nowhere above, which is a hole in the compared set rather than
- // agreement, and it is counted here so a reader can see it is zero.
+ // Rows this instrument could not place in the thread: no published ordinal and absent from the
+ // message list when observed. They appear nowhere above, which is a hole in the compared set
+ // rather than agreement, so it is counted here.
unplaced_rows: VIS.unplaced,
- // Mounted, ever-visible rows whose ordinal a row already written had. See the note in
- // the loop: the digest map and `ever_visible` both lose one of the two, so a nonzero
- // count here means neither of them can be compared.
+ // Mounted, ever-visible rows whose ordinal a row already written had: the digest map and
+ // `ever_visible` both lose one of the two, so a nonzero count means neither can be compared.
ordinal_collisions,
collided_ordinals: collided.sort((a, b) => a - b),
messages: byOrdinal,
@@ -772,103 +507,59 @@
};
window.__sb.parity = {
- // Exposed so the offline unit tests can drive the exact regexes that ship, rather than a
- // second copy of them that is free to drift.
+ // Exposed so the offline unit tests drive the exact regexes that ship rather than a second copy free to drift.
normText,
normUrl,
signature,
hash,
- // One digest for the whole thread, plus a digest PER MESSAGE so a mismatch says which message
- // moved rather than only that something did. A whole-page digest that differs is a fact you
- // cannot act on; the per-message row is what makes it a bug report.
- //
- // `opts.raw` additionally returns the signature TEXT. It is large, so it is off by default and
- // used only by the null-control hunt, which has to name the volatile rather than count it.
+ // One digest for the whole thread plus one PER MESSAGE, so a mismatch says which message moved
+ // rather than only that something did. `opts.raw` also returns the signature TEXT: large, so it
+ // is off by default and used only by the null-control hunt.
capture(opts) {
const want_raw = Boolean(opts && opts.raw);
try {
const dom = D();
const rootFn = dom.threadRoot;
const found = rootFn ? rootFn() : null;
- // WHICH root was digested travels with the digest. `threadRoot()` falls back to
- // `document.body`, which digests the sidebar, the header and every relative timestamp in
- // them. Two arms that silently digested different roots would still produce two
- // comparable-looking hashes, and the mismatch would be read as a UI change.
+ // WHICH root was digested travels with the digest: `threadRoot()` falls back to `document.body`,
+ // and two arms that silently digested different roots would still produce comparable-looking
+ // hashes.
const isThread = Boolean(found && found !== document.body &&
found.classList && found.classList.contains("aui-thread-root"));
const root = found || document.body;
const nodes = (dom.messages && dom.messages()) || [];
- // ── THE STREAMED MESSAGE, AND WHY IT GETS ITS OWN DIGEST ────────────────────────────────
- //
- // A message that is still being written has no defined moment. The two arms of an A/B are
- // two cells run back to back against ONE pacer: the bytes on the wire are identical by
- // construction, but each arm has its own send click, its own `t0` and its own paint clock,
- // and the digest is taken at a wall-clock offset in the film rather than at a character
- // count in the stream. So the two arms are compared at two different points in the same
- // reply, and the difference that comes back is wall clock wearing the shape of a UI change.
- //
- // That is the failure this whole file's neighbours were written against: measuring at a
- // moment whose meaning is not stable across the things being compared. It is the same
- // mistake as a census taken on a `data-state` flip, and it is worse here because the
- // renderer amplifies it. Mid-stream, Unsloth does not show a PREFIX of the finished reply:
- // `parseIncompleteMarkdown` runs remend over the tail and closes whatever construct is
- // half-arrived, KaTeX renders the repaired formula (and, while it will not parse, writes
- // the parse error and its character offset into a `title`), Shiki re-tokenises the repaired
- // fence, and the trailing code block carries `data-incomplete`. None of that is monotonic
- // in how much text has arrived, which is why the difference cannot be recognised by its
- // size and why a same-length mismatch here is not evidence of a missed volatile.
- //
- // MEASURED, on the shipped corpus (unit 9, 4,238 characters) driven through remend, KaTeX
- // and Shiki into the shipped `signature()`. Stepping the arrived text by the pacer's own
- // 24-character chunk, 175 of the 175 adjacent pairs produce a different digest -- one chunk
- // of skew is enough to fail a stable action outright. At one-character resolution the
- // serialised length moves DOWNWARDS at 52 of 4,237 steps and 34 pairs of distinct stream
- // positions serialise to exactly the same length with different digests, which is the
- // same-length drift that reads as a normaliser gap. It also cuts the other way: 398 of the
- // 4,237 one-character advances move nothing at all.
- //
- // WHAT IS DONE ABOUT IT. The in-flight message is named, and a SECOND whole-thread digest
- // is taken with its subtree elided -- a marker in its place, so its presence, its position
- // and its role are still compared and a message that vanished still moves the digest. The
- // comparison layer scores the settled document on the settled digest and refuses a verdict
- // on the in-flight message rather than calling it a difference. It is NOT normalised away
- // and it is not folded into a pass: see `analysis/parity.compare`.
- //
- // THE SCAFFOLD DIGEST ELIDES EVERY MESSAGE, NOT ONLY THE STREAMING ONES, and that is not
- // over-reach -- it is the only version of this that is comparable across arms. Whether a
- // given message is in flight is a property of ONE arm at the moment ITS digest was taken,
- // and the ordinary case is precisely that the arms disagree about it: one has finished the
- // reply and the other has not. A thread digest that elided each arm's own in-flight set
- // would then be two different walks, and the two would differ because of the elision
- // itself. Eliding all of them makes the walk identical on both sides by construction.
- //
- // Nothing is given up by decomposing it this way. The whole-thread digest is the scaffold
- // plus every message subtree in place; the scaffold keeps a marker carrying each message's
- // tag, role and position, and every message subtree is digested on its own row below. So a
- // message that moved, vanished, changed role or changed content still moves something, and
- // the comparison layer can withhold ONE of those rows without losing the rest.
+ // THE STREAMED MESSAGE GETS ITS OWN DIGEST. A message still being written has no defined moment:
+ // the two arms share one pacer so the bytes are identical, but each has its own send click and
+ // paint clock while the digest is taken at a wall-clock offset in the film, so the arms are
+ // compared at two different points in the same reply.
+ // The renderer amplifies it: mid-stream `parseIncompleteMarkdown` remends the tail, KaTeX renders
+ // the repaired formula, Shiki re-tokenises the repaired fence, and the trailing code block
+ // carries `data-incomplete`. None of that is monotonic in how much text has arrived, so a
+ // same-length mismatch is not evidence of a missed volatile.
+ // MEASURED on the shipped corpus (unit 9, 4,238 characters): stepping by the pacer's 24-character
+ // chunk, 175 of 175 adjacent pairs produce a different digest, so one chunk of skew fails a stable
+ // action outright. At one-character resolution the serialised length even moves DOWNWARDS at 52
+ // of 4,237 steps.
+ // SO: the in-flight message is named and a SECOND whole-thread digest is taken with its subtree
+ // elided, a marker keeping its presence, position and role. The comparison layer scores the
+ // settled document on the settled digest and refuses a verdict on the in-flight message.
+ // THE SCAFFOLD DIGEST ELIDES EVERY MESSAGE, not only the streaming ones: whether a message is in
+ // flight is a property of ONE arm at the moment ITS digest was taken, and the ordinary case is
+ // that the arms disagree, so per-arm elision would make the two walks differ because of the
+ // elision itself. Nothing is given up -- the scaffold plus the per-message rows IS the digest.
const inFlightNodes = (dom.streamingMessages && dom.streamingMessages()) || [];
const inFlight = new Set(inFlightNodes);
const running = Boolean(dom.isRunning && dom.isRunning());
- // The narrower reading, and the positive control below is the one place that needs it.
- // `isRunning()` is true whenever the composer is refusing a fresh send, and a prompt
- // waiting in the queue on an IDLE thread refuses one too. See `dom.generating`.
+ // The narrower reading, needed by the positive control below: `isRunning()` is true whenever the
+ // composer refuses a fresh send, including a prompt queued on an IDLE thread.
+ // `dom.generating` is the reading; `analysis/parity.compare` refuses to fold it into a pass.
const generating = dom.generating ? Boolean(dom.generating()) : running;
- // Whether the app still publishes the status contract at all. See
- // `dom.statusHookPresent`: a hook that is GONE is a blind instrument, a hook that is
- // present with nothing running is an ordinary settled or not-yet-mounted thread.
- // WHY THE PROBE IS QUIET, which is three questions and not one. See
- // `dom.statusHookPresent` / `dom.lastAssistantPublishesStatus`.
- //
- // the last assistant message IS publishing parts and none of them says running
- // -> the hook's VOCABULARY changed. Blind, unless this arm is windowing, in which case
- // the message being written may not be the last one mounted and nothing here can
- // tell the two apart.
- // the last assistant message publishes NOTHING
- // -> it has no parts yet, which is ordinary -- unless no assistant message anywhere
- // publishes anything, and then the hook itself is gone. Blind on any arm, windowed
- // or not, because a settled message would still be carrying it.
+ // Whether the app still publishes the status contract at all: a hook that is GONE is a blind
+ // instrument, while one present with nothing running is an ordinary settled thread.
+ // WHY THE PROBE IS QUIET is three questions, not one: parts published with none saying running
+ // means the hook's VOCABULARY changed; nothing published at all is ordinary unless no assistant
+ // message anywhere publishes, and then the hook is gone.
const lastPublishes = dom.lastAssistantPublishesStatus
? Boolean(dom.lastAssistantPublishesStatus())
: false;
@@ -894,9 +585,9 @@
if (want_raw) row.raw = sig;
messages.push(row);
}
- // Surfaces that live OUTSIDE the thread root and are therefore invisible to the digest
- // above: an open dialog, an open menu, the model picker. A perf change that alters a
- // popover would otherwise pass a thread-only parity check.
+ // Surfaces OUTSIDE the thread root and therefore invisible to the digest above -- an open dialog,
+ // an open menu, the model picker -- so a perf change that alters a popover cannot pass a
+ // thread-only parity check.
const overlays = [];
for (const sel of ['[data-slot="dialog-content"]', '[role="menu"]', '[role="dialog"]',
".unsloth-model-selector-menu", "[data-radix-popper-content-wrapper]"]) {
@@ -914,80 +605,55 @@
root_kind: isThread ? "thread" : "body",
digest: hash(whole),
chars: whole.length,
- // The thread with every message replaced by a marker: the viewport, the composer, the
- // empty state, and each message's tag, role and position. `digest` = this plus the
- // per-message rows below, which is what lets a single message be withheld from the
- // comparison without the other two thirds of the reading going with it.
+ // The thread with every message replaced by a marker carrying its tag, role and position.
+ // `digest` = this plus the per-message rows below, which is what lets one message be withheld
+ // from the comparison without the rest of the reading going with it.
digest_scaffold: hash(scaffold),
chars_scaffold: scaffold.length,
- // Mounted indices of the messages that were still being written. An EMPTY list on a
- // page where nothing is running is the ordinary case and means what it says.
+ // Mounted indices of the messages still being written; an empty list on a page where nothing is
+ // running is the ordinary case.
in_flight,
streaming: running,
- // THE POSITIVE CONTROL, and it is not decoration. `streamingMessages()` walks selectors
- // written against Unsloth's markup: rename `data-status` and it goes quiet, matches
- // nothing, and every reading silently becomes "nothing was streaming" -- which is the
- // strongest claim this capture can make about the stream and would be supported by no
- // observation at all. The app says a reply is running through a different control (the
- // Stop button), so when the two disagree the disagreement is carried out rather than
- // resolved here, and `analysis/parity.comparability` refuses the pair.
- //
- // IT IS `generating()` AND NOT `isRunning()` that is compared against, and that
- // difference is most of this field's meaning. `isRunning()` is also true while a queued
- // prompt waits on a thread that is doing NOTHING, and on that thread no message
- // publishes a running status because none is running -- so reading it here would refuse
- // an ordinary settled pair before `compare()` ever reached its settled digests, and
- // would report the instrument as broken to say it. That is this file's own failure
- // mode: reading at a moment whose meaning is not stable across the things being
- // compared. See `dom.generating`.
- //
- // AND IT REQUIRES THE HOOK TO BE GONE, not merely quiet. `streamingMessages()` scans
- // MOUNTED DOM, so it also returns nothing in the gap between a send being accepted and
- // the reply's first part rendering, and on an arm that has unmounted the message it is
- // writing into. Neither is a broken instrument. See `dom.statusHookPresent`.
+ // THE POSITIVE CONTROL: `streamingMessages()` walks selectors written against Unsloth's markup,
+ // so renaming `data-status` makes every reading silently become "nothing was streaming". The app
+ // also reports a running reply through the Stop button, so a disagreement is carried out rather
+ // than resolved here, and `analysis/parity.comparability` refuses the pair. It compares
+ // `generating()` and NOT `isRunning()`, which is also true while
+ // a queued prompt waits on an idle thread. And it requires the hook to be GONE, not merely quiet:
+ // `streamingMessages()` scans mounted DOM.
in_flight_unplaced: Boolean(
generating && inFlightNodes.length === 0 && probeBlind
),
status_hook_present: anyPublishes,
- // The queued-idle interval itself, recorded rather than resolved away, so a reader can
- // tell why a capture with `streaming: true` placed no message in flight.
- // INCLUDING THE DISPATCHED WAIT. `isRunning()` matches "Stop generating" and
- // "Queue message" only, so the interval where a dispatched queue entry renders
- // "Stop queued message" recorded `streaming` and `queued_idle` both false -- identical
- // to a settled Send arm. A pair straddling that transient then had a differing composer
- // with no run-state difference to account for it, and the comparison layer is entitled
- // to call that a rendering regression. It is queue timing.
+ // The queued-idle interval itself, recorded rather than resolved away, so a reader can tell why a
+ // capture with `streaming: true` placed no message in flight.
+ // INCLUDING THE DISPATCHED WAIT: `isRunning()` matches "Stop generating" and "Queue message"
+ // only, so the interval rendering "Stop queued message" recorded both flags false, identical to a
+ // settled Send arm, and a pair straddling it looked like a rendering regression.
queued_idle: Boolean(
(running || (dom.stopQueuedButton ? Boolean(dom.stopQueuedButton()) : false)) &&
!generating
),
- // The composer's run-state slot, as a token. Carried so the comparison layer can tell a
- // scaffold that differs because the two arms were at different points in one turn from a
- // scaffold that differs because something was rendered differently. See
- // `dom.runStateControl` and `analysis/parity.generation_disagrees`.
+ // The composer's run-state slot as a token, so the comparison layer can tell a scaffold that
+ // differs because the arms were at different points in one turn from one that differs because
+ // something rendered differently.
+ // See `dom.runStateControl` and `analysis/parity.generation_disagrees`.
composer_control: dom.runStateControl ? dom.runStateControl() : null,
messages,
overlays,
styles,
- // HOW MUCH OF THE THREAD THIS DIGEST COVERS.
- //
- // Every per-message row above is keyed by `i`, its position in the MOUNTED list. On the
- // shipped build that is also its position in the conversation, so the key is meaningful
- // and two arms can be compared row by row. On an arm that mounts a window it is not:
- // msg3 on one side and msg3 on the other are different messages, and comparing their
- // digests produces a wall of mismatches that says nothing about either build.
- //
- // Carrying the two numbers means the comparison layer can REFUSE rather than report
- // eighteen false differences. Identical on the shipped build, so nothing changes for a
- // normal pair.
+ // HOW MUCH OF THE THREAD THIS DIGEST COVERS. Every per-message row is keyed by its position in
+ // the MOUNTED list, which equals its conversation position only on the shipped build; on a
+ // windowed arm msg3 and msg3 are different messages and a row-by-row comparison says nothing.
+ // Carrying the two numbers lets the comparison layer REFUSE instead.
mounted_messages: messages.length,
thread_total: (dom.threadTotal && dom.threadTotal()) || messages.length,
};
if (want_raw) out.raw = whole;
return out;
} catch (err) {
- // A failure is reported as a failure. A parity check that returns an empty digest on
- // error would read as "everything matched", which is the single worst thing it could do.
+ // A failure is reported as a failure: a parity check that returned an empty digest on error would
+ // read as "everything matched".
return { parity_attempted: false, reason: String(err && err.message ? err.message : err) };
}
},
diff --git a/tests/studio/studiobench/scene/schedule.py b/tests/studio/studiobench/scene/schedule.py
index cc5a8b7b628..20bb0c5b3a9 100644
--- a/tests/studio/studiobench/scene/schedule.py
+++ b/tests/studio/studiobench/scene/schedule.py
@@ -76,11 +76,9 @@ def _slots(spec: list[tuple[str, int, Optional[int]]]) -> list[Slot]:
]
-# The standard film. Ordered so that the actions that must happen DURING generation come first,
-# then the ones that need a finished reply, then the destructive ones last -- delete and reopen
-# change the thread, so anything after them would be measuring a different thread.
-#
-# Timings are offsets from the moment the send button was pressed.
+# The standard film, ordered so actions that must happen DURING generation come first, then
+# ones needing a finished reply, then the destructive ones last, since delete and reopen change
+# the thread. Timings are offsets from the send button press.
STANDARD = Scene(
name = "standard",
slots = _slots(
@@ -88,10 +86,9 @@ STANDARD = Scene(
# ── during generation ────────────────────────────────────────
("scroll_during_generation", 3_000, 8_000),
("keystroke", 12_000, 6_000),
- # 12s. The tail is a JITTERED clip capped at 6,000 characters, so its drain time varies from
- # about 14s to 18s across the ladder, and a during-generation slot has to open before the
- # SHORTEST of those, not the longest. At 19s, then 15s, this slot ran against a finished reply
- # at the top rung while still being labelled "during generation".
+ # 12s: the tail is a JITTERED clip capped at 6,000 characters, so its drain varies from about
+ # 14s to 18s and a during-generation slot must open before the SHORTEST of those. At 19s, then
+ # 15s, this slot ran against a finished reply at the top rung.
("scroll_during_generation", 12_000, 8_000),
("stop_generation", 28_000, 8_000),
# ── after the reply is complete ──────────────────────────────
@@ -102,61 +99,48 @@ STANDARD = Scene(
("copy_markdown", 86_000, 6_000),
("select_text", 93_000, 6_000),
("send_turn", 100_000, 12_000),
- # 35s, not 10s. Selecting and copying the whole thread at the 1M rung took 27,690 ms -- two
- # million characters through the clipboard -- so it blew a 10s budget by nearly three times
- # and the OVERRUN pushed the next slot past its own start, which was recorded as
- # `composer_fill: slot missed`. The 1M cell lost two slots that way and the cause looked like
- # a slow machine rather than one under-budgeted action.
- #
- # This film is the one the 500K and 1M rungs use, so its budgets are sized from what those
- # rungs actually cost, not from what the small ones do. At 100K the same action takes 2,476 ms.
+ # 35s, not 10s: selecting and copying the whole thread at 1M took 27,690 ms, nearly three times
+ # a 10s budget, and the OVERRUN pushed the next slot past its own start, recorded as
+ # `composer_fill: slot missed`. This film is what 500K and 1M use, so its budgets are sized
+ # from what those rungs cost; at 100K the same action takes 2,476 ms.
("select_all_copy", 113_000, 35_000),
("composer_fill", 149_000, 10_000),
("model_change", 160_000, 10_000),
("settings", 171_000, 12_000),
("image_upload", 184_000, 12_000),
- # ── destructive, last ────────────────────────────────────────
# 22,382 ms at 1M, so 30s stands.
+ # ── destructive, last ────────────────────────────────────────
("thread_reopen", 197_000, 30_000),
("delete_message", 228_000, 15_000),
]
),
)
-# The quick film. The SAME fifteen actions in the same order -- a tier that drops actions cannot be
-# compared with one that does not -- on a shorter clock, for the small rungs where the stream is
-# over in seconds.
+# The quick film: the SAME fifteen actions in the same order, since a tier that drops actions
+# cannot be compared with one that does not, on a shorter clock for the small rungs.
QUICK = Scene(
name = "quick",
slots = _slots(
[
- # BUDGETS ARE SIZED FROM MEASURED ACTION COST, not guessed. At the 100K rung -- the largest
- # this film is used for -- every action finished inside 2.5 s: select_all_copy 2,476 ms,
- # thread_reopen 2,234 ms, reasoning_toggle 1,788 ms, keystroke 1,041 ms, everything else under
- # 500 ms. The budgets below carry roughly 2.5x headroom over those.
- #
- # The film previously ran 162 s while its actions used 6.4 s, so 96% of it was waiting for the
- # next slot to open. That waiting is not free: it is multiplied by every cell, every arm and
- # every repetition, and an A/B at four reps is sixteen films. Halving the film halves the cost
- # of every comparison this tool exists to make.
- #
- # What CANNOT be compressed is the stream phase. The opening turn drains in 12 to 18 s, the
- # during-generation slots have to open inside the shortest of those (14.1 s at 1M) and the
- # after-generation slots have to open after the longest (17.8 s at 100K). The gap between
- # 12 s and 20 s below is that constraint, not slack.
+ # BUDGETS ARE SIZED FROM MEASURED ACTION COST: at 100K, the largest rung this film is used for,
+ # every action finished inside 2.5 s (select_all_copy 2,476 ms, thread_reopen 2,234 ms,
+ # reasoning_toggle 1,788 ms, keystroke 1,041 ms, the rest under 500 ms), and the budgets carry
+ # roughly 2.5x headroom. The film previously ran 162 s while its actions used 6.4 s, and that
+ # waiting is multiplied by every cell, arm and repetition. What CANNOT be compressed is the
+ # stream phase: the opening turn drains in 12 to 18 s, during-generation slots must open inside
+ # the shortest (14.1 s at 1M) and after-generation slots after the longest (17.8 s at 100K), so
+ # the gap between 12 s and 20 s is that constraint, not slack.
("scroll_during_generation", 1_500, 2_500),
("keystroke", 5_000, 3_000),
("scroll_during_generation", 9_500, 2_500),
("stop_generation", 20_000, 3_000),
("scroll_after", 23_500, 2_500),
("reasoning_toggle", 26_500, 4_500),
- # 1,500 rather than 4,000, and the reason is the gap AFTER it rather than the cost of
- # the send itself. `send_turn` is a sub-100 ms action; a 4,000 ms window to start in is
- # not headroom, it is permission to begin the follow-up four seconds late without
- # recording a miss. The drain runs from when the send actually fires, so every one of
- # those four seconds comes off `message_menu`'s window: measured from the latest legal
- # send, this film left 3,500 ms for a 4,562 ms drain. The fast film had the same shape
- # and CI failed on it. See test_settled_actions_open_after_the_follow_up_drains.
+ # 1,500 rather than 4,000, because of the gap AFTER it rather than the cost of the send:
+ # `send_turn` is a sub-100 ms action, so a 4,000 ms window is permission to begin the follow-up
+ # four seconds late without recording a miss, and every one of those seconds comes off
+ # `message_menu`'s window. Measured from the latest legal send, this film left 3,500 ms for a
+ # 4,562 ms drain; the fast film had the same shape and CI failed on it.
("send_turn", 31_500, 1_500),
("message_menu", 36_000, 3_000),
("copy_markdown", 39_500, 2_500),
@@ -173,78 +157,51 @@ QUICK = Scene(
),
)
-# The fast film. FOR ITERATION, NOT FOR REPORTING -- see the banner the CLI prints.
-#
-# Same eighteen actions in the same order as the other two films, because a tier that drops
-# actions cannot tell you that your fix broke the one it dropped. What changes is the waiting.
-#
-# Budgets are sized from the costs this tool MEASURED at the 100K rung over eight cells of a null
-# control, at roughly 1.5x rather than the 2.5x the quick film carries:
-#
-# thread_reopen 3,163 ms (close 2,205 + reopen 958) select_all_copy 2,454 ms
-# reasoning_toggle 2,250 ms (open 1,688 + close 563) keystroke 1,002 ms
-# settings 485 ms model_change 483 ms scroll 466 ms stop 335 ms copy 204 ms
-#
-# Everything else is under 100 ms. The eighteen actions cost about 12 s of real work; the standard
-# film spends 243 s and the quick film 77.5 s to deliver them. At 1.5x headroom an action that
-# overruns records `slot_missed` instead of silently pushing the next slot, which is the honest
-# failure mode and is exactly what the fixed-duration design exists to provide.
-#
-# WHAT CANNOT BE COMPRESSED, and why this film is 47 s rather than 20 s: the opening turn streams
-# a 6,000 character tail at field cadence (328.8 chars/s), so it drains in 14 to 18 s. The
-# during-generation slots must open inside the SHORTEST of those and the after-generation slots
-# after the LONGEST. The gap between 8 s and 19 s below is that constraint, not slack. Shrinking
-# it means shrinking the streamed tail, which changes the load being measured -- and streaming
-# frame cost at length is the thing most of these fixes are about.
+# The fast film. FOR ITERATION, NOT FOR REPORTING. Same eighteen actions in the same order as
+# the other two, because a tier that drops actions cannot tell you that your fix broke the one
+# it dropped; what changes is the waiting. Budgets are sized from costs measured at 100K over
+# eight null-control cells at roughly 1.5x rather than the quick film's 2.5x (thread_reopen
+# 3,163 ms and select_all_copy 2,454 ms at the top, everything past model_change under 500 ms).
+# At 1.5x headroom an action that overruns records `slot_missed` instead of silently pushing
+# the next slot. WHAT CANNOT BE COMPRESSED, and why this film is 47 s: the opening turn streams
+# a 6,000 character tail at field cadence (328.8 chars/s), draining in 14 to 18 s, so the gap
+# between 8 s and 19 s below is that constraint, and shrinking it means shrinking the streamed
+# tail, which changes the load being measured.
FAST = Scene(
name = "fast",
slots = _slots(
[
- # 18.4 s, not 8 s. `stop_generation` starts and stops its OWN turn, so opening it while the
- # opening tail is still draining starts a second turn on top of the first and truncates the
- # reply being measured -- which is the defect that made the seeded-vs-streamed equivalence
- # check read a false 20% drift earlier in this project.
- #
- # KEYED TO THE DECLARED TAIL, NOT TO WHAT THE CORPUS HAPPENS TO STREAM. This said 17.8 s
- # and sat at 18.2 s, which was under the ladder's own ceiling the whole time and only
- # passed because the 1M rung was streaming recycled text: the manifest was sized at
- # exactly the top rung's seeded target, so all three streamed turns clamped onto the last
- # unit. Sizing the manifest from the ladder gave 1M its own material and the drain went
- # to 18.23 s, which is what a corpus this tier can be pointed at was always allowed to
- # do. STREAM_TAIL_CHARS (6,000) at field cadence is 18.25 s and every rung is held under
- # it, so that ceiling is the bound to clear, and clearing it costs 200 ms here.
- #
- # THE SECOND PACKING CONSTRAINT, learned the expensive way. `send_turn` starts a FOLLOW-UP
- # turn, and a follow-up is FOLLOW_UP_CHARS (1,500) at field cadence, so it streams for 4.6 s.
- # Anything needing a SETTLED reply -- the action bar's More and Copy buttons, select-all,
- # delete -- does not exist while that turn is running. The first fast film opened
- # `message_menu` 1.7 s after a `send_turn` and it recorded `NOT RUN: no More button` on
- # 312 of 312 attempts across a 36 job sweep, silently removing the four actions that carry
- # the largest known effect in this codebase (the message menu is the 19x one). Every
- # post-send slot below clears 4.6 s, and test_settled_actions_open_after_the_follow_up_drains
- # now fails any film that does not.
+ # 18.4 s, not 8 s: `stop_generation` starts and stops its OWN turn, so opening it while the
+ # opening tail is still draining truncates the reply being measured, the defect that made the
+ # seeded-vs-streamed check read a false 20% drift. KEYED TO THE DECLARED TAIL, not to what the
+ # corpus happens to stream: at 17.8 s it passed only because the 1M rung was streaming recycled
+ # text, and STREAM_TAIL_CHARS (6,000) at field cadence is 18.25 s, the ceiling to clear.
+ # THE SECOND PACKING CONSTRAINT, learned the expensive way: `send_turn` starts a FOLLOW-UP turn
+ # of FOLLOW_UP_CHARS (1,500) at field cadence, so it streams for 4.6 s, and anything needing a
+ # SETTLED reply does not exist while it runs. The first fast film opened `message_menu` 1.7 s
+ # after a `send_turn` and recorded `NOT RUN: no More button` on 312 of 312 attempts across a 36
+ # job sweep. Every post-send slot below clears 4.6 s, and
+ # test_settled_actions_open_after_the_follow_up_drains fails any film that does not.
("scroll_during_generation", 1_500, 1_200),
("keystroke", 3_000, 1_800),
("scroll_during_generation", 6_000, 1_200),
("stop_generation", 18_400, 3_000),
("scroll_after", 21_500, 1_200),
- # THE BUDGET IS SIZED FROM WHAT CI MEASURES, not from the null control's 2,250 ms.
- # On the GitHub runner this action costs 4,415.9 ms and 4,434.4 ms on two consecutive
- # jobs -- reaching the open state alone takes 2,898.8 ms there, which already exceeds
- # the whole-action figure the 3,500 ms budget was derived from. At 3,500 it overran by
- # 934 ms on every run, and an overrun here lands on `send_turn`, the one slot whose
- # downstream gap is load-bearing. 5,000 ms covers the measured cost with room.
+ # THE BUDGET IS SIZED FROM WHAT CI MEASURES, not the null control's 2,250 ms: on the GitHub
+ # runner this action costs about 4,420 ms, of which reaching the open state alone is 2,898.8
+ # ms. At 3,500 it overran by 934 ms on every run, and an overrun here lands on `send_turn`, the
+ # one slot whose downstream gap is load-bearing.
+ # 5,000 ms covers the measured cost with room.
("reasoning_toggle", 23_000, 5_000),
- # MOVED OUT FROM UNDER `reasoning_toggle`, which now nominally ends at 28,000. A send
- # slot that opens before the action before it can finish is a slot that starts late,
- # and a send that starts late shortens the drain window of everything after it while
- # reporting no miss of its own.
+ # MOVED OUT FROM UNDER `reasoning_toggle`: a send slot that opens before the action before it
+ # can finish starts late, and a late send shortens the drain window of everything after it
+ # while reporting no miss of its own.
+ # Which now nominally ends at 28,000.
("send_turn", 28_100, 1_500),
- # THE GAP IS THE POINT, and it is measured from 29,600 -- the latest this film's send
- # can fire -- not from 28,100. The follow-up drains in 4,562 ms nominal and 4,400 to
- # 4,700 ms observed, so a window closing at 35,400 leaves about 1.1 s of real margin
- # against the worst drain seen. The old packing left 38 ms, which is why CI failed on
- # a runner no slower than the one that passed.
+ # THE GAP IS THE POINT, measured from 29,600, the latest this film's send can fire, not from
+ # 28,100. The follow-up drains in 4,562 ms nominal and 4,400 to 4,700 observed, so a window
+ # closing at 35,400 leaves about 1.1 s of margin; the old packing left 38 ms, which is why CI
+ # failed on a runner no slower than the one that passed.
("message_menu", 34_400, 1_000),
("copy_markdown", 35_400, 600),
("select_text", 36_200, 400),
@@ -254,12 +211,10 @@ FAST = Scene(
("model_change", 46_900, 1_000),
("settings", 48_100, 1_200),
("image_upload", 49_500, 800),
- # thread_reopen 6.5 s and delete 2.5 s, not 5 s and 0.6 s. Measured at 100K the pair
- # costs about 3.2 s, but the ACTION overran its 5 s window and pushed the last slot: the
- # machine arrived at delete_message 834 ms after its 600 ms budget had already closed, so
- # on a 36 job sweep delete recorded NOT EXERCISED on the base arm of every null-control
- # cell. A last slot with no slack is a slot that measures nothing, and the film's own end
- # is the one place an overrun has nowhere to go.
+ # thread_reopen 6.5 s and delete 2.5 s, not 5 s and 0.6 s: measured at 100K the pair costs about
+ # 3.2 s, but the ACTION overran its 5 s window and pushed the last slot, so delete recorded NOT
+ # EXERCISED on the base arm of every null-control cell in a 36 job sweep. The film's own end is
+ # the one place an overrun has nowhere to go.
("thread_reopen", 50_500, 6_500),
("delete_message", 57_200, 2_500),
]
@@ -285,11 +240,9 @@ class SceneRunner:
"""`t0` is the driver monotonic time the film started, i.e. when send was pressed."""
rows: list[dict] = []
for i, slot in enumerate(scene.slots):
- # The GAP before this slot is itself a measured window. Without it, frame rate and
- # blocked time would only ever be sampled inside actions -- and the quiet stretches
- # between them, which is where the stream is doing its work unaided, would be the one
- # part of the session nothing observed. Continuous coverage, no sampler thread, and
- # no window ever overlaps another.
+ # The GAP before this slot is itself a measured window: without it, frame rate and blocked time
+ # would only be sampled inside actions, and the quiet stretches where the stream does its work
+ # unaided would be the one part of the session nothing observed.
self._gap_window(f"stream:gap{i}", slot.t_start_ms, t0)
row = self._run_slot(slot, t0)
rows.append(row)
@@ -312,8 +265,8 @@ class SceneRunner:
try:
self.page.evaluate("() => window.__sb.parityVisible.watch()")
except Exception: # noqa: BLE001
- # A page that cannot install it still gets a structural digest; the visible-region
- # capture reports itself absent rather than empty, which the analysis refuses.
+ # A page that cannot install it still gets a structural digest; the visible-region capture
+ # reports itself absent rather than empty, which the analysis refuses.
pass
def _visible(self) -> dict:
@@ -380,32 +333,18 @@ class SceneRunner:
now_ms = (time.monotonic() - t0) * 1000
if until_ms - now_ms < 250:
return
- # `gap`, NOT `stream`. These windows were labelled `stream` and the label was read by
- # people, including this project, as meaning the stream was running in them. It does not.
- # A gap window is opened before EVERY slot, so on the standard film eighteen of them exist
- # and only the first four contain any streaming at all; the rest are the quiet stretches
- # between post-generation actions. Measured on a 100K cell, `stream:gap12` ran for 32.9 s
- # at 1.6% busy with the reply finished thirty seconds earlier, and `stream:drain` -- the
- # one window that really is about the stream -- was 7 ms long.
- #
- # Anyone filtering on `kind == "stream"` to find the streaming phase therefore selected
- # mostly post-stream idle and diluted whatever they were measuring. The phase has to be
- # detected from the SSE traffic (see instruments/streamcost.py), and this label no longer
- # invites the mistake.
- #
- # The window NAME is deliberately left as `stream:gapN`. It is the join key in every
- # payload this tool has already written and in the analysis scripts pointed at them;
- # renaming it would break those silently, which is a worse failure than a misleading name
- # that is now documented in INTERFACES.md and contradicted by the kind beside it.
- # THE CENSUS IS TAKEN BEFORE THE GAP WINDOW OPENS, not at its close. Same defect as the
- # action windows (workspace task #102) and the same fix, but the placement is different:
- # a gap window is the QUIET stretch, and its whole purpose is to measure the page doing
- # nothing but stream. Nineteen querySelectorAll passes over 195,000 elements at the end of
- # it is not nothing, and it landed on the frame-rate reading for the idle phase.
- #
- # Before rather than after, because the gap ends the instant the next slot is due: taking
- # it afterwards would push the action's start past its own slot and turn an instrument cost
- # into a missed slot, which is a worse failure than the one being fixed.
+ # `gap`, NOT `stream`. These windows were labelled `stream` and read as meaning the stream was
+ # running in them. A gap window opens before EVERY slot, so on the standard film eighteen exist
+ # and only the first four contain any streaming: on a 100K cell, `stream:gap12` ran 32.9 s at
+ # 1.6% busy with the reply finished thirty seconds earlier, while `stream:drain` was 7 ms.
+ # Anyone filtering on `kind == "stream"` therefore selected mostly post-stream idle. The window
+ # NAME is deliberately left as `stream:gapN`, since it is the join key in every payload already
+ # written.
+ # THE CENSUS IS TAKEN BEFORE THE GAP WINDOW OPENS, not at its close (workspace task #102): a gap
+ # window is the QUIET stretch, and nineteen querySelectorAll passes over 195,000 elements at the
+ # end of it landed on the frame-rate reading for the idle phase. Before rather than after,
+ # because the gap ends the instant the next slot is due, so taking it afterwards would turn an
+ # instrument cost into a missed slot.
census = self._census()
with self.open_window(name, "gap") as window:
window.note("census_before_gap", census)
@@ -422,8 +361,8 @@ class SceneRunner:
).row(slot.action, window_name, self.cell.cell_id)
now_ms = (time.monotonic() - t0) * 1000
- # Wait for the slot to open. Sleeping in small steps rather than one long sleep so a
- # renderer crash is noticed within a fifth of a second rather than at the end of the wait.
+ # Wait for the slot to open, in small steps rather than one long sleep, so a renderer crash is
+ # noticed within a fifth of a second.
while now_ms < slot.t_start_ms:
time.sleep(min(0.2, (slot.t_start_ms - now_ms) / 1000))
now_ms = (time.monotonic() - t0) * 1000
@@ -431,8 +370,8 @@ class SceneRunner:
deadline_ms = slot.t_start_ms + slot.budget_ms
remaining = deadline_ms - now_ms
if remaining <= 0:
- # THE SLOT WAS MISSED. Not an error and not a slow timing: this machine could not get
- # here in time, the film carries on, and the row says exactly that.
+ # THE SLOT WAS MISSED. Not an error and not a slow timing: this machine could not get here in
+ # time, the film carries on, and the row says exactly that.
self.log(
f" slot missed: {slot.action} "
f"(due at {slot.t_start_ms}ms, reached at {now_ms:.0f}ms)"
@@ -467,86 +406,53 @@ class SceneRunner:
window.note("action", slot.action)
window.note("ran", result.ran)
- # THE CENSUS AND THE PARITY DIGEST ARE TAKEN OUTSIDE THE WINDOW. Workspace task #102.
- #
- # Both used to run inside the `with`, one line above this comment, on the strength of a
- # measurement -- "0.2ms on a 1,500-element tree" -- taken on a tree three orders of
- # magnitude smaller than the one this benchmark exists to study. At 100K+ tokens the
- # census walks nineteen querySelectorAll passes over ~195,000 elements and the parity
- # digest serialises 5.6 MB of structure, and every millisecond of it was being charged to
- # the action that happened to precede it. Measured cost of the mistake:
- #
- # delete_message reported 14.3 fps, true 49.0 fps
- # message_menu reported 17.1 fps, true 73.8 fps
- #
- # That is not a correction at the margin. It inverted the ranking of the actions this
- # campaign is about, and it did it in the direction that makes the standing DOM look like
- # a smaller problem than it is, because the instrument's own cost grows with exactly the
- # quantity under investigation.
- #
- # They still run at the same MOMENT -- immediately after the window closes, before the
- # scheduler starts waiting for the next slot -- so the digest and the occupancy still come
- # from one reading of one DOM taken at the end of the action. Only the accounting moved.
- #
- # The consequence, stated: the emitted `window` row no longer carries them in its notes,
- # because the row is emitted when the window closes. They live on the ACTION row, which is
- # where every consumer already reads them from.
- # THE DEADLINE IS SAMPLED HERE, BEFORE THE OBSERVATIONS. Moving the census and the digest
- # out of the measured window but leaving `over_ms` to be taken after them would have left
- # half the defect in place: an action that finished comfortably inside its budget would
- # still be flagged `over_budget` on the strength of a multi-megabyte serialisation that is
- # explicitly not its cost. Charging instrument time to the thing being measured is the
- # whole of task #102, and the budget flag is not exempt from it.
+ # THE CENSUS AND THE PARITY DIGEST ARE TAKEN OUTSIDE THE WINDOW (workspace task #102). Both used
+ # to run inside the `with`, on the strength of "0.2ms on a 1,500-element tree" measured on a
+ # tree three orders of magnitude smaller: at 100K+ the census walks nineteen querySelectorAll
+ # passes over ~195,000 elements and the digest serialises 5.6 MB, and every millisecond was
+ # charged to the preceding action, so delete_message reported 14.3 fps against a true 49.0. That
+ # inverted the ranking of the actions this campaign is about, in the direction that makes
+ # standing DOM look like a smaller problem, because the instrument's cost grows with the
+ # quantity under investigation. They still run at the same MOMENT, so the digest and occupancy
+ # come from one reading; only the accounting moved, and they now live on the ACTION row.
+ # And message_menu 17.1 fps against a true 73.8.
+ # THE DEADLINE IS SAMPLED HERE, BEFORE THE OBSERVATIONS: leaving `over_ms` to be taken after them
+ # would still flag an action `over_budget` on the strength of a multi-megabyte serialisation
+ # that is explicitly not its cost.
window_closed_at = time.monotonic()
over_ms = ((window_closed_at - t0) * 1000) - deadline_ms
- # THE VISIBLE CAPTURE GOES FIRST, AND THE ORDER IS THE MEASUREMENT. The other two read a
- # DOM that is sitting still; this one CLOSES an observation that has been accumulating
- # since `_watch_visible`, and every millisecond it stays open is another millisecond in
- # which a row can scroll into view and be counted as having been visible during the
- # action. Taken after the census and the digest, that tail was as long as those two probes
- # take -- and they are the one thing on this path whose cost is proportional to the arm.
- #
- # A fully mounted arm serialises the whole thread, a windowed arm serialises the dozen
- # rows it has mounted, so the two arms' observers stayed open for materially different
- # intervals: the numbers above this comment are 14.3 fps against 49.0, which is the same
- # asymmetry expressed as time. With streaming and autoscroll still running at the close of
- # `scroll_during_generation`, the mounted arm had the longer tail in which to admit an
- # ordinal the windowed arm never saw, and `compare_visible` has no tolerance for that --
- # it returns DIFFER on strict set inequality of `ever_visible`. The null control cannot
- # absorb it either, being same-build against same-build, where both sides pay the same
- # digest and the asymmetry does not exist.
- #
- # Taking it first does not make the tail zero: `capture()` still waits two frames, on
- # purpose, so a quiet action does not report an empty viewport. It makes the tail the SAME
- # ON BOTH ARMS, which is the property the comparison actually needs.
+ # THE VISIBLE CAPTURE GOES FIRST, AND THE ORDER IS THE MEASUREMENT. The other two read a DOM
+ # sitting still; this one CLOSES an observation accumulating since `_watch_visible`, so every
+ # millisecond it stays open is another in which a row can scroll into view and be counted as
+ # visible during the action. Taken after the census and digest, that tail was as long as those
+ # two probes take, and they are the one thing here whose cost is proportional to the arm, so the
+ # two arms' observers stayed open for materially different intervals (14.3 fps against 49.0 is
+ # the same asymmetry expressed as time). `compare_visible` returns DIFFER on strict set
+ # inequality of `ever_visible` and the null control cannot absorb it, being same-build against
+ # same-build. Taking it first does not make the tail zero, since `capture()` still waits two
+ # frames on purpose; it makes it the SAME ON BOTH ARMS.
visible = self._visible()
census = self._census()
parity = self._parity()
- # The observations DO consume wall clock before the next slot, so their cost is recorded
- # rather than dropped -- it is simply recorded as theirs.
+ # The observations DO consume wall clock before the next slot, so their cost is recorded as
+ # theirs rather than dropped.
observation_ms = (time.monotonic() - window_closed_at) * 1000
row = result.row(slot.action, window_name, self.cell.cell_id)
row["window_ms"] = window.duration_ms
- # READ FROM THE LOCALS ABOVE, not from `window.notes`. The three observations were moved
- # out of the measured window, so by the time this row is built they are values this method
- # holds rather than notes the window carries.
+ # READ FROM THE LOCALS ABOVE, not from `window.notes`: the three observations were moved out of
+ # the measured window, so by now they are values this method holds.
row["census"] = census
row["parity"] = parity
row["visible"] = visible
- # The observation cost itself, so it is never invisible again. `census_cost_ms` is the
- # page's own timing of the walk; if this pair ever comes to dominate the film's wall clock
- # the number is in the payload rather than in someone's hypothesis.
+ # The observation cost itself, so it is never invisible again. `census_cost_ms` is the page's
+ # own timing of the walk.
row["observation_outside_window"] = True
row["observation_ms"] = round(observation_ms, 1)
- # THE SCREENSHOT IS TAKEN OUTSIDE THE WINDOW, on purpose and not as a tidiness point.
- # The film runs on a wall clock and every slot has an absolute start, so an encode charged
- # to the measured window eats the gap before the next slot -- which is how an action comes
- # to report a MISSED SLOT on a contended runner. `--assert-liveness` counts those, so a
- # camera inside the window could turn a healthy run red and it would look like the harness
- # failing rather than like the instrument taxing itself. Out here it costs the gap, which
- # is what the gap is for. The DOM has moved on by a few milliseconds; for a picture that
- # is nothing, and for the digest, which was taken inside, it is not true at all.
+ # THE SCREENSHOT IS TAKEN OUTSIDE THE WINDOW: the film runs on a wall clock with absolute slot
+ # starts, so an encode charged to the measured window eats the gap before the next slot, which
+ # is how an action comes to report a MISSED SLOT on a contended runner, and `--assert-liveness`
+ # counts those. Out here it costs the gap, which is what the gap is for.
if isinstance(row.get("parity"), dict) and row["parity"].get("parity_attempted"):
row["parity"].update(self._parity_shot(slot.action))
# An action that ran but overran its budget has pushed nothing (the next slot has its own
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py b/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py
index 51c0685a3f9..18b16849db5 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py
@@ -49,16 +49,15 @@ from studiobench.scene.actions import ACTION_BAR_WAIT_MS, MENU_JS # noqa: E402
DOM_JS = Path(__file__).resolve().parents[1] / "dom.js"
#: The shim mounts the bar this long after the run starts, standing in for a follow-up turn whose
-#: last chunks land just after the slot opens. Comfortably inside `ACTION_BAR_WAIT_MS` and
-#: comfortably outside anything a single sample could catch.
+#: last chunks land just after the slot opens. Comfortably inside `ACTION_BAR_WAIT_MS` and outside
+#: anything a single sample could catch.
LATE_MOUNT_MS = 400
-#: The mount deadline is armed by the action's own first look (see `HARNESS_JS`), so it can only
-#: start level with or after `waitForActionButton`'s clock and `waitedMs` cannot undershoot
-#: `LATE_MOUNT_MS` by any elapsed time at all. What is left is dom.js rounding `waitedMs` to a
-#: tenth. This is an allowance for clock granularity, not a tolerance for load: the wait ends at
-#: the first sample taken at or after the bar mounts, and a slow machine can only push that sample
-#: LATER. Nothing here scales with `LATE_MOUNT_MS`.
+#: The mount deadline is armed by the action's own first look, so it can only start level with or
+#: after `waitForActionButton`'s clock and `waitedMs` cannot undershoot `LATE_MOUNT_MS` by any
+#: elapsed time. What is left is dom.js rounding `waitedMs` to a tenth: an allowance for clock
+#: granularity, not a tolerance for load.
+# See `HARNESS_JS`.
MOUNT_CLOCK_SLACK_MS = 1
HARNESS_JS = r"""
@@ -237,10 +236,9 @@ def test_a_control_that_arrives_late_is_waited_for_rather_than_reported_missing(
"""
out = run_menu(LATE_MOUNT_MS)
assert out["ran"] is True, out
- # Still a liveness assertion, not a timing one: the bar was NOT there when the wait began, and
- # the action stayed until it arrived. The bound is tight because the harness arms the mount
- # deadline off the same clock and the same instant the action starts looking, so anything short
- # of `LATE_MOUNT_MS` means the wait ended early -- a control that was there all along.
+ # Still a liveness assertion, not a timing one: the bar was NOT there when the wait began and the
+ # action stayed until it arrived. The bound is tight because the harness arms the mount deadline
+ # off the same clock and instant, so anything short of `LATE_MOUNT_MS` means the wait ended early.
assert out["waitedMs"] >= LATE_MOUNT_MS - MOUNT_CLOCK_SLACK_MS, out
assert out["waitedMs"] < ACTION_BAR_WAIT_MS, out
# The menu really opened and closed on the control that was waited for, so the wait produced a
@@ -254,8 +252,8 @@ def test_the_wait_is_bounded_and_a_control_that_never_appears_still_reports_not_
out = run_menu(float("inf"), wait_ms = 300)
assert out["ran"] is False, out
assert out["waitedMs"] >= 250, out
- # The reason has to separate the two cases, because they are different bugs: a reply that had
- # not settled, and a control that is not there at all.
+ # The reason has to separate the two cases, because they are different bugs: a reply that had not
+ # settled, and a control that is not there at all.
assert "STILL GENERATING" in out["reason"], out
assert out["running"] is True, out
@@ -265,6 +263,6 @@ def test_a_settled_reply_costs_the_action_nothing():
out = run_menu(0)
assert out["ran"] is True, out
assert out["waitedMs"] < 50, out
- # Hovered before the control is read: the bar unmounts on every message that is not hovered,
- # so a read without the hover is a read of a tree the control was never in.
+ # Hovered before the control is read: the bar unmounts on every message that is not hovered, so a
+ # read without the hover is a read of a tree the control was never in.
assert out["hovers"] >= 1, out
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py
index a54c30d1bcc..20cfb4c7ea9 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py
@@ -173,8 +173,8 @@ def test_a_failed_sentinel_write_does_not_re_admit_the_unchanged_clipboard(page)
plausible non-empty copy of the whole thread with the 250ms settle beside it as `copy_ms`. That
is the exact measurement the sentinel exists to refuse, re-admitted by its own fallback.
"""
- # Whatever an earlier action in the film left behind. Non-empty and plausible, which is what
- # makes it dangerous: an empty clipboard would have been caught by `clipboard_chars > 0`.
+ # Whatever an earlier action in the film left behind. Non-empty and plausible, which is what makes
+ # it dangerous: an empty clipboard would have been caught by `clipboard_chars > 0`.
page.evaluate("async () => await navigator.clipboard.writeText('x'.repeat(2400))")
_refuse_the_sentinel_write(page)
_swallow_the_copy(page)
@@ -224,10 +224,9 @@ def test_the_guard_is_on_the_clipboard_and_not_on_the_engine_name():
src = inspect.getsource(A.select_all_copy)
assert "sentinel" in src
- # EXECUTABLE LINES ONLY. The engine names belong in the comment that explains which engine was
- # observed failing and with what numbers; what must not exist is a BRANCH on one.
+ # EXECUTABLE LINES ONLY. The engine names belong in the comment explaining which engine was
+ # observed failing and with what numbers; what must not exist is a BRANCH on one. The word
+ # "engine" is allowed because it appears in the refusal MESSAGE; a specific engine's NAME is not.
code = "\n".join(line for line in src.splitlines() if not line.lstrip().startswith("#"))
- # The word "engine" is allowed: it appears in the refusal MESSAGE, which is where it belongs.
- # A specific engine's NAME is not, because that is what a branch would need.
for name in ("webkit", "WebKit", "firefox", "Firefox", "browser_name", "browser_type"):
assert name not in code, f"the guard branches on {name!r} rather than on the clipboard"
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py
index bc3377ad92b..fcec329cae4 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py
@@ -39,14 +39,12 @@ if str(_STUDIO_TESTS) not in sys.path:
_DOM_JS = _STUDIO_TESTS / "studiobench" / "scene" / "dom.js"
-#: A running thread: the stop button is what `isRunning()` looks for. The viewport is scrollable so
-#: "at the bottom" is a real question.
-#:
+#: A running thread: the stop button is what `isRunning()` looks for, and the viewport is scrollable
+#: so "at the bottom" is a real question.
#: The jump-to-bottom control is present and its `invisible` class is kept in sync with the scroll
#: position by a listener, because that class IS the app's own answer to "are we at the bottom" and
-#: `appSaysAtBottom()` reads nothing else. Omitting the control makes every sample `running_unknown`
-#: and `pinned_fraction` null, which is correct behaviour on a build that renders no control and
-#: useless as a fixture for scoring one.
+#: `appSaysAtBottom()` reads nothing else. Omitting the control makes every sample
+#: `running_unknown` and `pinned_fraction` null.
FIXTURE = """
<!doctype html><meta charset="utf-8">
<style>
@@ -230,23 +228,20 @@ def test_the_coverage_travels_with_the_verdict(page):
)
-# ── the run the user started is a fresh intent to be at the end ─────
-#
-# THE GESTURE ON THE REAL FILM NEVER ENDS AT THE BOTTOM. `scene/actions.py::SCROLL_JS` jumps to
-# the bottom and then steps 14 x 420px away from it, so on any thread taller than 5,880px the
-# gesture ends thousands of pixels up and the `resume()` re-attachment above cannot fire. The film
-# then starts two more runs of its own -- `stop_generation` and `send_turn` both submit a turn --
-# and the app pins to the bottom for them, which `runtime/session.py` already documents as
-# intended rather than a violation.
-#
+# The run the user started is a fresh intent to be at the end. THE GESTURE ON THE REAL FILM NEVER
+# ENDS AT THE BOTTOM: `SCROLL_JS` jumps to the bottom and then steps 14 x 420px away, so on any
+# thread taller than 5,880px the gesture ends thousands of pixels up and the `resume()`
+# re-attachment cannot fire. The film then starts two more runs of its own and the app pins to the
+# bottom for them, which `runtime/session.py` documents as intended.
+# `stop_generation` and `send_turn` both submit a turn.
# Measured at head over every 100K payload in `outputs/`: attached_fraction_of_stream 0.07 to 0.15
-# with reattachments 0, on the BASE arm as well as the treatment and on pure null controls, so the
-# gate's FOLLOW_MIN_STREAM_COVERAGE of 0.50 failed every 100K cell of every run -- two copies of
-# the shipped build included -- and a failed gate excludes the cell from scoring entirely. It
-# passed only on the 1K smoke film, where the thread is short enough that the gesture's reversal
-# lands back at the bottom by accident: a verdict about the thread's height, not about the app.
+# with reattachments 0, on the BASE arm as well as the treatment and on pure null controls, so
+# FOLLOW_MIN_STREAM_COVERAGE of 0.50 failed every 100K cell of every run and a failed gate excludes
+# the cell from scoring. It passed only on the 1K smoke film, where the gesture's reversal lands
+# back at the bottom by accident: a verdict about the thread's height, not the app.
+# ── the run the user started is a fresh intent to be at the end ─────
def _end_run(page) -> None:
"""The reply finishes: the stop control goes, so `isRunning()` is false."""
page.evaluate(
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py
index ebd51621f82..fc8e1f0e26e 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py
@@ -44,9 +44,9 @@ if str(_STUDIO_TESTS) not in sys.path:
from studiobench.runtime.types import ActionContext, Cell # noqa: E402
from studiobench.scene import actions as A # noqa: E402
-#: Unsloth's own geometry: a 20x20 action inside a 279x49 sticky header row, at the same offset the
-#: live probe measured. The numbers matter -- a control large enough to be hit by chance would let
-#: the test pass for the wrong reason.
+#: Unsloth's own geometry: a 20x20 action inside a 279x49 sticky header row, at the offset the live
+#: probe measured. The numbers matter: a control large enough to be hit by chance would let the test
+#: pass for the wrong reason.
FIXTURE = """
<!doctype html><meta charset="utf-8">
<style>
@@ -78,8 +78,8 @@ FIXTURE = """
SELECTOR = 'button[aria-label="New chat"]'
-#: A URL that actually loads. Pointing the fallback at a dead port exercises the "navigation
-#: failed too" branch instead of the navigation branch under test.
+#: A URL that actually loads. Pointing the fallback at a dead port exercises the "navigation failed
+#: too" branch instead of the navigation branch under test.
FALLBACK_URL = "about:blank"
@@ -108,8 +108,8 @@ def browser():
@pytest.fixture()
def page(browser):
- # A PAGE PER TEST, because the mouse position is page state and a previous test's hover would
- # carry the control into the next one already revealed.
+ # A PAGE PER TEST, because the mouse position is page state and a previous test's hover would carry
+ # the control into the next one already revealed.
pg = browser.new_page(viewport = {"width": 1280, "height": 900})
pg.set_content(FIXTURE)
yield pg
@@ -155,9 +155,9 @@ def test_hovering_reveals_the_control_and_returns_a_point_on_it(page):
point = A._reveal_by_hover(_ctx(page), SELECTOR)
assert point is not None, "hovering did not make the control hit-testable"
x, y = point
- # Against the control's OWN box, read from the page. Hard-coding the live app's (243, 319)
- # asserted the fixture's layout rather than the behaviour, and failed on a fixture that was
- # working correctly.
+ # Against the control's OWN box, read from the page. Hard-coding the live app's (243, 319) asserted
+ # the fixture's layout rather than the behaviour, and failed on a fixture that was working
+ # correctly.
box = page.eval_on_selector(SELECTOR, "(el) => el.getBoundingClientRect().toJSON()")
assert box["left"] <= x <= box["right"], (point, box)
assert box["top"] <= y <= box["bottom"], (point, box)
@@ -192,10 +192,9 @@ def test_an_ordinary_control_is_not_hovered_first(page):
the log rather than by asking `_reveal_by_hover` what it would have done."""
page.eval_on_selector(
SELECTOR,
- # `transition: none` as well: `opacity` is transitioned over 150ms, so `getComputedStyle`
- # sampled on the next tick still reports "0" and the control looks hidden to any check
- # that reads it. That is a real hazard for the production code too, and the reason the
- # reveal waits out `_REVEAL_SETTLE_MS` rather than sampling immediately.
+ # `transition: none` as well: `opacity` is transitioned over 150ms, so `getComputedStyle` sampled
+ # on the next tick still reports "0" and the control looks hidden to any check that reads it. A
+ # real hazard for the production code too, and the reason the reveal waits out `_REVEAL_SETTLE_MS`.
"(el) => { el.style.transition = 'none'; el.style.opacity = '1';"
" el.style.pointerEvents = 'auto'; }",
)
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py b/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py
index 54b3d328f33..6a6a6569ab6 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py
@@ -156,14 +156,13 @@ class _Page:
*_a,
**_k,
):
- # The end of the thread, which `thread_reopen` reads out of the DOM before it touches
- # anything so that it has a string to recognise the rebuilt thread by. It has to be a
- # string here or the action refuses on that precondition and never reaches the transition
- # this file is about.
+ # The end of the thread, which `thread_reopen` reads out of the DOM before it touches anything so
+ # it has a string to recognise the rebuilt thread by. It has to be a string here or the action
+ # refuses on that precondition.
if "data-role=" in script and "user" in script:
return "studiobench turn 8: continue with unit 3"
- # Otherwise a thread with messages in it, so `thread_reopen` gets past its own precondition
- # and reaches the transition under test.
+ # Otherwise a thread with messages in it, so `thread_reopen` gets past its own precondition and
+ # reaches the transition under test.
return 18
def wait_for_timeout(self, _ms):
@@ -244,9 +243,9 @@ class _RunningPage:
return 18
def wait_for_timeout(self, ms):
- # Actually sleeps, like Playwright's. A no-op here lets the bounded wait spin thousands of
- # times inside its deadline, which exhausted the scripted reply and made a test about a
- # reply that never lands pass for the opposite reason.
+ # Actually sleeps, like Playwright's. A no-op lets the bounded wait spin thousands of times inside
+ # its deadline, which exhausted the scripted reply and made a test about a reply that never lands
+ # pass for the opposite reason.
self.waits += 1
time.sleep(ms / 1000.0)
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py b/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py
index 6c73c7ec036..faceb7fd33c 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py
@@ -75,8 +75,8 @@ def test_no_shot_is_taken_when_none_was_asked_for():
def test_the_digest_call_never_takes_a_picture():
- # The digest is taken INSIDE the measured window and the shot outside it, so `_parity` must
- # not be a camera. If it becomes one again, the encode is charged to the film's own clock.
+ # The digest is taken INSIDE the measured window and the shot outside it, so `_parity` must not be a
+ # camera; if it becomes one again, the encode is charged to the film's own clock.
page = StubPage()
got = runner(page, parity_shots = "/nonexistent", arm_label = "base")._parity()
assert page.shots == []
@@ -84,8 +84,8 @@ def test_the_digest_call_never_takes_a_picture():
def test_the_shot_is_named_for_its_cell_action_and_arm(tmp_path):
- # The arm has to be IN the name. Both arms share a fixture, a film and a password, so a file
- # that says only "settings" cannot be attributed to a side once it leaves this directory.
+ # The arm has to be IN the name. Both arms share a fixture, a film and a password, so a file that
+ # says only "settings" cannot be attributed to a side once it leaves this directory.
page = StubPage(scroll = 940)
got = runner(page, parity_shots = str(tmp_path), arm_label = "treatment")._parity_shot("settings")
assert got["shot"] == "r100K.treatment.rep1__settings__treatment.png"
@@ -102,8 +102,8 @@ def test_the_two_arms_do_not_collide_on_one_filename(tmp_path):
def test_a_camera_failure_does_not_cost_the_measurement(tmp_path):
- # The digest is the reading; the picture is evidence about it. Losing the second must not
- # discard the first, or a flaky screenshot turns a green run red for no reason.
+ # The digest is the reading; the picture is evidence about it. Losing the second must not discard
+ # the first, or a flaky screenshot turns a green run red for no reason.
page = StubPage(shot_raises = True)
got = runner(page, parity_shots = str(tmp_path), arm_label = "base")._parity_shot("settings")
assert "shot" not in got
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py
index 0320ca08c28..9fe2ff7bcd6 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py
@@ -67,7 +67,7 @@ _QUEUE_BUTTON_RUNNING = """
</div>
"""
-#: The queued branch: `isQueueRunning && !thread.isRunning`, with the active item undispatched.
+#:The queued branch: `isQueueRunning && !thread.isRunning`, with the active item undispatched.
_QUEUE_BUTTON_IDLE = """
<button class="aui-composer-send ml-1.5 size-9 rounded-full" aria-label="Queue message">
<span class="aui-sr-only">Queue message</span>
@@ -81,10 +81,10 @@ _STOP_BUTTON = """
"""
#: The DISPATCHED queued branch: `isQueueRunning && !thread.isRunning` with `queueEntry.dispatched`,
-#: which renders a stop control the thread does not report itself running behind. Neither
-#: `stopButton()` nor `queueButton()` matches it, so on its own it reads exactly like a settled
-#: composer. `getPromptQueueUIItemsForRun` drops dispatched items, so the queue surface can be gone
-#: here too -- which is why this state has to be recognised from its own control.
+#: rendering a stop control the thread does not report itself running behind. Neither
+#: `stopButton()` nor `queueButton()` matches it, so on its own it reads like a settled composer,
+#: and `getPromptQueueUIItemsForRun` drops dispatched items so the queue surface can be gone too,
+#: which is why this state must be recognised from its own control.
_STOP_QUEUED_BUTTON = """
<button class="aui-composer-cancel ml-1.5 size-9 rounded-full" aria-label="Stop queued message">
<span class="aui-sr-only">Stop queued message</span>
@@ -92,8 +92,8 @@ _STOP_QUEUED_BUTTON = """
"""
#: `PromptQueueStack`, which renders inside the composer root whenever the run has an item left to
-#: show. An undispatched active item is always one of them, so this surface is up in every state
-#: that renders the queued-idle Queue button.
+#: show. An undispatched active item is always one, so this surface is up in every state that
+#: renders the queued-idle Queue button.
_QUEUE_STACK = """
<div aria-label="Prompt queue, 1 of 2">
<div>a prompt that has not been dispatched</div>
@@ -155,9 +155,9 @@ BLIND = dict(
queue_stack = False,
statuses = [None, None, None, None],
)
-#: The cost of reading the queue surface, pinned rather than left for a later reader to find: a
-#: queue run with a prompt still waiting, a reply genuinely streaming, and text in the composer.
-#: The surface is up and the Queue button is the only control, so the control is not armed.
+#: The cost of reading the queue surface, pinned rather than left for a later reader: a queue run
+#: with a prompt still waiting, a reply genuinely streaming, and text in the composer. The surface
+#: is up and the Queue button is the only control, so the control is not armed.
QUEUED_AND_STREAMING_BLIND = dict(
control = _QUEUE_BUTTON_RUNNING,
queue_stack = True,
@@ -202,7 +202,7 @@ def _capture(
) -> dict:
page.set_content(_page(tail = tail, **state))
# After the content, not before it: `set_content` does not reliably run init scripts, and the
- # symptom is `window.__sb` simply not existing, which reads like a broken instrument.
+ # symptom is `window.__sb` simply not existing.
page.add_script_tag(content = _DOM_JS.read_text(encoding = "utf-8"))
page.add_script_tag(content = _PARITY_JS.read_text(encoding = "utf-8"))
got = page.evaluate("() => window.__sb.parity.capture()")
@@ -298,14 +298,14 @@ def test_the_shipped_composer_still_renders_the_two_queue_buttons():
assert 'aria-label="Stop queued message"' in src
+#: An overlay is walked from `document`, OUTSIDE `.aui-thread-root`, so its digest carries neither
+#: the streamed message nor the composer, which makes it readable on a pair whose stream could not
+#: be placed.
# ── what the blind-probe refusal may NOT take out with it ────────────
-#: An overlay is walked from `document`, OUTSIDE `.aui-thread-root`. Its digest therefore carries
-#: neither the streamed message nor the composer, which is what makes it readable on a pair whose
-#: stream could not be placed.
_MENU = '<div role="menu"><div class="item">Rename</div></div>'
_MENU_CHANGED = '<div role="menu"><div class="item">Rename thread</div></div>'
-#: The composer of a thread that is NOT generating. `_STOP_BUTTON` is the same composer generating.
+#:The composer of a thread that is NOT generating. `_STOP_BUTTON` is the same composer generating.
_SEND_BUTTON = (
'<button class="aui-composer-send" aria-label="Send message">'
'<span class="aui-sr-only">Send message</span></button>'
@@ -396,12 +396,12 @@ def test_the_scaffold_is_not_an_independent_surface_and_here_is_why(page):
)
-# ── the composer is not a rendering difference ───────────────────────
-#
-# The pair this whole change is about: one arm has finished its reply, the other is still writing
-# it. Its messages are withheld correctly. Its COMPOSER was not, because the dock is inside
-# `.aui-thread-root` and the scaffold therefore carries Stop on one arm and Send on the other.
+# The composer is not a rendering difference. The pair this change is about: one arm has finished
+# its reply, the other is still writing it. Its messages are withheld correctly; its COMPOSER was
+# not, because the dock is inside `.aui-thread-root` and the scaffold therefore carries Stop on
+# one arm and Send on the other.
+# ── the composer is not a rendering difference ───────────────────────
_SETTLED_STATUSES = ["complete", "complete", "complete", "complete"]
_STREAMING_STATUSES = ["complete", "complete", "complete", "running"]
@@ -613,16 +613,14 @@ def test_the_queued_idle_arm_against_a_settled_one_is_still_refused(page):
assert P.compare(base, treat)["verdict"] == P.NOT_COMPARABLE
-# ── the style probe walks the run-state control too ──────────────────
-#
-# `report` now collects the style verdict BEFORE it buckets a structural refusal, because the
-# computed-style probe is an independent reading and the refusal is not about it. That makes the
-# probe's own reading of the composer swap visible for the first time, and the probe walks
-# `button[aria-label="Send message"]` and `button[aria-label="Stop generating"]` as SEPARATE
-# selectors whose names go into its signature. So the pairs the refusals above exist to withhold
-# arrived at the advisory line instead.
+# The style probe walks the run-state control too. `report` now collects the style verdict BEFORE
+# it buckets a structural refusal, because the computed-style probe is an independent reading,
+# and that makes its own reading of the composer swap visible for the first time: it walks the
+# Send and Stop buttons as SEPARATE selectors whose names go into its signature, so the pairs the
+# refusals exist to withhold arrived at the advisory line instead.
+# ── the style probe walks the run-state control too ──────────────────
def _capture_html_raw(page, html: str) -> dict:
"""`_capture_html`, keeping `styles.sig` so a test can say WHY the digest moved."""
page.set_content(html)
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py
index 7557cc10d09..9b7fca78677 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py
@@ -135,7 +135,7 @@ def test_it_opens_exactly_one_pane_and_leaves_the_rest_shut(make_page):
assert got.expect_ok is True, got.reason
assert got.expect["open_after_expand"] == 1
assert got.expect["open_after_collapse"] == 0
- # The thread's pane count is reported as CONTEXT beside a fixed gesture. Without it the reading
+ # The thread's pane count is reported as CONTEXT beside a fixed gesture; without it the reading
# cannot be told apart from the thread-wide one in a payload.
assert got.expect["panes"] == 8
assert got.expect["panes_opened"] == 1
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py
index b3cbce9c544..cf01d0a4144 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py
@@ -208,8 +208,8 @@ def test_a_slow_state_flip_does_not_bank_quiet_frames_before_it():
)
assert out["spansOpen"] == SPANS_SETTLED
assert out["openCensored"] is False
- # The flip and the settled read are different moments, and both are reported so a reader can
- # see how much of open_ms was spent after the state arrived.
+ # The flip and the settled read are different moments, and both are reported so a reader can see
+ # how much of open_ms was spent after the state arrived.
assert out["openStateReachedMs"] < out["openMs"]
@@ -307,16 +307,16 @@ def test_the_state_reached_mark_shares_the_timing_origin():
assert out["openStateReachedMs"] <= out["openMs"]
-# ── the close direction ─────────────────────────────────────────────────────────────────────────
-#
-# The same defect the quiet streak fixes for the open direction, arriving from the other side.
-# `data-state` flips on the click, and BOTH collapse mechanisms keep the children in the document
-# until the exit animation ends: Radix's `Presence` until `animationend`, the grid arm's
-# `UnmeasuredCollapsibleContent` until the `grid-template-rows` `transitionend` or its 250 ms
-# backstop. So `pre span` is frozen at its open value for that whole window, and a streak that only
-# asks whether the census has stopped moving is satisfied by a census that has not started.
+# The close direction: the same defect the quiet streak fixes for the open direction, from the
+# other side. `data-state` flips on the click and BOTH collapse mechanisms keep the children in
+# the document until the exit animation ends (Radix's `Presence` until `animationend`, the grid
+# arm until `transitionend` or its 250 ms backstop) so `pre span` is frozen at its open value and
+# a streak that only asks whether the census has stopped moving is satisfied by one that has not
+# started.
+# The grid arm is `UnmeasuredCollapsibleContent`.
+# ── the close direction ─────────────────────────────────────────────────────────────────────────
def test_a_collapse_is_not_settled_while_its_panes_are_still_mounted():
"""THE REGRESSION. `close_ms` must not name the state flip plus four frames.
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py b/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py
index 3d09ca0f419..455a38233c9 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py
@@ -72,18 +72,18 @@ from studiobench.scene.actions import ( # noqa: E402
)
from studiobench.scene.schedule import FAST, QUICK, STANDARD # noqa: E402
-#: One driver call over CDP. Measured at 43 - 130 ms across nine to thirteen `page.evaluate` calls
-#: in one run of the shipped action against real chromium, and the two big ones in that total are
-#: page-side waits charged separately below, so about 4 ms is the round trip itself.
+#: One driver call over CDP. Measured at 43 - 130 ms across nine to thirteen `page.evaluate` calls in
+#: one run of the shipped action against real chromium, and the two big ones are page-side waits
+#: charged separately below, so about 4 ms is the round trip itself.
ROUND_TRIP_MS = 4.0
-#: Enter pressed to `isRunning()` answering true: a POST to the relay and the first SSE frame back.
+#:Enter pressed to `isRunning()` answering true: a POST to the relay and the first SSE frame back.
START_MS = 120.0
-#: Stop clicked to `isRunning()` answering false: the abort round trip.
+#:Stop clicked to `isRunning()` answering false: the abort round trip.
STOP_MS = 90.0
-#: The delete inside `STOP_CLEANUP_JS`: the click, the removal, and the two `__sbNextPaint` frames
+#:The delete inside `STOP_CLEANUP_JS`: the click, the removal, and the two `__sbNextPaint` frames the loop waits on.
#: the loop waits on.
CLEANUP_MS = 90.0
@@ -114,10 +114,9 @@ class _Keyboard:
if not self._page.accepts_send:
# `queueDisabled` in thread.tsx: the press queued nothing and the box keeps its text.
return
- # SENT, NOT STARTED, and the difference is the whole of the P1 above. The app takes the
- # text out of the composer and puts the user turn and its reply into the thread NOW; the
- # reply begins generating `start_ms` later, and the action has to poll for it -- which is
- # the cost the first version of this shim handed out free.
+ # SENT, NOT STARTED, and the difference is the whole of the P1 above: the app takes the text out of
+ # the composer and puts the user turn and its reply into the thread NOW, while the reply begins
+ # generating `start_ms` later and the action has to poll for it.
self._page.composer = ""
self._page.messages += 2
self._page.sent_at_ms = self._page.elapsed_ms
@@ -210,10 +209,9 @@ class _Page:
return self.composer
if "messageCount" in script:
return self.messages
- # The thread's length as well as the mounted count. `stop_generation` proves its own turn
- # was added by `threadTotal()`, so that a windowed arm whose window refills is not read as
- # a send that added nothing and left with nothing to clean up. This page models a fully
- # mounted arm, where the two are the same number.
+ # The thread's length as well as the mounted count: `stop_generation` proves its own turn was added
+ # by `threadTotal()`, so a windowed arm whose window refills is not read as a send that added
+ # nothing. This page models a fully mounted arm, where the two are the same number.
if "threadTotal" in script:
return self.messages
if "assistantChars" in script:
@@ -294,8 +292,8 @@ def test_a_reply_that_drains_at_the_end_of_the_slot_does_not_spend_the_next_one(
f"{scene.name}: stop_generation spent {page.elapsed_ms:.0f}ms of a {stop.budget_ms}ms "
f"slot with only {slack_ms}ms before {nxt.action} opens"
)
- # And it stopped by declining, not by stopping the cell's own reply, which is the whole point
- # of the wait it just gave up on.
+ # And it stopped by declining, not by stopping the cell's own reply, which is the whole point of
+ # the wait it just gave up on.
assert result.ran is False
assert page.clicked == 0
assert "one more" not in page.filled
@@ -401,8 +399,8 @@ def test_a_turn_that_starts_after_the_slot_bound_is_not_left_generating(
stop, _nxt, _slack = _stop_slot(scene)
settled = _thread_a_measured_turn_leaves(monkeypatch, stop.budget_ms)
- # Past the slot bound, which is the budget less what the rest of the turn costs, and still
- # inside the time the turn is worth waiting for so it can be taken back.
+ # Past the slot bound, which is the budget less what the rest of the turn costs, and still inside
+ # the time the turn is worth waiting for so it can be taken back.
start_ms = stop.budget_ms - 1_000 + late_by_ms
result, page = _run(
@@ -586,8 +584,8 @@ def test_the_reserve_is_still_the_whole_of_what_its_two_halves_reserve():
assert OWN_TURN_POLL_MS == OWN_TURN_START_POLL_MS + OWN_TURN_STOP_POLL_MS
# The 80 ms that settles the fill is the only fixed sleep before the send.
assert OWN_TURN_FIXED_MS - OWN_TURN_FIXED_AFTER_SEND_MS == 80
- # And what the turn-start wait holds back has to leave the start poll something to spend, or
- # the wait is an immediate refusal and the throwaway turn is unreachable on the tightest drain.
+ # And what the turn-start wait holds back has to leave the start poll something to spend, or the
+ # wait is an immediate refusal and the throwaway turn is unreachable on the tightest drain.
assert (
OWN_TURN_RESERVE_MS - 80 - OWN_TURN_FIXED_AFTER_SEND_MS - OWN_TURN_STOP_POLL_MS
== OWN_TURN_START_POLL_MS
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py b/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py
index 5d22d8749b4..6cab4b7ff25 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py
@@ -46,7 +46,7 @@ BASE_URL = "http://127.0.0.1:1"
THREAD_URL = f"{BASE_URL}/chat?thread=t1"
NEW_CHAT_URL = f"{BASE_URL}/chat?new=studiobench"
-#: The length of the seeded thread, and the marker the seeder wrote into its last user turn.
+#:The length of the seeded thread, and the marker the seeder wrote into its last user turn.
TOTAL = 18
MARKER = "studiobench turn 8: continue with unit 3"
@@ -80,13 +80,13 @@ REBUILD = (
)
#: A rebuild that never arrives: the store keeps declaring eighteen messages and the thread stops
-#: at three of them, settled, with the end of the conversation nowhere on screen. The exact shape
-#: the old condition scored as a fast, successful re-open.
+#: at three, settled, with the end of the conversation nowhere on screen. The exact shape the old
+#: condition scored as a fast, successful re-open.
STALLED = (_Frame(mounted = 3, elements = 1_200, scroll_height = 2_000, spans = 0, marker = False),)
-#: A correctly windowed rebuild: a window of six rows out of eighteen, anchored at the end, with
-#: the last user turn among them. `full` conditions would refuse this forever, which is why the
-#: action picks its mode from the mount it left rather than assuming one.
+#: A correctly windowed rebuild: six rows out of eighteen, anchored at the end, with the last user
+#: turn among them. `full` conditions would refuse this forever, which is why the action picks its
+#: mode from the mount it left.
WINDOWED = (
_Frame(mounted = 2, elements = 900, scroll_height = 11_800, spans = 0, marker = False),
_Frame(mounted = 6, elements = 3_400, scroll_height = 12_400, spans = 2_100, marker = True),
@@ -120,8 +120,6 @@ class _ThreadPage:
self.goto_calls: list[str] = []
self.probes = 0
- # ── what the page would report ──────────────────────────────────
-
@property
def frame(self) -> _Frame:
return self.frames[min(self.step, len(self.frames) - 1)]
@@ -169,8 +167,6 @@ class _ThreadPage:
"pinning": False,
}
- # ── the Playwright surface the action uses ──────────────────────
-
def evaluate(
self,
script,
@@ -186,9 +182,9 @@ class _ThreadPage:
return MARKER if self.phase != "gone" else None
if "pre span" in script:
return self.frame.spans if self.phase == "rebuild" else 0
- # The hit-test spread and the hover target, both of which report "no reachable point" for
- # an unclickable control. Returning None here is what sends `_click_or_navigate` down the
- # branch under test rather than into an off-centre click.
+ # The hit-test spread and the hover target, both of which report "no reachable point" for an
+ # unclickable control. Returning None sends `_click_or_navigate` down the branch under test
+ # rather than into an off-centre click.
return None
def query_selector(self, selector):
@@ -208,8 +204,8 @@ class _ThreadPage:
def wait_for_timeout(self, _ms):
# A REAL, TINY SLEEP. The readiness wait is bounded on the monotonic clock, so a poll that
- # returned instantly would spin thousands of times inside the timeout instead of walking
- # the scripted frames.
+ # returned instantly would spin thousands of times inside the timeout instead of walking the
+ # scripted frames.
time.sleep(0.002)
if self.phase == "rebuild":
self.step += 1
@@ -425,12 +421,12 @@ def test_a_thread_whose_end_cannot_be_identified_is_refused_before_it_is_touched
assert page.phase == "thread" and page.goto_calls == []
+#: How long the centre click burns before it gives up. `_click_or_navigate` passes `timeout = 2000`
+#: to Playwright and the hit-target check retries for the whole of it against a control it cannot
+#: hit at the centre. Scaled down here; what is asserted is that the timings do not contain it, at
+#: whatever size.
# ── defect three: the harness's own retry, billed to the rebuild ────
-#: How long the centre click burns before it gives up. `_click_or_navigate` passes
-#: `timeout = 2000` to Playwright, and the hit-target check retries for the whole of it against a
-#: control it cannot hit at the centre. Scaled down here so the test is fast; what is asserted is
-#: that the timings do not contain it, at whatever size it is.
RETRY_MS = 400
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py
index 803ad9528ed..dac07a4b42b 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py
@@ -70,34 +70,33 @@ def _page(
rows = []
for i in range(1, 5):
if drop_tail and i == 4:
- # The windowed case: the arm still DECLARES four messages through `aria-setsize`, and
- # has unmounted the one it is writing into. That is a thread the instrument can see
- # three quarters of, not an instrument that has stopped working.
+ # The windowed case: the arm still DECLARES four messages through `aria-setsize` and has unmounted
+ # the one it is writing into. That is a thread the instrument can see three quarters of, not an
+ # instrument that has stopped working.
continue
role = "user" if i % 2 else "assistant"
last = i == 4
if last:
- # `live_role` is what the arm CALLS the row it is writing into. The shipped build says
- # assistant; a build that says otherwise is the regression ordinal-role parity exists
- # to catch, and it is asked here rather than assumed.
+ # `live_role` is what the arm CALLS the row it is writing into. The shipped build says assistant; a
+ # build that says otherwise is the regression ordinal-role parity exists to catch, and it is asked
+ # here rather than assumed.
role = live_role
body = tail if last else f"message {i}"
if i == 3:
body = user_body
if last and not tail_has_parts:
- # The gap between the send being accepted and the reply's first part arriving. The
- # assistant message is mounted and publishes nothing, because it has nothing to
- # publish: thread.tsx renders "Generating..." in place of any part.
+ # The gap between the send being accepted and the reply's first part arriving. The assistant
+ # message is mounted and publishes nothing, because thread.tsx renders "Generating..." in place of
+ # any part.
rows.append(
f'<div class="row" aria-posinset="{i}" aria-setsize="4">'
f'<div data-role="{role}"><span>Generating...</span></div></div>'
)
continue
status = running_value if (last and tail_running) else "complete"
- # THE HOOK IS RENAMED ON EVERY MESSAGE, not only the streamed one. `data-status` is a single
- # line in markdown-text.tsx and it is rendered for `complete` parts as well as `running`
- # ones, so a build that renames it renames it everywhere. A fixture that renamed it on one
- # message would be describing a build that does not exist, and would then be the only
+ # THE HOOK IS RENAMED ON EVERY MESSAGE, not only the streamed one: `data-status` is a single line
+ # in markdown-text.tsx rendered for `complete` parts as well as `running` ones. A fixture that
+ # renamed it on one message would describe a build that does not exist, and would then be the only
# evidence that the control fires.
attr = f'{hook}="{status}"'
rows.append(
@@ -167,14 +166,13 @@ def _capture(browser, **kw) -> dict:
page.close()
-#: The base arm: an ordinary settled thread on the shipped build.
+#:The base arm: an ordinary settled thread on the shipped build.
_SETTLED = dict(tail = "the whole reply, arrived", tail_running = False, generating = False)
-#: THE TREATMENT THAT WENT BLIND, and the shape matters. `data-status` is one line in
-#: markdown-text.tsx and it is rendered for `complete` parts as well as `running` ones, so a build
-#: that renames the ATTRIBUTE renames it on every message and every settled row differs too -- which
-#: is a rendering difference in its own right and is reported as one. The interesting blindness is a
-#: build that changed the status VOCABULARY: `complete` still reads `complete`, so every settled row
-#: is byte-identical, and the only thing lost is the ability to see that a reply is being written.
+#: THE TREATMENT THAT WENT BLIND, and the shape matters. A build that renames the ATTRIBUTE renames
+#: it on every message, so every settled row differs too, which is a rendering difference in its
+#: own right. The interesting blindness is a build that changed the status VOCABULARY: `complete`
+#: still reads `complete`, so every settled row is byte-identical and the only thing lost is the
+#: ability to see that a reply is being written.
_BLIND = dict(
tail = "the whole reply, arr", running_value = "streaming", tail_running = True, generating = True
)
@@ -183,7 +181,7 @@ _BLIND = dict(
_BLIND_ATTR = dict(
tail = "the whole reply, arr", hook = "data-state", tail_running = True, generating = True
)
-#: The same mid-stream moment on a build whose hook IS known. Already handled, as residue.
+#:The same mid-stream moment on a build whose hook IS known. Already handled, as residue.
_MIDSTREAM = dict(tail = "the whole reply, arr", tail_running = True, generating = True)
@@ -254,16 +252,14 @@ def test_a_lost_conversation_is_still_a_finding_while_a_reply_runs(browser):
assert "DIFFERENT MESSAGES on screen" in got["reason"]
-# ── what the refusal may NOT take out with it ────────────────────────
-#
-# The blind-probe refusal is about rows whose meaning depends on where the stream had got to. Two
-# kinds of row in this payload provably do not: a row both arms call the user's, because a reply is
-# written into an assistant message; and a row whose ROLE changed, because a role is captured beside
-# the digest and how far a reply has arrived says nothing about whose it is. Both used to leave here
-# as NOT COMPARABLE with an empty `moved`, and `visible_report` buckets a refusal as blind and never
-# consults it for the exit code, so the run went green on them.
+# What the refusal may NOT take out with it. The blind-probe refusal is about rows whose meaning
+# depends on where the stream had got to; two kinds provably do not: a row both arms call the
+# user's, and a row whose ROLE changed. Both used to leave here as NOT COMPARABLE with an empty
+# `moved`, and `visible_report` buckets a refusal as blind and never consults it for the exit
+# code, so the run went green on them.
+# ── what the refusal may NOT take out with it ────────────────────────
def test_a_changed_user_row_survives_the_blind_refusal(browser):
base = _capture(browser, **_SETTLED)
treat = _capture(browser, **dict(_BLIND, user_body = "the user message, rewritten"))
diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py
index f238343509d..73b84cd9579 100644
--- a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py
+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py
@@ -40,9 +40,9 @@ if str(_STUDIO_TESTS) not in sys.path:
_DOM_JS = _STUDIO_TESTS / "studiobench" / "scene" / "dom.js"
_PARITY_JS = _STUDIO_TESTS / "studiobench" / "scene" / "parity.js"
-#: A thread of tall messages in a short viewport, so only a few are ever on screen at once. Each
-#: message publishes `aria-posinset`, which is how a windowed arm states thread position; the
-#: capture keys on it so a window and a full mount are comparable.
+#: A thread of tall messages in a short viewport, so only a few are ever on screen. Each message
+#: publishes `aria-posinset`, which is how a windowed arm states thread position, and the capture
+#: keys on it so a window and a full mount are comparable.
FIXTURE = """
<!doctype html><meta charset="utf-8">
<style>
@@ -108,7 +108,7 @@ def page(browser):
pg.set_content(FIXTURE)
# `add_script_tag` after the content, not `add_init_script` before it: Playwright's
# `set_content` does not always run init scripts, and the symptom is `window.__sb` simply not
- # existing, which reads like a broken instrument rather than a mis-ordered fixture.
+ # existing, which reads like a broken instrument.
pg.add_script_tag(content = _DOM_JS.read_text(encoding = "utf-8"))
pg.add_script_tag(content = _PARITY_JS.read_text(encoding = "utf-8"))
yield pg
@@ -119,7 +119,7 @@ def _watch(page) -> None:
got = page.evaluate("() => window.__sb.parityVisible.watch()")
assert got.get("visible_attempted") is True, got
# IntersectionObserver's first delivery is asynchronous, so give it a frame before anything
- # scrolls. Without this the initial viewport contents are attributed to whatever came next.
+ # scrolls, or the initial viewport contents are attributed to whatever came next.
page.wait_for_timeout(120)
@@ -306,20 +306,17 @@ def test_a_row_mounted_during_the_action_is_still_picked_up_cheaply(page):
assert 3 in got["ever_visible"], got["ever_visible"]
-# ── where the virtualizer publishes its ordinals is not a UI change ──
-#
-# `runtime/readiness.py` accepts `aria-posinset` / `aria-setsize` on the `[data-role]` message OR on
-# an ancestor row wrapper -- it walks with `closest()`, because the ordinal belongs on whichever
-# element is the member of the set, and refusing the first option would refuse a correctly
-# implemented arm for putting the attribute in a place the gate itself calls right.
-#
-# The visible-region digest then read every attribute on the message, so an arm that took that
-# option differed from the fully mounted arm on EVERY message -- which publishes neither attribute
-# anywhere -- while the rendered content was identical. Auto-mode parity was unusable for a DOM
-# shape the gate explicitly permits, and a wall of differences that are all the same non-finding
-# buries anything real underneath it.
+# Where the virtualizer publishes its ordinals is not a UI change. `runtime/readiness.py` accepts
+# `aria-posinset` / `aria-setsize` on the `[data-role]` message OR on an ancestor row wrapper,
+# walking with `closest()`, because the ordinal belongs on whichever element is the member of
+# the set.
+# The visible-region digest then read every attribute on the message, so an arm taking that
+# option differed from the fully mounted arm on EVERY message while the rendered content was
+# identical. Auto-mode parity was unusable for a DOM shape the gate explicitly permits, and a
+# wall of identical non-findings buries anything real.
+# ── where the virtualizer publishes its ordinals is not a UI change ──
def _arm_html(ordinals: str, suffix: str = "") -> str:
"""One thread of twenty messages, with the virtualization ordinals published `on_the_message`,
`on_the_row` wrapper, or `nowhere` -- which is what the shipped build does."""
@@ -423,20 +420,17 @@ def test_a_real_rendering_difference_is_still_caught(browser):
assert verdict["verdict"] == P.DIFFER, verdict
-# ── a rebuilt row is at its own position, not at the end of a lifetime count ──
-#
-# `thread_reopen` leaves the thread and comes back, and a FULLY MOUNTED arm answers that by
-# removing every message row and creating a new one for every message inside the same document.
-# Those rebuilt rows publish no `aria-posinset` -- only a windowed arm publishes one -- so their
-# ordinal comes from the fallback, and the fallback used to be a LIFETIME counter of observed
-# nodes. It already stood at N, so the rebuilt rows were stamped N+1..2N while the windowed arm on
-# the other side of the A/B stamped its real 1..N. `compare_visible` compares the two sets of
-# ordinals first, found them disjoint, and reported "the two arms put DIFFERENT MESSAGES on
-# screen" -- a hard visible-difference verdict -- for a rebuild that was identical.
+# A rebuilt row is at its own position, not at the end of a lifetime count. `thread_reopen`
+# leaves the thread and comes back, and a FULLY MOUNTED arm removes every message row and
+# creates a new one inside the same document. Those rebuilt rows publish no `aria-posinset`, so
+# their ordinal came from a LIFETIME counter already standing at N: they were stamped N+1..2N
+# while the windowed arm stamped its real 1..N, so `compare_visible` found the two ordinal sets
+# disjoint and reported a hard visible difference for a rebuild that was identical.
#: The shipped shape: every message mounted, and NOTHING publishing a virtualization ordinal.
#: `__rebuild()` is the thread_reopen rebuild, in one commit, in the same document.
+# ── a rebuilt row is at its own position, not at the end of a lifetime count ──
REBUILD_FIXTURE = """
<!doctype html><meta charset="utf-8">
<style>
@@ -606,15 +600,13 @@ def test_the_structural_digest_still_sees_the_ordinals(browser):
assert all(numbered[i] != plain[i] for i in numbered), (numbered, plain)
+#: An EMPTY viewport, so a row appended to it is on screen immediately. `__churn` mounts and
+#: unmounts a row inside ONE task, which is how a row comes to be observed while detached: the
+#: MutationObserver batch runs after both operations, so the node is in `addedNodes` and no
+#: longer among the document's messages and has no position the instrument can honestly claim.
+#: `__remount` hands THE SAME NODE back, as a virtualizer that recycles rows does.
# ── a recycled node, and the ordinal it used to be denied ──────────────────
-
-#: An EMPTY viewport, so a row appended to it is on screen immediately. `__churn` mounts a row and
-#: unmounts it inside ONE task, which is how a row comes to be observed while detached: the
-#: MutationObserver batch runs after both operations, the node is in `addedNodes` and is no longer
-#: among the document's messages, so it has no position the instrument can honestly claim.
-#: `__remount` then hands THE SAME NODE back, which is what a virtualizer that recycles its rows
-#: does on the next scroll step.
RECYCLE_FIXTURE = """
<!doctype html><meta charset="utf-8">
<style>
@@ -674,9 +666,9 @@ def test_a_recycled_row_is_placed_when_it_finally_mounts(browser):
assert got["ever_visible"] == [1], got
-#: A row that is RENUMBERED IN PLACE. It stays connected, stays intersecting, and is handed to
-#: another message -- which is what a virtualizer that recycles its row nodes does, and it is not a
-#: childList mutation, so nothing about it reaches a childList-only observer.
+#: A row that is RENUMBERED IN PLACE: it stays connected, stays intersecting, and is handed to
+#: another message, as a virtualizer that recycles row nodes does. It is not a childList
+#: mutation, so nothing about it reaches a childList-only observer.
RENUMBER_FIXTURE = """
<!doctype html><meta charset="utf-8">
<style>
@@ -728,7 +720,7 @@ def test_a_row_renumbered_in_place_is_restamped_and_reported(browser):
assert "42" in got["messages"], got["messages"]
assert got["unplaced_rows"] == 0, got
# A LEGITIMATE RENUMBER IS NOT A COLLISION. One node holds one position at a time here, so the
- # collision counter added for the two-rows-one-ordinal case must not fire on this, or a correct
+ # collision counter added for the two-rows-one-ordinal case must not fire, or a correct
# recycling virtualizer would be refused on every action it recycles a row in.
assert got["ordinal_collisions"] == 0, got
@@ -793,8 +785,8 @@ def test_two_rows_sharing_a_thread_position_are_counted_rather_than_overwritten(
got = _capture_with_ghost(browser, before = before)
assert got["ordinal_collisions"] == 1, (before, got)
assert got["collided_ordinals"] == [1], (before, got)
- # The counter is not derived from the arithmetic, and this is why: two of the three rows
- # on screen are filed under one key, so the map still holds two entries.
+ # The counter is not derived from the arithmetic, and this is why: two of the three rows on
+ # screen are filed under one key, so the map still holds two entries.
assert set(got["messages"]) == {"1", "2"}, got["messages"]
@@ -834,8 +826,8 @@ def test_losing_the_thread_outranks_the_collision_refusal(browser):
ghosted = _capture_with_ghost(browser, before = True)
assert ghosted["ordinal_collisions"] == 1, ghosted
- # The other arm ended with nothing on screen at all, which is the 100K `model_change` shape:
- # 12 mounted messages to 0, never recovered.
+ # The other arm ended with nothing on screen at all, the 100K `model_change` shape: 12 mounted
+ # messages to 0, never recovered.
empty = dict(ghosted)
empty["messages"] = {}
empty["ordinal_collisions"] = 0
diff --git a/tests/studio/studiobench/scene/surface_sweep.py b/tests/studio/studiobench/scene/surface_sweep.py
index 2fab49e6ba0..d925dc4d938 100644
--- a/tests/studio/studiobench/scene/surface_sweep.py
+++ b/tests/studio/studiobench/scene/surface_sweep.py
@@ -29,14 +29,14 @@ from typing import Any, Callable, Optional
from . import surfaces as registry
-#: How long a settle condition is given before the surface is recorded as unreached. Generous:
-#: the settings panels are lazily imported chunks and the hub's first paint waits on a network
-#: round trip, and a sweep that calls those failures is a sweep nobody believes.
+#: How long a settle condition is given before the surface is recorded as unreached. Generous: the
+#: settings panels are lazily imported chunks and the hub's first paint waits on a network round
+#: trip, and a sweep that calls those failures is a sweep nobody believes.
SETTLE_TIMEOUT_MS = 8000
SETTLE_POLL_MS = 150
-#: A per-surface ceiling covering reach, settle and capture together. A surface whose reach
-#: navigates into a redirect loop would otherwise hold the sweep open indefinitely.
+#: A per-surface ceiling covering reach, settle and capture together, so a reach that navigates
+#: into a redirect loop cannot hold the sweep open indefinitely.
SURFACE_BUDGET_MS = 25_000
@@ -71,10 +71,9 @@ class _Driver:
if verb == "goto":
page.goto(f"{self.base_url}{args[0]}", wait_until = "domcontentloaded", timeout = 60_000)
elif verb == "click":
- # A REAL mouse click through the driver, not element.click(). Radix menus open on
- # pointerdown, and a synthetic click never opens them -- which reads downstream as a
- # menu that opened in zero milliseconds rather than as a menu that did not open. This
- # is the same trap dom.js documents for the film's actions.
+ # A REAL mouse click through the driver, not element.click(): Radix menus open on pointerdown and
+ # a synthetic click never opens them, which reads downstream as a menu that opened in zero
+ # milliseconds. The same trap dom.js documents for the film's actions.
page.click(args[0], timeout = 6000)
elif verb == "click_if":
el = page.query_selector(args[0])
@@ -144,9 +143,8 @@ def _row(surface: registry.Surface, cell_id: Optional[str]) -> dict:
"reason": "the sweep did not get to this surface",
"conditional": surface.conditional,
"also_in_film": surface.also_in_film,
- # Carried on the ROW, not only in the registry, so a reader of a payload can tell a
- # digest that is a parity signal from one that moves on its own without needing this
- # source tree in front of them.
+ # Carried on the ROW, not only in the registry, so a reader of a payload can tell a digest that is
+ # a parity signal from one that moves on its own without this source tree in front of them.
"volatile": surface.volatile,
"parity": {"parity_attempted": False, "reason": "not captured"},
"settle": None,
@@ -188,9 +186,9 @@ def sweep(
raise registry.RegistryError(f"no such surface(s): {missing}")
entries = [s for s in entries if s.id in wanted]
- # Does parity.capture() honour a moved root? If it does not, every surface digest is the same
- # page-wide reading and the sweep's forty passes mean nothing. Asked ONCE, up front, and the
- # answer is carried on every row rather than assumed.
+ # Does parity.capture() honour a moved root? If not, every surface digest is the same page-wide
+ # reading and the sweep's forty passes mean nothing. Asked ONCE, up front, and the answer is
+ # carried on every row.
home = driver.run(registry.HOME)
scoping = {
"scoped": False,
@@ -216,9 +214,9 @@ def sweep(
started = time.monotonic()
deadline = started + budget_ms / 1000
- # From the KNOWN STATE, never from wherever the previous surface left the app. A reach
- # that happens to work only because the last surface left a menu open is a reach that
- # breaks the first time the order changes, and it would break silently.
+ # From the KNOWN STATE, never from wherever the previous surface left the app: a reach that works
+ # only because the last surface left a menu open breaks the first time the order changes, and
+ # silently.
reset = driver.run(registry.HOME)
if reset is not None:
row["reason"] = f"the known state could not be restored before this surface: {reset}"
@@ -230,8 +228,8 @@ def sweep(
failed = driver.run(surface.reach)
if failed is not None:
row["reason"] = f"the reach failed: {failed}"
- # The facts are still recorded. Where the app ACTUALLY ended up is the first thing a
- # reader of a failed reach needs, and it is gone by the time anyone looks.
+ # The facts are still recorded. Where the app ACTUALLY ended up is the first thing a reader of a
+ # failed reach needs, and it is gone by the time anyone looks.
row["facts"] = driver.facts(surface.root)
row["reach_ms"] = round((time.monotonic() - started) * 1000, 1)
rows.append(row)
@@ -258,9 +256,9 @@ def sweep(
parity = driver.capture(surface.root)
row["parity"] = parity
if not parity.get("parity_attempted"):
- # Reached, but no digest. Recorded as unreached FOR PARITY PURPOSES, because a
- # surface with no digest contributes nothing to the parity claim, and counting it as
- # covered is exactly the inflation this file is written to avoid.
+ # Reached, but no digest. Recorded as unreached FOR PARITY PURPOSES, because a surface with no
+ # digest contributes nothing to the parity claim and counting it as covered is the inflation this
+ # file avoids.
row["reason"] = f"the surface rendered but no digest was taken: {parity.get('reason')}"
rows.append(row)
_emit(recorder, row)
@@ -285,9 +283,9 @@ def sweep(
row["restore_reason"] = restored
_recover(driver, log)
elif (clean.get("open_dialogs") or 0) or (clean.get("open_menus") or 0):
- # The declared restore ran and the app is still dirty. Said out loud: the next
- # surface would otherwise be reached from a state nobody declared, and its digest
- # would quietly include a leftover overlay.
+ # The declared restore ran and the app is still dirty. Said out loud: the next surface would
+ # otherwise be reached from a state nobody declared and its digest would include a leftover
+ # overlay.
row["restore_reason"] = (
f"the restore ran but left {clean.get('open_dialogs')} dialog(s) and "
f"{clean.get('open_menus')} menu(s) open"
@@ -306,8 +304,8 @@ def _emit(recorder: Any, row: dict) -> None:
try:
recorder.emit(dict(row))
except Exception: # noqa: BLE001
- # A recorder that rejects a row must not cost the sweep the rest of its surfaces. The row
- # is still in the returned list and in the manifest.
+ # A recorder that rejects a row must not cost the sweep the rest of its surfaces. The row is still
+ # in the returned list and in the manifest.
pass
@@ -323,9 +321,9 @@ def build_manifest(rows: list, entries: list, scoping: dict) -> dict:
"""The coverage manifest: what was swept, what was not, and what is out of reach by design."""
reached = [r for r in rows if r.get("reached")]
failed = [r for r in rows if not r.get("reached")]
- # A conditional surface that did not render is not a hole in the registry: it is a property of
- # this installation. Kept separate so the coverage figure is not quietly deflated by a host
- # that has no GPU, and not quietly inflated by counting it as covered either.
+ # A conditional surface that did not render is not a hole in the registry but a property of this
+ # installation. Kept separate so the coverage figure is not quietly deflated by a host with no
+ # GPU, nor inflated by counting it as covered.
conditional_misses = [r for r in failed if r.get("conditional")]
hard_misses = [r for r in failed if not r.get("conditional")]
registered = len(entries)
@@ -337,8 +335,8 @@ def build_manifest(rows: list, entries: list, scoping: dict) -> dict:
"not_reached": len(failed),
"not_reached_conditional": len(conditional_misses),
"not_reached_hard": len(hard_misses),
- # Against the surfaces that COULD have rendered on this host. Reported next to the raw
- # count, never instead of it.
+ # Against the surfaces that COULD have rendered on this host. Reported next to the raw count,
+ # never instead of it.
"coverage_pct": (
round(100.0 * len(reached) / (registered - len(conditional_misses)), 1)
if registered - len(conditional_misses) > 0
@@ -363,9 +361,9 @@ def build_manifest(rows: list, entries: list, scoping: dict) -> dict:
for r in rows
if r.get("restored") is False
],
- # Reached, digested, and NOT a parity signal. Counted separately because the number that
- # matters to somebody comparing two arms is how many surfaces can carry a verdict, and
- # that is `comparable`, not `reached`.
+ # Reached, digested, and NOT a parity signal. Counted separately because what matters to somebody
+ # comparing two arms is how many surfaces can carry a verdict, which is `comparable`, not
+ # `reached`.
"volatile": len([r for r in reached if r.get("volatile")]),
"comparable": len([r for r in reached if not r.get("volatile")]),
"volatile_surfaces": [
diff --git a/tests/studio/studiobench/scene/surfaces.js b/tests/studio/studiobench/scene/surfaces.js
index 2b20bd60dc2..372a9404aed 100644
--- a/tests/studio/studiobench/scene/surfaces.js
+++ b/tests/studio/studiobench/scene/surfaces.js
@@ -1,38 +1,31 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-//
// The selector adapter for Unsloth's surfaces OUTSIDE the chat thread: the other routes, the
// settings dialog's twelve tabs, the sidebar and its menus, the hub, the media pages.
-//
-// Separate from dom.js on purpose. dom.js is the chat thread's adapter and the eighteen film
-// actions all read it; this file is read only by the surface sweep, so a selector added for a
-// route cannot break an action.
-//
+// Separate from dom.js on purpose: dom.js is the chat thread's adapter that all eighteen film
+// actions read, while this is read only by the surface sweep, so a selector added for a route
+// cannot break an action.
// THE TRAP THIS FILE EXISTS TO GUARD. ChatPage, ImagesPage, VideoPage and AudioPage are mounted
-// PERSISTENTLY by the root layout (`shouldMountChat` and friends in app/routes/__root.tsx) so an
-// in-flight generation survives leaving the tab. Off-route they are `class="hidden"` and `inert`,
-// but they are still in the document. So on /hub, `document.querySelector(".aui-thread-root")`
-// still returns the chat thread, and a digest taken with parity.js's default root would digest
-// the HIDDEN CHAT THREAD on every route in the sweep -- forty surfaces reporting one identical
-// digest, and every one of them reading as a pass. Every surface therefore names its own root and
-// the digest is scoped to it.
-//
-// HOW THE SCOPING IS DONE, and why it is not a second digest implementation. Surface digests are
+// PERSISTENTLY by the root layout so an in-flight generation survives leaving the tab; off-route
+// they are `class="hidden"` and `inert` but still in the document. So on /hub,
+// `document.querySelector(".aui-thread-root")` still returns the chat thread, and a digest taken
+// with parity.js's default root would digest the HIDDEN CHAT THREAD on every route: forty
+// surfaces reporting one identical digest, every one reading as a pass. Every surface therefore
+// names its own root.
+// HOW THE SCOPING IS DONE, and why it is not a second digest implementation: surface digests are
// only worth taking if they are comparable with the film's action digests, which means the same
-// normalisation and the same hash -- so this calls `window.__sb.parity.capture()` rather than
-// walking the DOM itself. parity.js reads its root from `window.__sb.dom.threadRoot()`, so the
-// root is moved for the duration of one capture and put back afterwards. `probeScoping()` below
-// checks that the move is actually observed, because if a future parity.js stopped reading
-// threadRoot this would silently go back to digesting the thread on every surface, which is the
-// failure this file was written to prevent.
+// normalisation and hash, so this calls `window.__sb.parity.capture()` rather than walking the
+// DOM. parity.js reads its root from `window.__sb.dom.threadRoot()`, so the root is moved for one
+// capture and put back. `probeScoping()` checks the move is observed, because a future parity.js
+// that stopped reading threadRoot would silently go back to digesting the thread everywhere.
(() => {
if (window.__sb && window.__sb.surfaces) return;
window.__sb = window.__sb || {};
const q = (sel) => {
- // Selector strings come from a registry, and one bad selector must cost that surface a
- // reason rather than cost the whole sweep an exception.
+ // Selector strings come from a registry, and one bad selector must cost that surface a reason
+ // rather than cost the whole sweep an exception.
try {
return document.querySelector(sel);
} catch (err) {
@@ -48,8 +41,8 @@
}
};
- // Rendered, not merely mounted. `hidden`, `display:none` and a zero box all mean the user
- // cannot see it, and the keep-alive route containers are exactly the first case.
+ // Rendered, not merely mounted: `hidden`, `display:none` and a zero box all mean the user cannot
+ // see it, and the keep-alive route containers are exactly the first case.
const isVisible = (el) => {
if (!el) return false;
if (el.hasAttribute && el.hasAttribute("hidden")) return false;
@@ -59,15 +52,12 @@
return style.display !== "none" && style.visibility !== "hidden";
};
- // The ACTIVE route's container, found by the property the root layout actually sets.
- //
- // The layout renders the keep-alive pages and the routed page as siblings, and marks every
- // page that is NOT the current route `inert` (plus `class="hidden"`). So the active container
- // is the one visible sibling of an inert one. Keyed on `inert` rather than on the class string
- // because `hidden` is a Tailwind utility that a restyle can change, while `inert` is what makes
- // the off-route pages unreachable and cannot be dropped without changing behaviour.
- //
- // When nothing is inert, no page has been kept alive yet and the inset itself is the container.
+ // The ACTIVE route's container, found by the property the root layout actually sets. The layout
+ // renders the keep-alive pages and the routed page as siblings and marks every non-current page
+ // `inert` (plus `class="hidden"`), so the active container is the one visible sibling of an inert
+ // one. Keyed on `inert` rather than the class string because `hidden` is a Tailwind utility a
+ // restyle can change, while `inert` is what makes the off-route pages unreachable. When nothing
+ // is inert, no page has been kept alive yet and the inset itself is the container.
const routeContainer = () => {
const inset = q('[data-slot="sidebar-inset"]');
if (!inset) return null;
@@ -75,8 +65,8 @@
if (!inertNode || !inertNode.parentElement) return inset;
const siblings = Array.from(inertNode.parentElement.children);
const active = siblings.filter((el) => !el.hasAttribute("inert") && isVisible(el));
- // Exactly one, or the assumption is wrong and the inset is the honest answer. Silently
- // picking the first of several would scope the digest to an arbitrary part of the page.
+ // Exactly one, or the assumption is wrong and the inset is the honest answer. Silently picking
+ // the first of several would scope the digest to an arbitrary part of the page.
return active.length === 1 ? active[0] : inset;
};
@@ -89,10 +79,8 @@
const resolve = (sel) => (SPECIAL[sel] ? SPECIAL[sel]() : q(sel));
const S = {
- // ── roots ────────────────────────────────────────────────────
- //
- // `candidates` is ordered: the first one that is present AND visible wins. The auth-flow
- // routes render no sidebar wrapper at all, so every list ends at a fallback that exists.
+ // `candidates` is ordered: the first present AND visible wins. The auth-flow routes render no
+ // sidebar wrapper at all, so every list ends at a fallback that exists.
resolveRoot(candidates) {
for (const sel of candidates || []) {
const el = resolve(sel);
@@ -100,14 +88,13 @@
}
for (const sel of candidates || []) {
const el = resolve(sel);
- // Present but not visible is still reported, with the selector, so a surface that
- // rendered into a hidden container is a readable finding rather than a fallback to body.
+ // Present but not visible is still reported, with the selector, so a surface that rendered into a
+ // hidden container is a readable finding rather than a fallback to body.
if (el) return { el, sel, visible: false };
}
return { el: document.body, sel: "body", fallback: true };
},
- // ── the scoped capture ───────────────────────────────────────
capture(candidates) {
const parity = (window.__sb || {}).parity;
if (!parity || typeof parity.capture !== "function") {
@@ -136,9 +123,8 @@
},
// Does parity.capture() actually honour a moved root? Pointed at a detached element with one
- // short text node, an honouring capture returns a signature of a few dozen characters and a
- // non-honouring one returns the whole page. The threshold is two orders of magnitude clear of
- // both, so this cannot answer "yes" for a page that ignored the move.
+ // short text node, an honouring capture returns a few dozen characters and a non-honouring one
+ // the whole page; the threshold is two orders of magnitude clear of both.
probeScoping() {
const dom = (window.__sb || {}).dom;
const parity = (window.__sb || {}).parity;
@@ -167,8 +153,8 @@
reason: "parity.capture() returned no `chars`, so scoping cannot be verified" };
}
if (chars > 500) {
- // The digest is the whole page, not the probe. Every surface digest would then be the
- // same page-wide reading and the sweep would report forty identical passes.
+ // The digest is the whole page, not the probe: every surface digest would then be the same
+ // page-wide reading and the sweep would report forty identical passes.
return { scoped: false, scoping_attempted: true, probe_chars: chars,
reason: "parity.capture() ignored the moved root (" + chars + " chars from a " +
"detached probe element), so surface digests would not be scoped" };
@@ -176,10 +162,8 @@
return { scoped: true, scoping_attempted: true, probe_chars: chars };
},
- // ── settle predicates ────────────────────────────────────────
- //
- // Evaluated by the sweep in a poll loop. Each returns a plain boolean plus the observation it
- // was made from, so a surface that never settled records WHAT it was waiting for.
+ // Evaluated by the sweep in a poll loop. Each returns a plain boolean plus the observation it was
+ // made from, so a surface that never settled records WHAT it was waiting for.
settled(spec) {
if (!spec) return { ok: true, detail: "no settle condition" };
if (spec.visible) {
@@ -212,8 +196,8 @@
return { ok: true, detail: "unrecognised settle spec, treated as satisfied" };
},
- // What the sweep records alongside every surface so a digest can be read against the state it
- // was taken in. A surface whose root holds three elements did not render.
+ // What the sweep records alongside every surface so a digest can be read against the state it was
+ // taken in. A surface whose root holds three elements did not render.
facts(candidates) {
const found = S.resolveRoot(candidates);
return {
@@ -230,8 +214,8 @@
};
},
- // The known state the sweep returns to between surfaces. Reported rather than asserted: the
- // sweep decides what to do about a dirty state, and it needs the observation to decide.
+ // The known state the sweep returns to between surfaces. Reported rather than asserted: the sweep
+ // decides what to do about a dirty state and needs the observation to decide.
isClean() {
return {
clean_attempted: true,
diff --git a/tests/studio/studiobench/scene/surfaces.py b/tests/studio/studiobench/scene/surfaces.py
index 247b963cdea..720a4c3f07d 100644
--- a/tests/studio/studiobench/scene/surfaces.py
+++ b/tests/studio/studiobench/scene/surfaces.py
@@ -40,32 +40,26 @@ import dataclasses
from dataclasses import dataclass
from typing import Any, Optional
-#: The state every surface is reached from and restored to: a fresh, empty chat. Fresh matters.
-#: The keep-alive chat container is inside `@route`'s siblings and inside `@shell`, so a sweep run
-#: against a loaded thread would carry that thread's DOM into the digest of every other surface,
-#: and any thread difference would flip all forty of them at once.
+#: The state every surface is reached from and restored to: a fresh, empty chat. Fresh matters,
+#: since the keep-alive chat container is inside `@route`'s siblings and inside `@shell`, so a
+#: sweep against a loaded thread would carry that thread's DOM into every other surface's
+#: digest and any thread difference would flip all forty at once.
KNOWN_STATE_PATH = "/chat"
-#: Steps are (verb, *args). Interpreted by surface_sweep, which owns the browser; keeping them
+#: Steps are (verb, *args), interpreted by surface_sweep, which owns the browser; keeping them
#: declarative is what lets the unit tests check the registry with no browser present.
-#:
-#: goto <path> navigate, relative to the Unsloth base url
-#: click <selector> a REAL mouse click through the driver. Not element.click():
-#: Radix menus open on pointerdown and a synthetic click misses
-#: them entirely, which reads as a menu that opened instantly
-#: click_if <selector> click only when present. For controls that exist in one of two
-#: states, e.g. the sidebar is already collapsed
-#: hover <selector> park the pointer over an element. REQUIRED before four of the
-#: sidebar's controls: the row actions ship
-#: `opacity-0 pointer-events-none` and only take
-#: `group-hover:pointer-events-auto` when their row is hovered, so
-#: a click on them times out against an element that is present,
-#: sized and reported visible by every check except the one that
-#: matters
-#: press <key> a key on the focused element
-#: fill <selector> <text> set an input's value through the driver
-#: wait <ms> a bounded pause. Only ever ALONGSIDE a settle condition, never
-#: instead of one
+#: goto <path>: navigate, relative to the Unsloth base url.
+#: click <selector>: a REAL mouse click through the driver, not element.click(). Radix menus
+#: open on pointerdown and a synthetic click misses them, reading as a menu that opened
+#: instantly.
+#: click_if <selector>: click only when present, for controls that exist in one of two states
+#: (e.g. an already-collapsed sidebar).
+#: hover <selector>: park the pointer over an element. REQUIRED before four of the sidebar's
+#: controls, whose row actions ship `opacity-0 pointer-events-none` and only take
+#: `group-hover:pointer-events-auto` when hovered, so a click times out against an element
+#: every check except the relevant one reports as visible.
+#: press <key>: a key on the focused element. fill <selector> <text>: set an input's value
+#: through the driver. wait <ms>: a bounded pause, only ever ALONGSIDE a settle condition.
Step = tuple
@@ -81,21 +75,19 @@ class Surface:
root: tuple[str, ...]
settle: Optional[dict] = None
#: Set when the surface's absence is a legitimate property of THIS installation rather than a
- #: broken selector: the Connected tab of the model picker needs an external provider, the hub
- #: catalog needs network. A surface that fails to reach still records a reason either way;
- #: this decides whether the manifest counts it against coverage.
+ #: broken selector: the Connected tab needs an external provider, the hub catalog needs
+ #: network. A surface that fails to reach records a reason either way; this decides whether the
+ #: manifest counts it against coverage.
conditional: Optional[str] = None
- #: Surfaces the film already drives. Swept anyway -- the sweep runs on an EMPTY chat and the
- #: film on a loaded one, so the two digests are of different states -- but flagged so the
- #: coverage figure is not inflated by re-counting what was already covered.
+ #: Surfaces the film already drives. Swept anyway, since the sweep runs on an EMPTY chat and the
+ #: film on a loaded one, but flagged so the coverage figure is not inflated by re-counting.
also_in_film: bool = False
#: The mechanism by which this surface's digest differs between two runs of the SAME build.
#: Measured, not assumed: three consecutive sweeps against one Unsloth agreed on 44 of 53
#: surfaces, and every entry below is the mechanism behind one of the nine that did not.
- #:
- #: This is the same distinction `scripts/sbench_ui_parity.py` already draws for the film's
- #: actions with UNSTABLE_ACTIONS, and for the same reason: a comparison that reports a live
- #: memory gauge as a UI change gets ignored within a day, and then so does the rest of it.
+ #: The same distinction `scripts/sbench_ui_parity.py` draws for the film's actions with
+ #: UNSTABLE_ACTIONS, and for the same reason: a comparison that reports a live memory gauge as
+ #: a UI change gets ignored within a day.
volatile: Optional[str] = None
notes: str = ""
@@ -115,14 +107,14 @@ class Surface:
}
+#: Back to the known state the hard way. The restore for anything that navigated, and the
+#: recovery path when a surface's own restore left the app dirty.
# ── shared fragments ────────────────────────────────────────────────
-#: Back to the known state the hard way. Used as the restore for anything that navigated, and as
-#: the recovery path when a surface's own restore left the app dirty.
HOME: tuple[Step, ...] = (("goto", KNOWN_STATE_PATH), ("wait", 400))
-#: Escape twice, not once. A settings tab that has opened a sub-view (Data -> Archived chats,
-#: Voice -> Dictionary) eats the first Escape closing the sub-view and stays open on the dialog.
+#: Escape twice, not once: a settings tab that opened a sub-view (Data -> Archived chats, Voice
+#: -> Dictionary) eats the first Escape closing the sub-view and stays open on the dialog.
ESCAPE_OUT: tuple[Step, ...] = (
("press", "Escape"),
("wait", 200),
@@ -130,7 +122,7 @@ ESCAPE_OUT: tuple[Step, ...] = (
("wait", 300),
)
-#: The chat composer. Its presence is what "the chat route has rendered" means, everywhere.
+#:The chat composer. Its presence is what 'the chat route has rendered' means, everywhere.
COMPOSER = 'textarea[aria-label="Message input"]'
ROOT_ROUTE = ("@route",)
@@ -138,31 +130,29 @@ ROOT_SHELL = ("@shell",)
ROOT_SIDEBAR = ("@sidebar",)
-#: The empty chat prints a randomly chosen greeting. `pickRandom` in
+#: The empty chat prints a randomly chosen greeting: `pickRandom` in
#: components/assistant-ui/thread.tsx:2019 selects from a time-of-day list, so two loads of one
-#: build render different headings, and every digest whose root contains the welcome screen
-#: differs run to run for that reason alone.
+#: build render different headings and every digest containing the welcome screen differs.
GREETING_IS_RANDOM = (
"the empty chat's greeting is chosen by pickRandom "
"(components/assistant-ui/thread.tsx:2019), so two loads of the same build "
"render different headings"
)
-#: The hub catalogue is fetched from Hugging Face. Download counts, trending order and the arrival
-#: time of the list all move independently of the build under test.
+#: The hub catalogue is fetched from Hugging Face; download counts, trending order and the
+#: list's arrival time all move independently of the build under test.
HUB_IS_REMOTE = (
"the catalogue is fetched from Hugging Face, so its content and its arrival time "
"both move independently of the build under test"
)
-#: MEASURED TO MOVE, MECHANISM NOT ESTABLISHED. Three sweeps of one build disagreed on these, and
-#: a back-to-back pair of visits produced byte-identical markup, so it is not a timer and not a
-#: render race inside the surface. It is state that something earlier in the sweep, or the install,
-#: carries between sweeps -- and which state has not been pinned down.
-#:
-#: Flagged rather than left comparable, and flagged as UNEXPLAINED rather than given a plausible
-#: cause. A wrong mechanism in this field is worse than an admitted gap: it is the sentence a
-#: reader uses to dismiss a real difference.
+#: MEASURED TO MOVE, MECHANISM NOT ESTABLISHED. Three sweeps of one build disagreed on these
+#: while a back-to-back pair of visits produced byte-identical markup, so it is not a timer and
+#: not a render race inside the surface: it is state something earlier in the sweep, or the
+#: install, carries between sweeps.
+#: Flagged as UNEXPLAINED rather than given a plausible cause, because a wrong mechanism here
+#: is worse than an admitted gap: it is the sentence a reader uses to dismiss a real
+#: difference.
UNEXPLAINED_DRIFT = (
"measured to differ between sweeps of the SAME build while a back-to-back "
"pair of visits produced byte-identical markup. The mechanism is NOT "
@@ -194,9 +184,8 @@ def _route(
def _settings_tab(tab_id: str, title: str) -> Surface:
- # `/settings` is not a page. Its route component calls `openDialog()` and immediately
- # redirects to /chat, so the dialog is the surface and the URL never stays on /settings --
- # a settle condition written against the URL would never be satisfied.
+ # `/settings` is not a page: its route component calls `openDialog()` and redirects to /chat,
+ # so the dialog is the surface and a settle written against the URL would never be satisfied.
return Surface(
id = f"settings:{tab_id}",
group = "settings",
@@ -207,10 +196,9 @@ def _settings_tab(tab_id: str, title: str) -> Surface:
("click", f'[data-testid="settings-tab-{tab_id}"]'),
("wait", 400),
),
- # The panel is lazily imported per tab, so "the dialog is open" is not enough: the tab
- # would be recorded as reached while its chunk was still in flight and the digest would
- # be of the Suspense fallback. The tab's own pressed state plus a settled panel body is
- # the observation that the panel arrived.
+ # The panel is lazily imported per tab, so 'the dialog is open' is not enough: the tab would be
+ # recorded as reached with its chunk in flight and the digest would be of the Suspense
+ # fallback. The tab's pressed state plus a settled panel body is what says the panel arrived.
settle = {
"js": f"!!document.querySelector('[data-testid=\"settings-tab-{tab_id}\"]')"
f' && !!document.querySelector(".settings-surface main")'
@@ -225,7 +213,6 @@ def _settings_tab(tab_id: str, title: str) -> Surface:
# ── the registry ────────────────────────────────────────────────────
_SURFACES: list[Surface] = [
- # ── routes ──────────────────────────────────────────────────────
_route(
"route:chat",
"Chat, empty state",
@@ -247,10 +234,10 @@ _SURFACES: list[Surface] = [
"route:train",
"Train (Unsloth)",
"/studio",
- # NOT a URL check alone, and not the nav row either: the nav row is in the sidebar and
- # is there before the page is. StudioPage shows a spinner while the hardware verdict is
- # unmeasured and then redirects to /chat if the host is chat-only, so the wizard's own
- # last section is what says the page finished rendering.
+ # NOT a URL check alone, and not the nav row either: the nav row is in the sidebar and is
+ # there before the page is. StudioPage shows a spinner while the hardware verdict is
+ # unmeasured and then redirects to /chat if the host is chat-only, so the wizard's last
+ # section is what says the page finished rendering.
{
"js": 'location.pathname === "/studio" && '
'(document.body.innerText || "").includes("Run preview")'
@@ -280,8 +267,8 @@ _SURFACES: list[Surface] = [
"/api-monitor",
{"js": 'location.pathname === "/api-monitor"'},
),
- # The 404 is a rendered surface with its own mascot, heading and back-link, and it is exactly
- # the kind of page a bundler or asset-path change breaks without anyone noticing.
+ # The 404 is a rendered surface with its own mascot, heading and back-link, exactly the kind of
+ # page a bundler or asset-path change breaks unnoticed.
_route(
"route:not-found",
"Not found",
@@ -309,15 +296,14 @@ _SURFACES: list[Surface] = [
"timestamps. Two visits differed by 402 lines",
),
_settings_tab("about", "Settings: About"),
- # The dialog's own search index is a distinct rendering of every panel's labels, and it is the
- # one place a settings string appears without its panel being mounted.
+ # The dialog's own search index is a distinct rendering of every panel's labels, and the one
+ # place a settings string appears without its panel being mounted.
Surface(
id = "settings:search",
group = "settings",
title = "Settings: search results",
- # By aria-label, not `input[type="text"]`: the search box ships no `type` attribute at
- # all, so a type selector matches the Embedding model field further down the General
- # panel and fills THAT instead -- a reach that succeeds and lands somewhere else.
+ # By aria-label, not `input[type="text"]`: the search box ships no `type` attribute, so a type
+ # selector matches the Embedding model field further down the General panel and fills THAT.
reach = (
("goto", "/settings"),
("wait", 500),
@@ -331,7 +317,6 @@ _SURFACES: list[Surface] = [
restore = ESCAPE_OUT + HOME,
root = (".settings-surface", '[data-slot="dialog-content"]'),
),
- # ── the sidebar ─────────────────────────────────────────────────
Surface(
id = "sidebar:expanded",
group = "sidebar",
@@ -346,8 +331,8 @@ _SURFACES: list[Surface] = [
group = "sidebar",
title = "Sidebar, collapsed to the rail",
reach = (("click", 'button[aria-label="Close sidebar"]'), ("wait", 500)),
- # The rail is the same container in a different data-state, so presence proves
- # nothing; the reopen control appearing is what says the collapse happened.
+ # The rail is the same container in a different data-state, so presence proves nothing; the
+ # reopen control appearing is what says the collapse happened.
settle = {"visible": 'button[aria-label="Open sidebar"]'},
restore = (("click_if", 'button[aria-label="Open sidebar"]'), ("wait", 400)) + HOME,
root = ROOT_SIDEBAR,
@@ -427,15 +412,14 @@ _SURFACES: list[Surface] = [
restore = (("click_if", 'button[aria-label="Hide workflows"]'), ("wait", 300)),
root = ROOT_SIDEBAR,
),
- # ── the chat composer's own overlays ────────────────────────────
Surface(
id = "chat:model-picker",
group = "chat",
title = "Model picker",
reach = (("click", ".unsloth-model-selector-trigger"), ("wait", 600)),
- # The rows, not the menu. The popover mounts before its list has been fetched, so a
- # settle on the container's presence digests an empty picker roughly half the time --
- # and an empty picker and a broken picker have the same digest.
+ # The rows, not the menu: the popover mounts before its list has been fetched, so a settle on
+ # the container's presence digests an empty picker roughly half the time, and an empty picker
+ # and a broken picker have the same digest.
settle = {"count_at_least": [".unsloth-model-selector-menu button", 2]},
restore = ESCAPE_OUT,
root = (".unsloth-model-selector-menu",),
@@ -472,10 +456,10 @@ _SURFACES: list[Surface] = [
reach = (("click", 'button[aria-label="Open run settings"]'), ("wait", 600)),
settle = {"visible": 'button[aria-label="Close run settings"]'},
restore = (("click_if", 'button[aria-label="Close run settings"]'), ("wait", 400)) + HOME,
- # The panel, not a sheet. Despite the file being called chat-settings-sheet it renders
- # as a docked column inside the chat layout, so it is in NO overlay list and a root of
- # `[role="dialog"]` falls through to body -- which digested 301,402 characters of
- # whole page and called it the run settings panel.
+ # The panel, not a sheet: despite the file being called chat-settings-sheet it renders as a
+ # docked column inside the chat layout, so it is in NO overlay list and a root of
+ # `[role="dialog"]` falls through to body, which digested 301,402 characters of whole page and
+ # called it the run settings panel.
root = ('[data-slot="chat-settings-panel"]',),
),
Surface(
@@ -491,9 +475,9 @@ _SURFACES: list[Surface] = [
id = "chat:system-prompt",
group = "chat",
title = "System prompt editor",
- # Through the run settings panel. The edit control is rendered at x=1458 on a 1440
- # viewport when the panel is closed -- present, sized, and off screen -- so clicking
- # it directly times out on an element every presence check reports as there.
+ # Through the run settings panel: the edit control is rendered at x=1458 on a 1440 viewport
+ # when the panel is closed (present, sized and off screen), so clicking it directly times out
+ # on an element every presence check reports as there.
reach = (
("click", 'button[aria-label="Open run settings"]'),
("wait", 800),
@@ -519,12 +503,11 @@ _SURFACES: list[Surface] = [
group = "chat",
title = "Composer holding text",
reach = (("fill", COMPOSER, "studiobench surface sweep"), ("wait", 400)),
- # The send control is what changes, and it is the same control the film's stop action
- # trips over: with text in the box the Stop button is replaced by Queue.
- # ONE EXPRESSION. The settle evaluator wraps the string in `return (...)`, so a
- # statement -- a `const` declaration, say -- raises SyntaxError at evaluation time,
- # and the surface is recorded as never settling for a reason that is about this file
- # rather than about the app.
+ # The send control is what changes, and it is the control the film's stop action trips over:
+ # with text in the box the Stop button is replaced by Queue.
+ # ONE EXPRESSION: the settle evaluator wraps the string in `return (...)`, so a statement
+ # raises SyntaxError at evaluation time and the surface is recorded as never settling for a
+ # reason about this file rather than about the app.
settle = {
"js": "(document.querySelector("
"'textarea[aria-label=\"Message input\"]') || {}).value ? true : false"
@@ -534,7 +517,6 @@ _SURFACES: list[Surface] = [
also_in_film = True,
volatile = GREETING_IS_RANDOM,
),
- # ── the hub ─────────────────────────────────────────────────────
Surface(
id = "hub:datasets",
group = "hub",
@@ -545,9 +527,9 @@ _SURFACES: list[Surface] = [
("click", 'button[aria-label="Datasets"]'),
("wait", 800),
),
- # The URL flips the instant the tab is pressed, well before the list arrives. Settling
- # on the URL alone digested an empty table on one visit and a populated one on the
- # next, a 424-line difference that had nothing to do with any build.
+ # The URL flips the instant the tab is pressed, well before the list arrives. Settling on the
+ # URL alone digested an empty table on one visit and a populated one on the next, a 424-line
+ # difference unrelated to any build.
settle = {
"js": 'location.search.includes("kind=datasets") && '
'(document.body.innerText || "").includes("Downloads")'
@@ -557,9 +539,9 @@ _SURFACES: list[Surface] = [
conditional = "needs the catalogue, which needs network",
volatile = HUB_IS_REMOTE,
),
- # The hub's two filter popovers are LISTBOXES, not menus. `[role="menuitem"]` counted zero on
- # a popover that was open and visible on screen, and zero from a wrong selector is the failure
- # mode this tool exists to refuse -- so both the settle and the root read `option`/`listbox`.
+ # The hub's two filter popovers are LISTBOXES, not menus: `[role="menuitem"]` counted zero on a
+ # popover open and visible on screen, and zero from a wrong selector is the failure mode this
+ # tool exists to refuse, so both the settle and the root read `option`/`listbox`.
Surface(
id = "hub:format-filter",
group = "hub",
@@ -623,7 +605,6 @@ _SURFACES: list[Surface] = [
conditional = "needs the catalogue, which needs network",
volatile = HUB_IS_REMOTE,
),
- # ── the media pages ─────────────────────────────────────────────
Surface(
id = "images:presets",
group = "media",
@@ -673,7 +654,6 @@ _SURFACES: list[Surface] = [
root = ('[role="menu"]', "[data-radix-popper-content-wrapper]"),
conditional = "a host without video support renders the gate instead of the page",
),
- # ── train ───────────────────────────────────────────────────────
Surface(
id = "train:model-picker",
group = "train",
@@ -738,9 +718,9 @@ _SURFACES: list[Surface] = [
("click", 'button[aria-label="Image training"]'),
("wait", 1200),
),
- # It NAVIGATES. The control sits on the Train page but lands on /images with the
- # training panel selected, so a settle written against /studio waits out its timeout
- # on a page that arrived correctly.
+ # It NAVIGATES: the control sits on the Train page but lands on /images with the training panel
+ # selected, so a settle written against /studio waits out its timeout on a page that arrived
+ # correctly.
settle = {
"js": 'location.pathname === "/images" && '
'(document.body.innerText || "").includes("Train a LoRA")'
@@ -750,7 +730,6 @@ _SURFACES: list[Surface] = [
conditional = "a chat-only host redirects /studio to /chat",
volatile = UNEXPLAINED_DRIFT,
),
- # ── the api monitor ─────────────────────────────────────────────
Surface(
id = "api:status-filter",
group = "api",
@@ -772,10 +751,9 @@ _SURFACES: list[Surface] = [
#: Surfaces that exist and are NOT swept, each with the mechanism that puts them out of reach.
-#:
-#: This list is the honest half of the coverage number. A manifest that reports "38 of 38 reached"
-#: while eleven surfaces were never registered is a worse artefact than one that reports 38 of 38
-#: and names the eleven, because the first invites the reader to believe the app is fully covered.
+#: This list is the honest half of the coverage number: a manifest reporting '38 of 38 reached'
+#: while eleven surfaces were never registered invites the reader to believe the app is fully
+#: covered.
KNOWN_UNCOVERED: tuple[dict, ...] = (
{
"id": "route:login",
@@ -902,8 +880,8 @@ def validate_registry(entries: Optional[list[Surface]] = None) -> None:
raise RegistryError(f"surface {s.id!r} is missing an id, group or title")
if not s.root:
raise RegistryError(f"surface {s.id!r} has no digest root")
- # `reach` may be empty only for a surface that IS the known state; `restore` may be empty
- # only when reaching it changed nothing. Both are declared, never inferred.
+ # `reach` may be empty only for a surface that IS the known state; `restore` may be empty only
+ # when reaching it changed nothing. Both are declared, never inferred.
for name, steps in (("reach", s.reach), ("restore", s.restore)):
for step in steps:
if not step:
diff --git a/tests/studio/studiobench/scoring/ab.py b/tests/studio/studiobench/scoring/ab.py
index 5a997301c0d..972da921bff 100644
--- a/tests/studio/studiobench/scoring/ab.py
+++ b/tests/studio/studiobench/scoring/ab.py
@@ -73,8 +73,8 @@ class RunIdentity:
}
-#: The fields that must match. `session_id` is separate because its failure mode is different:
-#: mismatched ids mean different meanings, a mismatched session means an 8% drift term.
+#: The fields that must match. `session_id` is separate because its failure mode differs: mismatched
+#: ids mean different meanings, a mismatched session means an 8% drift term.
COMPARABILITY_FIELDS = ("bench_version", "corpus_hash", "rung_ladder_id", "weights_id")
@@ -184,13 +184,13 @@ class MetricComparison:
ci_high: float | None
verdict: str = "no_reading"
beyond_noise: bool = False
- #: The 95% CI of the paired geometric mean contains 1.0, so the data cannot rule out "no
- #: effect" however far the point estimate landed from the noise floor. Carried so the table
- #: can say the size is unresolved rather than print a direction as a finding.
+ #: The 95% CI of the paired geometric mean contains 1.0, so the data cannot rule out "no effect"
+ #: however far the point estimate landed from the noise floor. Carried so the table can say the
+ #: size is unresolved rather than print a direction as a finding.
ci_spans_no_effect: bool = False
- #: At least one contributing pair had a sub-floor arm, so the ratio is a BOUND: the true
- #: magnitude is larger. Carried so the table can say so rather than let a number derived from
- #: an instrument floor be quoted as a measurement.
+ #: At least one contributing pair had a sub-floor arm, so the ratio is a BOUND: the true magnitude
+ #: is larger. Carried so a number derived from an instrument floor is not quoted as a
+ #: measurement.
bounded: bool = False
@property
@@ -304,18 +304,16 @@ class AbResult:
if self.regressions:
return "FAIL"
if self.headline_ratio is None:
- # UNRESOLVED IS NOT UNMEASURED. `compare` keeps unresolved metrics out of the headline
- # entirely, so a run whose every moving metric straddled 1.0 arrives here with no
- # ratio at all. That is not the same as an empty payload: the data exists and says
- # nothing, which is INCONCLUSIVE, while NO READING means there was nothing to read.
+ # UNRESOLVED IS NOT UNMEASURED. `compare` keeps unresolved metrics out of the headline entirely, so
+ # a run whose every moving metric straddled 1.0 arrives with no ratio at all: the data exists and
+ # says nothing, which is INCONCLUSIVE, while NO READING means there was nothing to read.
if any(m.unresolved for m in self.metrics):
return "INCONCLUSIVE"
return "NO READING"
- # NO DIFFERENCE IS A CLAIM, AND A STRONGER ONE THAN THIS DATA SUPPORTS. Excluding an
- # unresolved mover from the headline can leave only flat metrics behind, putting the
- # aggregate back inside the noise floor. Reporting "no difference" there would assert the
- # change did nothing, when in fact one metric moved and could not resolve its own sign.
- # Only say that once some metric actually resolved a direction.
+ # NO DIFFERENCE IS A CLAIM, AND A STRONGER ONE THAN THIS DATA SUPPORTS. Excluding an unresolved
+ # mover can leave only flat metrics behind, putting the aggregate back inside the noise floor;
+ # reporting 'no difference' there would assert the change did nothing when one metric moved and
+ # could not resolve its own sign.
if any(m.unresolved for m in self.metrics) and not any(
m.resolves_direction for m in self.metrics
):
@@ -404,13 +402,11 @@ def compare(
if not comparison.beyond_noise:
comparison.verdict = "within noise"
elif worse:
- # THE ASYMMETRY IS THE POINT, ON BOTH THE LABEL AND THE HEADLINE. Withholding an
- # unresolved win costs a headline; withholding an unresolved loss ships the
- # regression. So the worse side keeps counting and keeps contributing to the
- # aggregate, where it can only drag the number toward worse, which is the
- # conservative direction. A single bounded pair over a zero base is the case that
- # proves it: `_divisible` exists because that regression once vanished from the
- # table, the headline and this list at once, and it must not vanish again.
+ # THE ASYMMETRY IS THE POINT, ON BOTH THE LABEL AND THE HEADLINE. Withholding an unresolved win
+ # costs a headline; withholding an unresolved loss ships the regression. So the worse side keeps
+ # counting and keeps contributing to the aggregate, where it can only drag the number toward
+ # worse. `_divisible` exists because a single bounded pair over a zero base once vanished from the
+ # table, the headline and this list at once.
comparison.verdict = (
"regressed (unresolved)" if comparison.ci_spans_no_effect else "regressed"
)
@@ -425,14 +421,12 @@ def compare(
)
else:
comparison.verdict = "inconclusive" if comparison.unresolved else "improved"
- # AN UNRESOLVED METRIC LENDS ITS MAGNITUDE TO NOTHING. The headline is a weighted
- # geometric mean of point estimates and carries no interval of its own, so a metric
- # that moved a long way but could not resolve its own sign would otherwise supply most
- # of the number that gets quoted. keystroke_p95_ms at 0.2, 0.2, 1.5, 1.5 (geomean
- # 0.548, CI 0.200-1.500) beside a resolved menu_open_ms of 0.900 produced a headline
- # of 0.631 and the word IMPROVED: a quoted 36.9% win almost entirely made of data the
- # table itself labels inconclusive. Metrics still within noise keep contributing --
- # they pull the headline toward 1.0, which is the conservative direction.
+ # AN UNRESOLVED METRIC LENDS ITS MAGNITUDE TO NOTHING. The headline is a weighted geometric mean
+ # of point estimates carrying no interval of its own, so a metric that moved a long way but could
+ # not resolve its sign would supply most of the quoted number: keystroke_p95_ms at 0.2, 0.2, 1.5,
+ # 1.5 beside a resolved menu_open_ms of 0.900 produced a headline of 0.631 and the word IMPROVED.
+ # That metric's own mean was 0.548, CI 0.200-1.500.
+ # Metrics still within noise keep contributing, pulling the headline toward 1.0.
if not comparison.withheld:
weight = anchor.weight if anchor else 1.0
weighted_logs.append((weight, math.log(geo)))
@@ -443,8 +437,8 @@ def compare(
result.headline_ratio = math.exp(sum(w * lg for w, lg in weighted_logs) / total_weight)
if is_null_control:
- # A null control that shows a difference is the harness moving, not the build. Whatever
- # it reports, no comparison run on the same machine at the same time can be believed.
+ # A null control that shows a difference is the harness moving, not the build. Whatever it
+ # reports, no comparison run on the same machine at the same time can be believed.
offenders = [
f"{m.metric_key}: {abs((m.ratio_geomean - 1.0) * 100):.1f}%"
for m in result.metrics
diff --git a/tests/studio/studiobench/scoring/anchors.py b/tests/studio/studiobench/scoring/anchors.py
index 5d1a1641052..54e7786f7e2 100644
--- a/tests/studio/studiobench/scoring/anchors.py
+++ b/tests/studio/studiobench/scoring/anchors.py
@@ -123,23 +123,23 @@ METRIC_ANCHORS: tuple[MetricAnchor, ...] = (
METRIC_BY_KEY: Mapping[str, MetricAnchor] = {m.key: m for m in METRIC_ANCHORS}
-#: The rung ladder, in tokens of thread content. Log-spaced on purpose: aggregation integrates
-#: over log(tokens), so evenly spaced rungs on the log axis give evenly spaced evidence.
+#: The rung ladder, in tokens of thread content. Log-spaced on purpose: aggregation integrates over
+#: log(tokens), so evenly spaced rungs on the log axis give evenly spaced evidence.
RUNG_TOKENS: tuple[int, ...] = (1_000, 10_000, 100_000, 500_000, 1_000_000)
-#: A rung counts as usable when its score clears this AND no single metric is catastrophic.
-#: The onset rung -- the largest usable rung -- is the human headline, because it survives being
-#: carried to a different machine in a way a 0-100 score does not.
+#: A rung counts as usable when its score clears this AND no single metric is catastrophic. The
+#: onset rung, the largest usable rung, is the human headline, because it survives being carried to
+#: a different machine in a way a 0-100 score does not.
ONSET_SCORE_THRESHOLD = 50.0
ONSET_METRIC_FLOOR = 10.0
#: A rung whose measured metrics do not cover at least this fraction of the declared weight is
-#: INCOMPLETE, which scores 0. It does not drop out: a build that crashes at 500K must not
-#: outscore one that limps through it, and dropping the rung is exactly how that happens.
+#: INCOMPLETE, which scores 0. It does not drop out: a build that crashes at 500K must not outscore
+#: one that limps through it.
MIN_WEIGHT_COVERAGE = 0.60
-#: Ratios inside this band are indistinguishable from run-to-run noise. Anything an A/B claims
-#: must clear it, and the NULL calibration arm must land inside it.
+#: Ratios inside this band are indistinguishable from run-to-run noise. Anything an A/B claims must
+#: clear it, and the NULL calibration arm must land inside it.
DEFAULT_NOISE_FLOOR_PCT = 5.0
diff --git a/tests/studio/studiobench/scoring/frames.py b/tests/studio/studiobench/scoring/frames.py
index 1747b015f08..2b5ce56cab4 100644
--- a/tests/studio/studiobench/scoring/frames.py
+++ b/tests/studio/studiobench/scoring/frames.py
@@ -42,21 +42,21 @@ from typing import Any, Sequence
from .schema import Measure
-#: Perceptual constant. A frame longer than this reads as an interruption regardless of display.
+#:Perceptual constant. A frame longer than this reads as an interruption regardless of display.
JANK_FRAME_MS = 100.0
-#: Detection floors. Below these the recorder cannot distinguish a reading from its own noise.
+#:Detection floors. Below these the recorder cannot distinguish a reading from its own noise.
FLOOR_FRAME_MS = 0.5
FLOOR_TIME_IN_JANK_PCT = 0.1
FLOOR_JANK_INDEX = 0.05
-#: A refresh interval outside this range is not a display, it is a broken recorder.
+#:A refresh interval outside this range is not a display, it is a broken recorder.
REFRESH_MIN_MS = 3.0
REFRESH_MAX_MS = 60.0
REFRESH_FALLBACK_MS = 1000.0 / 60.0
-#: Log-spaced histogram edges, in ms. Chosen so that the interesting region (one frame, a few
-#: frames, a visible hitch, a freeze) each gets its own bucket rather than all landing in ">33".
+#: Log-spaced histogram edges, in ms. Chosen so the interesting region (one frame, a few frames, a
+#: visible hitch, a freeze) each gets its own bucket rather than all landing in ">33".
HISTOGRAM_EDGES_MS: tuple[float, ...] = (
0.0,
4.0,
@@ -179,10 +179,10 @@ def compute_frame_stats(
budget_ms, refresh_source = measure_refresh_interval_ms(usable)
if not usable:
- # The recorder was installed and produced nothing. That is the rAF-unscheduled trap:
- # a compositor that decided nothing is visible stops SCHEDULING callbacks, and the naive
- # reading is "zero dropped frames". Every metric here reads as a failed measurement so
- # the tri-clock gate upstream is what decides whether the window survives.
+ # The recorder was installed and produced nothing: the rAF-unscheduled trap, where a compositor
+ # that decided nothing is visible stops SCHEDULING callbacks and the naive reading is "zero dropped
+ # frames". Every metric here reads as a failed measurement so the tri-clock gate upstream decides
+ # whether the window survives.
reason = "frame recorder produced no frames (rAF may be unscheduled)"
return FrameStats(
frames_total = 0,
diff --git a/tests/studio/studiobench/scoring/from_payload.py b/tests/studio/studiobench/scoring/from_payload.py
index e0ff367cc11..783cabe117b 100644
--- a/tests/studio/studiobench/scoring/from_payload.py
+++ b/tests/studio/studiobench/scoring/from_payload.py
@@ -32,21 +32,21 @@ from .anchors import METRIC_BY_KEY
from .frames import compute_frame_stats
from .schema import Measure
-# ── where each scored metric actually lives in the payload ──────────────────────────────────
# (action name, timing key). Anything not listed here comes from the window frame recorder.
+# ── where each scored metric actually lives in the payload ──────────────────────────────────
+
ACTION_SOURCES: Mapping[str, tuple[str, str]] = {
"keystroke_p95_ms": ("keystroke", "p95_ms"),
"menu_open_ms": ("message_menu", "open_ms"),
"scroll_settle_ms": ("scroll_after", "gesture_ms"),
}
-# The one mapping that is NOT an identity of meaning, so it is stated rather than buried. The
-# anchor was written for settle time (how long after the gesture the thread stops moving); the
-# scene records the gesture itself and the per-step cost, and never measures settle separately.
-# `gesture_ms` is the closest recorded quantity and is on the same scale, but a reader comparing
-# this column against the anchor's 100 ms / 3000 ms rationale is comparing against a slightly
-# different thing, and should be told so in the cell rather than in a changelog.
+# The one mapping that is NOT an identity of meaning. The anchor was written for settle time (how
+# long after the gesture the thread stops moving); the scene records the gesture and the
+# per-step cost and never measures settle. `gesture_ms` is the closest recorded quantity on the
+# same scale, but a reader comparing this column against the anchor's 100 ms / 3000 ms rationale
+# is comparing a slightly different thing.
SCROLL_SETTLE_NOTE = (
"recorded as scroll_after.gesture_ms; the scene does not measure settle separately, so this "
"is gesture duration, not post-gesture settle"
@@ -54,21 +54,18 @@ SCROLL_SETTLE_NOTE = (
FRAME_METRICS: tuple[str, ...] = ("time_in_jank_pct", "jank_index", "max_frame_ms")
+# The streaming phase, separated out and normalised per character.
+# The three metrics above are not blind to the stream (`_frame_measures` pools every non-`idle`
+# window) but they cannot separate it: one 57.3 s film collapses eighteen action windows and the
+# streaming stretch into one number the action windows dominate. On a measured 100K null control
+# `reasoning_toggle` alone contributed 2,865 ms blocked at 99.3% busy with a 1,866 ms worst frame
+# while the streaming stretch beside it ran at 3.6% busy with a 100 ms worst frame.
+# THE WINDOW KIND CANNOT BE USED TO SEPARATE THEM, and the name is what misleads:
+# `SceneRunner._gap_window` opens every inter-slot gap as `kind = "stream"`, so eighteen windows
+# are named `stream:gapN` and only the first four contain streaming, while `stream:drain` is
+# opened after the film (7 ms on that cell). The phase is taken from the `stream_cost`
+# instrument, which detects it from SSE traffic, never from the label.
# ── the streaming phase, separated out and normalised per character ─────────────────────────
-#
-# The three metrics above are not blind to the stream. `_frame_measures` pools every window whose
-# kind is not `idle`, and the streaming windows are in that pool. What they cannot do is separate
-# it: one 57.3 s film collapses eighteen action windows and the streaming stretch into a single
-# number, and the action windows dominate. On a measured 100K null control, `reasoning_toggle`
-# alone contributed 2,865 ms of blocked time at 99.3% busy with a 1,866 ms worst frame while the
-# streaming stretch beside it ran at 3.6% busy with a 100 ms worst frame.
-#
-# THE WINDOW KIND CANNOT BE USED TO SEPARATE THEM, and the name is what misleads.
-# `SceneRunner._gap_window` opens every inter-slot gap as `kind = "stream"`, so on the standard
-# film eighteen windows are named `stream:gapN` and only the first four contain streaming;
-# `stream:drain` is opened after the film has finished and measured 7 ms on that same cell. The
-# phase is therefore taken from the `stream_cost` instrument, which detects it from the SSE
-# traffic, and never from the label.
STREAM_METRICS: tuple[str, ...] = (
"stream_delta_cost_ms_per_kchar",
"stream_cost_ms_per_kchar",
@@ -78,39 +75,33 @@ STREAM_METRICS: tuple[str, ...] = (
"stream_max_frame_ms",
)
-# A window has to carry at least this much streamed text before its cost is divided by it. Below
-# it the denominator is small enough that the ratio is dominated by whatever else shared the
-# window: on a measured cell an `action:send_turn` window grew the reply by 13 characters while
-# accumulating 475 ms of blocked time, which as a rate is 36,500 ms per thousand characters and is
-# a statement about opening a menu, not about streaming.
+# A window must carry at least this much streamed text before its cost is divided by it. Below it
+# the ratio is dominated by whatever else shared the window: a measured `action:send_turn` window
+# grew the reply by 13 characters while accumulating 475 ms blocked, which as a rate is 36,500 ms
+# per thousand characters and is a statement about opening a menu.
MIN_STREAM_CHARS_PER_WINDOW = 100
# More timer ticks than the clamp says are possible means the clamp is wrong, and every blocked
# figure derived from it is a subtraction against the wrong floor. `clocks_agree` would be the
-# gate, but on a headless engine it is null by design (see instruments/pagejs.py: rAF has no vsync
-# to be checked against and the screencast is rate-limited), so `timer_clock_ratio` is the sound
-# availability signal and this is the bound it has to respect.
+# gate but is null by design on a headless engine (see instruments/pagejs.py), so
+# `timer_clock_ratio` is the sound availability signal and this is the bound it must respect.
MAX_TIMER_CLOCK_RATIO = 1.2
-# Windows that are not part of the film, and whose frames therefore say nothing about the build.
-#
-# `idle` is deliberately quiet: pooling it into the frame metrics would dilute every jank share
-# with idle time and make a bad build look average.
-#
-# `setup` is the opposite and is excluded for the opposite reason. The only one is the composer
-# click that starts the film, and most of it is Playwright's injected actionability script --
-# selector resolution, visibility, stability and the `elementsFromPoint` hit test -- running on
-# the page's own main thread, where it blocks frames indistinguishably from app work. At 500K
-# that window alone runs about 11 s against a `max_frame_ms` anchor whose worst case is 2,000 ms,
-# so pooling it would peg all three frame metrics on every run, including runs that never asked
-# for the click probe.
+# Windows that are not part of the film, whose frames say nothing about the build.
+# `idle` is deliberately quiet: pooling it would dilute every jank share with idle time and make
+# a bad build look average.
+# `setup` is excluded for the opposite reason. The only one is the composer click that starts the
+# film, mostly Playwright's injected actionability script running on the page's own main thread,
+# where it blocks frames indistinguishably from app work. At 500K that window alone runs about
+# 11 s against a `max_frame_ms` anchor whose worst case is 2,000 ms, so pooling it would peg all
+# three frame metrics on every run.
UNSCORED_WINDOW_KINDS: frozenset[str] = frozenset({"idle", "setup"})
-# The window kinds in which NO SCRIPTED ACTION IS RUNNING. `gap` is the scheduler's inter-slot
-# wait; `stream` is `stream:drain`, the window the session layer opens after the film to wait the
-# reply out. Both are quiet by construction, which is the property `_unaided` needs -- see there
-# for why the streaming numbers are taken only from these, and for what the `stream` half is worth.
-# `action` is excluded, and `idle` never reaches here (UNSCORED_WINDOW_KINDS strips it first).
+# The window kinds in which NO SCRIPTED ACTION IS RUNNING: `gap` is the scheduler's inter-slot
+# wait and `stream` is `stream:drain`, opened after the film to wait the reply out. Both are
+# quiet by construction, which is what `_unaided` needs. `action` is excluded, and `idle` never
+# reaches here.
+# `UNSCORED_WINDOW_KINDS` strips `idle` first.
UNAIDED_WINDOW_KINDS: frozenset[str] = frozenset({"gap", "stream"})
@@ -200,11 +191,9 @@ def _action_measure(metric_key: str, actions: Mapping[str, Mapping[str, Any]]) -
# Attempted-and-did-not-run, which is a fact about the run, not an absent instrument.
return Measure.failed(unit, f"{action_name} did not run: {reason}")
if row.get("expect_ok") is False:
- # RAN IS NOT DID WHAT IT CLAIMED. An action carries its own assertion -- the composer's
- # value grew by the characters that were typed, the menu that was opened has items, the
- # panes that were expanded are open -- and when that assertion fails the timing describes
- # something other than the action. `report/payload.py` already lists these cells under
- # EXCLUDED CELLS with "its timings exist and must not be quoted"; scoring them anyway let
+ # RAN IS NOT DID WHAT IT CLAIMED. An action carries its own assertion, and when it fails the
+ # timing describes something other than the action. `report/payload.py` already lists these cells
+ # under EXCLUDED CELLS with 'its timings exist and must not be quoted'; scoring them anyway let
# the same number be excluded in the report and load-bearing in the headline.
reason = row.get("reason") or "no reason recorded"
return Measure.failed(unit, f"{action_name} ran but its own assertion failed: {reason}")
@@ -257,17 +246,12 @@ def _frame_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]:
if frameless:
# ONE WINDOW THAT SAW NOTHING POISONS THE POOL, IT DOES NOT DROP OUT OF IT.
- #
- # A window whose recorder was installed and exported no deltas at all is the rAF-
- # unscheduled trap, and `compute_frame_stats` already refuses to score it: a single such
- # window reads `Measure.failed` on every metric with `no_frames_recorded` set, never zero
- # jank. Pooled, the same window was skipped by the `continue` above -- it contributed no
- # deltas, no wall time and (its `max_frame_ms` being null) no worst frame -- so the
- # REMAINING windows answered for the whole cell and a complete freeze during one action
- # came back as clean numbers: a 4 s frozen window beside a smooth one scored 0.0% time in
- # jank and a 40 ms worst frame, byte-identical to the cell without the freeze in it.
- # An unmeasured window is not an absent one, so the cell's frame metrics fail here for
- # the same reason and in the same shape as the single-window path.
+ # A window whose recorder was installed and exported no deltas is the rAF-unscheduled trap, and
+ # `compute_frame_stats` already refuses to score it. Pooled, such a window was skipped by the
+ # `continue` above (no deltas, no wall time, no worst frame) so the REMAINING windows answered
+ # for the whole cell and a complete freeze came back clean: a 4 s frozen window beside a smooth
+ # one scored 0.0% time in jank and a 40 ms worst frame, byte-identical to the cell without the
+ # freeze. An unmeasured window is not an absent one.
reason = (
f"{frameless} window(s) recorded no frames at all (rAF may be unscheduled), so the "
"pooled frame metrics would describe only the windows that were measured"
@@ -328,17 +312,13 @@ def _stream_windows(windows: Sequence[Mapping[str, Any]]) -> tuple[list[Mapping[
str(sc.get("reply_chars_delta_reason") or "the reply's growth was not measurable")
)
continue
- # THE INSTRUMENT'S OWN VERDICT ON ITS DENOMINATOR, and it is consulted here because this is
- # the only place that can act on it. `instruments/streamcost.py` marks a window unscoreable
- # when an SSE frame failed to parse inside it or an unterminated frame was still buffered
- # at its close -- and per the HTML standard an event is dispatched only at the blank line
- # that terminates it, so those characters have not been counted and cannot be recovered.
- # The delta is therefore short by an unknown amount, and every cost-per-character divided
- # by it comes out inflated. Publishing the flag and then summing the delta anyway left the
- # official metrics derived from a denominator the instrument had already disowned.
- #
- # `is False` and not falsiness: a payload recorded before the flag existed carries no key
- # at all, and those windows are admitted exactly as they were rather than voided wholesale.
+ # THE INSTRUMENT'S OWN VERDICT ON ITS DENOMINATOR, consulted here because this is the only place
+ # that can act on it. `instruments/streamcost.py` marks a window unscoreable when an SSE frame
+ # failed to parse or an unterminated frame was still buffered at its close, and per the HTML
+ # standard an event is dispatched only at the blank line that terminates it, so those characters
+ # cannot be recovered. Every cost-per-character divided by that short delta comes out inflated.
+ # `is False` and not falsiness: a payload recorded before the flag existed carries no key at all,
+ # and those windows are admitted as they were rather than voided wholesale.
if sc.get("reply_chars_scoreable") is False:
reject(
str(
@@ -449,24 +429,18 @@ def _stream_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]
for k, u in unit_by_key.items()
}
- # A CELL WHOSE RECORDER DIED IS A TRUNCATED CELL, not a short one. `rejected` above is read
- # only when NOTHING qualified, so once one window has qualified every later rejection is
- # discarded -- including the one that says the page went away. The streaming metrics are
- # integrals divided by the characters that were streamed, and both halves are then missing the
- # same unmeasured stretch, so the result is not a wide error bar, it is a number computed over
- # whatever ran before the crash and reported as if it described the cell.
- #
+ # A CELL WHOSE RECORDER DIED IS A TRUNCATED CELL, not a short one. `rejected` above is read only
+ # when NOTHING qualified, so once one window has qualified every later rejection is discarded,
+ # including the one saying the page went away. The streaming metrics are integrals divided by
+ # streamed characters, and both halves miss the same unmeasured stretch, so the result is not a
+ # wide error bar but a number computed over whatever ran before the crash.
# Measured on the payload corpus: 138 unaided windows record `unavailable` while a qualifying
- # window precedes them, carrying `TargetClosedError: Page.evaluate: Target page, context or
- # browser has been closed` and `Error: Page.evaluate: Target crashed`. They are all AFTER the
- # last qualifying window, which is not evidence that they are ordinary end-of-stream windows:
- # a crash is trailing by construction, because nothing can qualify once the page is gone. They
- # are also long, a median of 11.0 s and up to 36.0 s, so what went unmeasured is a real
- # fraction of the reply and not a rounding edge.
- #
- # `unavailable` was consumed nowhere in this module before this, so the crash signal the
- # instrument already emits was being dropped in full. Poisoning here costs 12 cells of 1,461
- # that currently publish, and every one of the 12 is a genuine page crash.
+ # window precedes them, carrying `TargetClosedError` / `Page.evaluate: Target crashed`. They are
+ # all AFTER the last qualifying window, which is not evidence they are ordinary end-of-stream
+ # windows (a crash is trailing by construction) and they are long, median 11.0 s.
+ # `unavailable` was consumed nowhere in this module before, so the crash signal was dropped in
+ # full. Poisoning here costs 12 cells of 1,461 that currently publish, and every one is a
+ # genuine page crash.
crashed = sorted(
{
str(sc.get("unavailable"))
@@ -485,7 +459,7 @@ def _stream_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]
return {k: Measure.failed(u, reason) for k, u in unit_by_key.items()}
# Every streaming quantity comes from the UNAIDED windows. See _unaided for why the targeted
- # numerator is not exempt from that, which is the one thing here that measurement overturned.
+ # numerator is not exempt, which is the one thing here that measurement overturned.
unaided = _unaided(picked)
chars = 0
delta_task_ms = 0.0
@@ -568,18 +542,13 @@ def _stream_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]
)
if frameless:
# THE SAME RULE AS `_frame_measures`, ON THE SAME SHAPE OF WINDOW, AND FOR THE SAME REASON.
- #
- # `instruments/frames.js` emits `frames_attempted: true` with `frame_gaps_ms: []` and a
- # null `max_frame_ms` whenever the rAF loop was never scheduled in the window, and rAF
- # going unscheduled is a rendering fact, not a quiet one: on a headless engine the loop
- # runs off the rendering pipeline, so a renderer that stalls stops delivering callbacks
- # while SSE keeps arriving. Skipped rather than refused, such a window contributes no
- # deltas, no wall time and no worst frame, so the REMAINING unaided windows answer for the
- # whole streaming stretch: one frozen window beside one smooth one reported a 16.7 ms worst
- # frame and 0.0% time in jank, byte-identical to the cell without the freeze in it.
- # An unmeasured window is not an absent one. Only the three FRAME metrics are poisoned --
- # the cost, busy and character figures come from `stream_cost`, which measured this window
- # perfectly well.
+ # `instruments/frames.js` emits `frames_attempted: true` with an empty `frame_gaps_ms` and a null
+ # `max_frame_ms` whenever the rAF loop was never scheduled, and that is a rendering fact: on a
+ # headless engine the loop runs off the rendering pipeline, so a stalled renderer stops
+ # delivering callbacks while SSE keeps arriving. Skipped rather than refused, one frozen window
+ # beside one smooth one reported a 16.7 ms worst frame and 0.0% time in jank. Only the three
+ # FRAME metrics are poisoned; cost, busy and characters come from `stream_cost`, which measured
+ # the window fine.
reason = (
f"{frameless} unaided streaming window(s) recorded no frames at all (rAF may be "
"unscheduled), so the pooled streaming frame metrics would describe only the windows "
@@ -635,13 +604,11 @@ def measures_from_records(
and str(w.get("kind") or "") not in UNSCORED_WINDOW_KINDS
]
frames = _frame_measures(windows)
- # The streaming phase is NOT in METRIC_BY_KEY, and that is deliberate rather than an
- # omission. The anchor table is hashed into `weights_id`, and a report that compares two
- # runs with different `weights_id` values is refused; adding a seventh weighted metric
- # would change every existing run's composite score and make it incomparable with every
- # run this tool has already taken. So these are scored through the per-metric floor table,
- # which judges a metric on its own floor, and the composite score is left alone. Giving
- # them anchors and a weight is a separate decision that should be argued in anchors.py.
+ # The streaming phase is NOT in METRIC_BY_KEY, deliberately. The anchor table is hashed into
+ # `weights_id` and a report comparing two runs with different values is refused, so a seventh
+ # weighted metric would make every existing run incomparable. These are scored through the
+ # per-metric floor table instead and the composite score is left alone; giving them anchors and
+ # a weight is a separate decision for anchors.py.
stream = _stream_measures(windows)
readings: dict[str, Measure] = {}
@@ -657,8 +624,8 @@ def measures_from_records(
METRIC_BY_KEY[key].unit, f"no source is wired for {key}"
)
- # Repetitions of the same rung: keep the first and let the caller ask for reps
- # explicitly. Silently averaging reps here would hide a bimodal rung.
+ # Repetitions of the same rung: keep the first and let the caller ask for reps explicitly.
+ # Silently averaging reps here would hide a bimodal rung.
by_rung.setdefault(rung, readings)
return by_rung
diff --git a/tests/studio/studiobench/scoring/payload_rules.py b/tests/studio/studiobench/scoring/payload_rules.py
index fd2e7dface7..a002397b097 100644
--- a/tests/studio/studiobench/scoring/payload_rules.py
+++ b/tests/studio/studiobench/scoring/payload_rules.py
@@ -125,8 +125,8 @@ def censored_metrics(records: Iterable[dict]) -> dict[str, set[str]]:
if key.endswith("_censored") and value:
metric = f"{action}.{key[:-len('_censored')]}_ms"
out.setdefault(metric, set()).add(cell)
- # The row still CARRIES the timings the scoring layer refuses to read, so the names are
- # known here even though the values will never be pooled.
+ # The row still CARRIES the timings the scoring layer refuses to read, so the names are known here
+ # even though the values will never be pooled.
if r.get("ran") and r.get("expect_ok") is False:
for key in r.get("timings") or {}:
out.setdefault(f"{action}.{key}", set()).add(cell)
@@ -179,9 +179,8 @@ def refuse_partial_censoring(records: list[dict], metric: str) -> str | None:
return None
measured = measured_cells(records, metric)
if not measured:
- # Censored everywhere it was attempted. Nothing survived to be biased, and there is no
- # pooled number to refuse -- refusing here would fire on every payload that simply cannot
- # answer, which is a different and much noisier rule than the one being enforced.
+ # Censored everywhere it was attempted: nothing survived to be biased and there is no pooled
+ # number to refuse, so refusing here would fire on every payload that simply cannot answer.
return None
rungs_censored = sorted({c.split(".", 1)[0] for c in censored if c})
rungs_measured = sorted({c.split(".", 1)[0] for c in measured if c})
@@ -225,10 +224,9 @@ def comparability_key(run_meta: dict) -> str:
import hashlib
import json
- # COMPUTED OVER `comparability_fields`, not over a second copy of the same dict. The two were
- # written out separately and had to be kept in step by hand, which is the drift this module
- # exists to argue against: a field added to one and forgotten in the other would make the key
- # and its own explanation disagree about what the key covers.
+ # COMPUTED OVER `comparability_fields`, not a second copy of the same dict. The two were written
+ # out separately and kept in step by hand, and a field added to one and forgotten in the other
+ # would make the key and its own explanation disagree about what the key covers.
blob = json.dumps(comparability_fields(run_meta), sort_keys = True, default = str).encode()
return "cmp:" + hashlib.sha256(blob).hexdigest()[:10]
@@ -296,62 +294,49 @@ def comparability_fields(run_meta: dict) -> dict:
"cadence": run_meta.get("cadence"),
"stream_tail_chars": run_meta.get("stream_tail_chars"),
"corpus_dollars": run_meta.get("corpus_dollars"),
- # THE THREE THAT CHANGE WHAT IS MEASURED, not merely what is measured on. The harness
- # already treats all three as identity axes that `--resume` refuses to toggle, and the
- # `run_meta` emission says why in its own words -- so a key that ignored them would call
- # two payloads comparable that the tool itself refuses to continue as one run.
- #
- # `inject_stream_cost_ms` is the sharpest: an arm running it is not a measurement of the
- # build at all, because the harness put the slowdown there on purpose. Nothing in the
- # scoring path refuses an injected payload, so `--compare` blessing it against a clean run
- # was a live route to publishing a difference the harness itself created.
- #
- # `click_probe` is normalised through `bool` so a payload written before the field existed
- # reads as False rather than None; the other two are `None` in the ordinary case already.
+ # THE THREE THAT CHANGE WHAT IS MEASURED, not merely what it is measured on. The harness already
+ # treats all three as identity axes that `--resume` refuses to toggle, so a key that ignored them
+ # would call two payloads comparable that the tool itself refuses to continue as one run.
+ # `inject_stream_cost_ms` is the sharpest: an arm running it is not a measurement of the build at
+ # all, because the harness put the slowdown there. Nothing in the scoring path refuses an
+ # injected payload, so `--compare` blessing it against a clean run was a live route to publishing
+ # a difference the harness created.
+ # `click_probe` is normalised through `bool` so a payload written before the field existed reads
+ # as False rather than None; the other two are already `None` in the ordinary case.
"click_probe": bool(run_meta.get("click_probe")),
"probe_init_script": run_meta.get("probe_init_script"),
"inject_stream_cost_ms": run_meta.get("inject_stream_cost_ms"),
# THE HOST, which the engine alone does not stand in for. `run_meta` records `system` and
# `machine` and the key took neither, so two payloads from different hardware and operating
- # systems hashed the same and `--compare` blessed them.
- #
- # It needs no unusual invocation: `browser.default_engine()` returns webkit on Darwin AND
- # on Linux, so a tester's Mac payload and the Linux dev box's payload, both with default
- # settings, differ in `system`, `machine` and `engine_note` and were declared comparable.
- # Only Windows was caught, and only incidentally, because its default engine differs.
- #
- # Nothing else in the tool refuses cross-host pooling: `floor_table.load` checks tier,
- # corpus and probe only, and `platform` is not an identity axis. The sole existing
- # statement is prose in `report/render.py` -- "machine-local; does not travel between
- # machines" -- so the authors knew the hazard and the one guard meant to police a prose
- # comparison was the place it was missing.
+ # systems hashed the same and `--compare` blessed them. It needs no unusual invocation:
+ # `browser.default_engine()` returns webkit on Darwin AND Linux, so a tester's Mac payload and
+ # the Linux dev box's, both default, differ in `system`, `machine` and `engine_note` and were
+ # declared comparable. Only Windows was caught, and only because its default engine differs.
+ # Nothing else refuses cross-host pooling: `floor_table.load` checks tier, corpus and probe only,
+ # and `platform` is not an identity axis. The sole existing statement is prose in
+ # `report/render.py` ('machine-local; does not travel between machines') so the one guard meant to
+ # police a prose comparison was the place it was missing.
"system": platform.get("system"),
"machine": platform.get("machine"),
- # AND WHICH PHYSICAL MACHINE, because the two fields above cannot say. `platform.machine()`
- # is the ARCHITECTURE, not a machine identifier: on two ordinary Linux x86_64 hosts it
- # returns `x86_64` on both while `system` returns `Linux` on both, so the pair that was
- # added here to stand for the host caught only the cross-OS case -- a tester's Mac against
- # the Linux dev box -- and left the commonest one, a dev box against a CI runner or a
- # second dev box, hashing identically. `platform.node()` is the host's network name and is
- # the field that separates them.
- #
- # This is the axis with the least slack in it: `floor_table.render` refuses a floor whose
- # comparability fields differ from the payload's, in the words "a floor is the scatter of
- # THIS measurement on THIS machine", and that refusal is computed from this dict. Without
- # the host in it a null control measured on one machine certifies a result measured on
- # another, which is the one comparison the report text says never travels.
- #
- # A payload recorded before this field existed carries None and is therefore not comparable
- # with one that carries a host. That is the honest reading rather than a cost: such a
- # payload does not record which machine produced it, so nothing can show it was this one.
+ # AND WHICH PHYSICAL MACHINE, because the two fields above cannot say. `platform.machine()` is the
+ # ARCHITECTURE: on two ordinary Linux x86_64 hosts it returns `x86_64` and `system` returns
+ # `Linux` on both, so the pair added to stand for the host caught only the cross-OS case and left
+ # the commonest one, a dev box against a CI runner, hashing identically. `platform.node()` is the
+ # host's network name and is what separates them.
+ # This is the axis with the least slack: `floor_table.render` refuses a floor whose comparability
+ # fields differ from the payload's, in the words 'a floor is the scatter of THIS measurement on
+ # THIS machine', and that refusal is computed from this dict. Without the host a null control
+ # measured on one machine certifies a result measured on another.
+ # A payload recorded before this field existed carries None and is therefore not comparable with
+ # one that carries a host. That is the honest reading: such a payload does not record which
+ # machine produced it.
"node": platform.get("node"),
- # WHICH BROWSER BINARY DREW THE FRAMES, which `engine` does not settle. Since Playwright
- # 1.57 headed and headless default to different executables -- `chrome` against
- # `chrome-headless-shell` -- and headless falls back to software rendering for
- # GPU-accelerated work while its compositor keeps its own pacing. For a tool whose output
- # is frames, jank and time-to-settle, those are two different renderers under one engine
- # name. Normalised through `bool` so a payload written before the field existed reads as
- # the headless default rather than as None.
+ # WHICH BROWSER BINARY DREW THE FRAMES, which `engine` does not settle: since Playwright 1.57
+ # headed and headless default to different executables (`chrome` against
+ # `chrome-headless-shell`), and headless falls back to software rendering while its compositor
+ # keeps its own pacing. For a tool whose output is frames, jank and time-to-settle those are two
+ # renderers under one engine name. Normalised through `bool` so a pre-field payload reads as the
+ # headless default rather than None.
"headed": bool(run_meta.get("headed")),
}
diff --git a/tests/studio/studiobench/scoring/schema.py b/tests/studio/studiobench/scoring/schema.py
index 7e67bbcd1b2..226a9fc76de 100644
--- a/tests/studio/studiobench/scoring/schema.py
+++ b/tests/studio/studiobench/scoring/schema.py
@@ -38,36 +38,28 @@ from typing import Any, Iterable, Mapping, Sequence
MEASURE_KIND = "measure"
-#: Keys whose numeric zero is a genuine, meaningful zero rather than a missing measurement.
-#: Everything here is an index, an identifier, a declared constant or a cardinality that is
-#: interesting precisely when it is zero. Anything not on this list must be a `Measure`.
+#: Keys whose numeric zero is a genuine zero rather than a missing measurement: indexes,
+#: identifiers, declared constants and cardinalities that are interesting precisely when zero.
+#: Anything not listed must be a `Measure`.
ZERO_OK_KEYS = frozenset(
{
"index",
- # The parity capture in scene/parity.js numbers its message rows with a bare loop
- # counter (`i`), not `index`, so the first message of every capture is `i: 0`. That is
- # an ordinal like every other name on this list, and rejecting it made `--report` fail
- # on any payload that carried parity rows at all.
+ # The parity capture in scene/parity.js numbers its message rows with a bare loop counter (`i`),
+ # so the first message of every capture is `i: 0`. Rejecting that ordinal made `--report` fail on
+ # any payload carrying parity rows.
"i",
+ # The visible-region capture, whose zeros are all true statements. NOT the whole `visible`
+ # subtree: exempting a subtree here once defeated a test requiring a zero-length `chars` inside
+ # `parity` to stay loud, and a visible message with a zero-character signature is a broken
+ # capture. Only the counters are named: no message shown was unmounted, a windowed arm unmounted
+ # them all (NOT COMPARABLE), or the scan observed nothing, which compare_visible refuses.
+ # The keys are `unmounted_at_capture`, `mounted_ever_visible` and `ever_visible_count`.
+ # The SSE wire's integrity counters, whose zeros are the good outcome, again the scalars and not
+ # the `wire` subtree. 'No frame failed to parse' and 'no unterminated frame was left buffered'
+ # are what make the character count trustworthy. `wireChars` is NOT listed: a zero there means
+ # nothing was counted and must stay loud.
# ── the visible-region capture, whose zeros are all true statements ──
- #
- # NOT the whole `visible` subtree, deliberately. Exempting a subtree here once already
- # defeated a test that required a zero-length `chars` inside `parity` to stay a loud
- # failure, and the same argument applies with more force to this one: a visible message
- # whose signature is zero characters long is a broken capture and must stay loud. Only the
- # counters are named, and each is a fact rather than a measurement:
- # unmounted_at_capture: 0 every message the viewport showed was still mounted
- # mounted_ever_visible: 0 a windowed arm had unmounted them all again, which the
- # analysis reports as NOT COMPARABLE rather than as agreement
- # ever_visible_count: 0 the scan observed nothing, which the positive control in
- # compare_visible refuses; it is not scored anywhere
# ── the SSE wire's integrity counters, whose zeros are the good outcome ──
- #
- # Again the scalars and not the `wire` subtree. "No frame failed to parse" and "no
- # unterminated frame was left buffered" are the statements that make the character count
- # trustworthy, and a run where they are zero is the run whose denominator can be scored.
- # `wireChars` itself is NOT on this list: a zero there means nothing was ever counted and
- # must stay loud.
"wire_parse_failures",
"wire_pending_chars",
"wire_parse_failures_in_window",
@@ -97,9 +89,8 @@ ZERO_OK_KEYS = frozenset(
"records_written",
"frames_total", # a window with no frames is a SYMPTOM; frames.py already flags it
"n_pairs",
- # -- harness-row counters whose zero is the GOOD result, not a missing reading. Each of
- # these is written unconditionally by the session layer for every cell, so "absent" and
- # "zero" are already distinguishable: absent means the row was never written at all.
+ # harness-row counters whose zero is the GOOD result, not a missing reading. Each is written
+ # unconditionally per cell, so absent means the row was never written at all.
"slots_missed", # 0 means the film kept time, which is exactly what we want to read
"expect_failures", # 0 means every action's expectation held
"over_budget_ms", # 0 means the action fitted inside its slot
@@ -107,26 +98,24 @@ ZERO_OK_KEYS = frozenset(
"seeded_messages",
"seed_seconds",
"bursts", # a pacer that never had to burst is a pacer that was never jammed
- # Seeded-vs-streamed equivalence: 0 drift on a field is the PASSING result, and it is the
- # single most load-bearing zero in the payload, because it is what licenses every rung
- # above 10K to be seeded rather than streamed.
+ # Seeded-vs-streamed equivalence: 0 drift on a field is the PASSING result, and the most
+ # load-bearing zero in the payload, because it licenses every rung above 10K to be seeded rather
+ # than streamed.
"drift",
- # The two counts `drift` is computed FROM, in that same block. They are censuses of the
- # DOM in each arm, so a field that is absent from both reads 0/0 and is the passing
- # result: on a real quick-tier run `reasoning_spans` is 0 streamed and 0 seeded, and the
- # payload marks the field `gating: false` in the same breath. Exempting the quotient but
- # not the two numbers under it failed every payload that carried an equivalence block.
+ # The two counts `drift` is computed FROM. They are censuses of the DOM in each arm, so a field
+ # absent from both reads 0/0 and passes: on a real quick-tier run `reasoning_spans` is 0 streamed
+ # and 0 seeded with `gating: false`. Exempting the quotient but not the two numbers failed every
+ # payload carrying an equivalence block.
"streamed",
"seeded",
}
)
-#: Subtrees the bare-zero walker does not descend into. These hold configuration, identity,
-#: histogram buckets and raw instrument bookkeeping, none of which is a measurement of the app.
-#: The boundary is a contract with the harness layer: anything that is a MEASUREMENT goes under
-#: a `metrics` key as a measure object, anything that is bookkeeping goes under `info`. Without
-#: that split the walker either fails on every legitimate counter or is switched off entirely,
-#: and switched off is how the ban stops existing.
+#: Subtrees the bare-zero walker does not descend into: configuration, identity, histogram buckets
+#: and raw instrument bookkeeping, none of which measures the app. The boundary is a contract with
+#: the harness layer (MEASUREMENTS under `metrics` as measure objects, bookkeeping under `info`)
+#: and without it the walker either fails on every legitimate counter or gets switched off, which
+#: is how the ban stops existing.
EXEMPT_SUBTREE_KEYS = frozenset(
{
"info",
@@ -135,21 +124,18 @@ EXEMPT_SUBTREE_KEYS = frozenset(
"env",
"identity",
"header",
- # The comparability key's own section. `fields` is the identity block the key is hashed
- # over -- `instrument_level`, `stream_tail_chars`, `corpus_dollars` -- every one of which
- # is a true statement about how the run was configured rather than a measurement of the
- # app, and every one of which is legitimately 0. Same rule as `identity` and `config`
- # directly above.
+ # The comparability key's own section. `fields` is the identity block the key is hashed over
+ # (`instrument_level`, `stream_tail_chars`, `corpus_dollars`), every one a true statement about
+ # how the run was configured and legitimately 0. Same rule as `identity` and `config` above.
"comparability",
"footer",
"record_counts",
"histogram",
"potency_counters",
- # -- OBSERVATION subtrees written by the session layer. These are censuses of the DOM and
- # the expectations checked against them: "0 code blocks" is a true statement about the
- # page at that instant, not a failed measurement. They are never scored and never quoted
- # as a cost; everything that IS scored reaches the report as a Measure via
- # scoring/from_payload.py, and that path stays under the strict rule.
+ # OBSERVATION subtrees written by the session layer: censuses of the DOM and the expectations
+ # checked against them, where '0 code blocks' is a true statement about the page rather than a
+ # failed measurement. They are never scored or quoted as a cost; everything scored reaches the
+ # report as a Measure via scoring/from_payload.py, under the strict rule.
"census",
"census_before",
"census_after",
@@ -157,11 +143,10 @@ EXEMPT_SUBTREE_KEYS = frozenset(
"streamed_census",
"seeded_census",
"expect",
- # The readiness gate's verdict and the completeness probe's, both written once per cell by
- # the session layer. Same rule as a census: `from_bottom: 0` is a true statement that the
- # viewport is exactly at the bottom, `waited_ms: 0` is a thread that was ready on the
- # first sample, and neither is a measurement of the app. Nothing here is scored; what the
- # gate produces for the report is a `gate` row, which carries a boolean.
+ # The readiness gate's verdict and the completeness probe's, written once per cell. Same rule as
+ # a census: `from_bottom: 0` says the viewport is exactly at the bottom and `waited_ms: 0` a
+ # thread ready on the first sample. Nothing here is scored; what the gate produces for the
+ # report is a `gate` row carrying a boolean.
"readiness",
"completeness",
"notes",
@@ -207,8 +192,6 @@ class Measure:
if not self.attempted and not self.note:
raise PayloadSchemaError("a not-attempted Measure must say why")
- # -- constructors -------------------------------------------------------------------
-
@classmethod
def not_attempted(cls, unit: str, reason: str) -> "Measure":
return cls(value = None, attempted = False, unit = unit, note = reason)
@@ -228,8 +211,6 @@ class Measure:
) -> "Measure":
return cls(value = float(value), attempted = True, unit = unit, floor = floor, note = note)
- # -- predicates ---------------------------------------------------------------------
-
@property
def has_reading(self) -> bool:
return self.attempted and self.value is not None
@@ -251,8 +232,8 @@ class Measure:
if self.value is None:
return f"no reading ({self.note or 'instrument failed'})"
if self.sub_floor:
- # A negative reading under the floor is bounded from below, not from above; printing
- # it as `< floor` would read as "small and positive" for a value that is neither.
+ # A negative reading under the floor is bounded from below, not above; printing it as `< floor`
+ # would read as 'small and positive' for a value that is neither.
if float(self.value) < 0:
return f"> -{_fmt(self.floor)} {self.unit} (instrument floor)"
return f"< {_fmt(self.floor)} {self.unit} (instrument floor)"
@@ -355,7 +336,7 @@ class ExcludedCell:
}
-#: Reasons a cell may be excluded. Free-text reasons are refused so the report can total them.
+#:Reasons a cell may be excluded. Free-text reasons are refused so the report can total them.
EXCLUSION_REASONS = frozenset(
{
"clock_disagreement",
@@ -382,8 +363,9 @@ def check_exclusion_reasons(cells: Iterable[ExcludedCell]) -> None:
)
-# ---------------------------------------------------------------------------------------
# payload validation
+
+
# ---------------------------------------------------------------------------------------
@@ -428,22 +410,19 @@ def _is_number_list(node: Any) -> bool:
def _walk_for_bare_zeros(node: Any, path: str, problems: list[str]) -> None:
if _is_measure(node):
- # A measure object is self-describing; its own zero is fine because `attempted` sits
- # next to it. Do not descend, the inner `value` is exactly the thing being exempted.
+ # A measure object is self-describing; its own zero is fine because `attempted` sits next to it.
+ # Do not descend, the inner `value` is exactly what is exempted.
if node.get("attempted") is None:
problems.append(f"{path}: measure object without `attempted`")
return
if isinstance(node, Mapping):
- # The harness layer cannot emit measure objects (its rows are JSON-safe scalars), so it
- # attests with a `<name>_attempted` flag. That flag is written ONCE PER INSTRUMENT BLOCK,
- # not once per counter: the frames instrument emits `frames_attempted: true` alongside
- # `frames_over_33`, `long_tasks` and the rest. An exact-name-sibling rule would demand
- # `frames_over_33_attempted` and fail on every real row, and a check that fails on every
- # real row gets switched off, which is how the ban stops existing.
- #
- # So a mapping that positively attests (any `*_attempted` key that is True) covers the
- # numeric zeros DIRECTLY inside it. It does not cover nested mappings, which carry their
- # own attestation or none.
+ # The harness layer cannot emit measure objects (its rows are JSON-safe scalars), so it attests
+ # with a `<name>_attempted` flag written ONCE PER INSTRUMENT BLOCK, not per counter:
+ # `frames_attempted: true` sits alongside `frames_over_33`, `long_tasks` and the rest. An
+ # exact-name-sibling rule would demand `frames_over_33_attempted` and fail on every real row, and
+ # a check that fails on every real row gets switched off.
+ # So a mapping that positively attests covers the numeric zeros DIRECTLY inside it, not nested
+ # mappings, which carry their own attestation or none.
attested = any(k.endswith("_attempted") and v is True for k, v in node.items())
for key, child in node.items():
if key in EXEMPT_SUBTREE_KEYS:
@@ -456,13 +435,10 @@ def _walk_for_bare_zeros(node: Any, path: str, problems: list[str]) -> None:
and covered
):
continue
- # The same attestation, for a plain numeric ARRAY directly inside the block. An
- # instrument that attests reports samples as well as counters: `frames` writes
- # `frames_attempted: true` next to `frame_gaps_ms`, whose entries are inter-frame
- # gaps in whole milliseconds, so two frames in the same millisecond record a
- # legitimate 0. Walking into the list dropped the attestation on the floor and every
- # such run failed validation, which is the scalar case one line above with an extra
- # pair of brackets around it.
+ # The same attestation for a plain numeric ARRAY directly inside the block: `frames` writes
+ # `frames_attempted: true` next to `frame_gaps_ms`, whose entries are whole-millisecond gaps, so
+ # two frames in one millisecond record a legitimate 0. Walking into the list dropped the
+ # attestation and failed every such run.
if covered and _is_number_list(child):
continue
_walk_for_bare_zeros(child, f"{path}.{key}", problems)
diff --git a/tests/studio/studiobench/scoring/score.py b/tests/studio/studiobench/scoring/score.py
index d6833463918..015ef5799ac 100644
--- a/tests/studio/studiobench/scoring/score.py
+++ b/tests/studio/studiobench/scoring/score.py
@@ -102,16 +102,16 @@ def score_metric(anchor: MetricAnchor, measure: Measure) -> MetricScore:
)
value = float(measure.value)
- # A sub-floor reading is at least as good as the floor; scoring the raw value would let
- # instrument noise on a fast machine invent a difference between two perfect builds.
+ # A sub-floor reading is at least as good as the floor; scoring the raw value would let instrument
+ # noise on a fast machine invent a difference between two perfect builds.
if measure.sub_floor and measure.floor is not None and anchor.lower_is_better:
value = min(abs(value), float(measure.floor))
good, bad = float(anchor.good), float(anchor.bad)
if value <= 0:
- # Log space has no zero. A non-positive reading on a positive-only metric is either
- # perfect (below every floor) or a broken instrument; the floor logic above has already
- # handled the honest case, so clamp to the good anchor and say so.
+ # Log space has no zero. A non-positive reading on a positive-only metric is either perfect (below
+ # every floor) or a broken instrument, and the floor logic above has handled the honest case, so
+ # clamp to the good anchor and say so.
value = good if anchor.lower_is_better else bad
span = math.log(bad) - math.log(good)
@@ -202,8 +202,8 @@ def score_rung(
zeroed_by = [m.key for m in scored if float(m.score) <= 0.0]
if zeroed_by:
- # Geometric mean of a set containing zero IS zero. Written out rather than left to
- # log(0) so the reason travels with the number.
+ # Geometric mean of a set containing zero IS zero. Written out rather than left to log(0) so the
+ # reason travels with the number.
rung_score = 0.0
else:
numerator = sum(m.weight * math.log(float(m.score)) for m in scored)
@@ -314,9 +314,9 @@ def score_ladder(rungs: Sequence[RungScore]) -> LadderScore:
"is above what this ladder measures"
)
- # Usability is expected to be monotone in thread size. When it is not, something other than
- # thread size moved -- thermal throttling, a background process, an unstable machine -- and
- # the onset rung is not trustworthy on its own. Say so rather than quietly reporting the max.
+ # Usability is expected to be monotone in thread size. When it is not, something other than thread
+ # size moved (throttling, a background process, an unstable machine) and the onset rung is not
+ # trustworthy on its own.
non_monotonic = False
seen_unusable = False
for rung in ordered:
diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py
index bc1aa8fca92..eeaca27ecce 100644
--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py
+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py
@@ -258,8 +258,8 @@ def test_real_payload_shape_round_trips(tmp_path):
ladder = score_payload(path, [1_000])
assert [r.tokens for r in ladder.rungs] == [1_000]
rung = ladder.rungs[0]
- # keystroke + menu + the three frame metrics = 85% of the weight, over the 60% floor, so the
- # rung scores despite scroll being legitimately unavailable at this size.
+ # keystroke + menu + the three frame metrics = 85% of the weight, over the 60% floor, so the rung
+ # scores despite scroll being legitimately unavailable at this size.
assert rung.usable is True
@@ -476,16 +476,14 @@ def _payload_with(cell):
}
-# ---------------------------------------------------------------------------------------
-# The latest attempt is the last one that WROTE anything, not the last one that finished
-# ---------------------------------------------------------------------------------------
-#
+# The latest attempt is the last one that WROTE anything, not the last one that finished.
# `CellRunner.run` writes its terminal cell row in a `finally`, which a SIGKILL, an OOM kill or a
-# lost machine never reaches -- while the Recorder has already flushed and fsynced every action and
+# lost machine never reaches, while the Recorder has already flushed and fsynced every action and
# window row before it. Keyed on cell rows alone, a resume hard-killed inside a cell left the
# older, completed attempt named as the latest, and `_resume_set` skipped it.
+# ---------------------------------------------------------------------------------------
def _stamped(row, session):
return {**row, "session_id": session}
diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py
index f7ebe445630..5d5d0207ea5 100644
--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py
+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py
@@ -287,10 +287,10 @@ def test_the_key_is_computed_over_the_fields_it_explains():
"""
corpus = "ac9d5d8e37be2a3844deed559fde6070247ad2322377295fb383b60b5eec5a0c"
a = _meta(corpus)
- #: Fields the key reads out of the nested `platform` dict rather than off the row itself.
- #: Written as a set rather than a single special case, because the single special case is what
- #: had to be edited when `system` and `machine` were added -- and a test that needs editing to
- #: keep passing when a field is added is a test that can be edited into silence instead.
+ #: Fields the key reads out of the nested `platform` dict rather than off the row itself. Written as
+ #: a set rather than a single special case, because the special case is what had to be edited when
+ #: `system` and `machine` were added, and a test that needs editing to keep passing can be edited
+ #: into silence instead.
nested = {"engine", "system", "machine", "node"}
for field in payload_rules.comparability_fields(a):
b = _meta(corpus)
diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py
index 58c8f8a9533..5db08304371 100644
--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py
+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py
@@ -45,11 +45,10 @@ from studiobench.scoring import ( # noqa: E402
from studiobench.scoring.anchors import ONSET_SCORE_THRESHOLD # noqa: E402
-# ---------------------------------------------------------------------------------------
# Measure: the ban on bare zeros
-# ---------------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
def test_not_attempted_cannot_carry_a_value():
with pytest.raises(PayloadSchemaError):
Measure(value = 1.0, attempted = False, unit = "ms", note = "nope")
@@ -214,11 +213,10 @@ def test_validate_payload_requires_excluded_cells():
validate_payload({"excluded_cells": None})
-# ---------------------------------------------------------------------------------------
# frames: both directions of jank
-# ---------------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
def test_uniform_mediocrity_is_caught_by_time_in_jank_and_missed_by_max():
"""Every frame 120 ms. `max` says 120, which sounds survivable. It is not."""
@@ -260,11 +258,10 @@ def test_histogram_is_always_present_and_totals_the_frames():
assert sum(b["bucket_count"] for b in stats.histogram) == len(deltas)
-# ---------------------------------------------------------------------------------------
# per-metric and per-rung scoring
-# ---------------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
def test_log_anchors_put_the_geometric_midpoint_at_fifty():
anchor = METRIC_BY_KEY["keystroke_p95_ms"]
midpoint = math.sqrt(anchor.good * anchor.bad)
@@ -332,11 +329,10 @@ def test_missing_most_metrics_makes_the_rung_incomplete_rather_than_easy():
assert "weight" in rung.incomplete_reason
-# ---------------------------------------------------------------------------------------
# aggregation, and the three ways naive AUC lies
-# ---------------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
def test_log_rung_weights_do_not_let_the_top_rung_be_the_whole_score():
weights = log_rung_weights([1_000, 10_000, 100_000, 500_000, 1_000_000])
assert sum(weights) == pytest.approx(1.0)
@@ -409,11 +405,10 @@ def test_non_monotone_usability_is_flagged_rather_than_maximised():
assert len(rungs) == 3
-# ---------------------------------------------------------------------------------------
# A/B
-# ---------------------------------------------------------------------------------------
+# ---------------------------------------------------------------------------------------
def _identity(session: str = "s1", **overrides) -> RunIdentity:
fields = {
"bench_version": "studiobench/1",
@@ -537,17 +532,15 @@ def test_bootstrap_ci_brackets_the_geometric_mean():
assert metric.ci_low <= metric.ratio_geomean <= metric.ci_high
-# ---------------------------------------------------------------------------------------
-# A CI that contains 1.0 is not a result: the pairs have to agree on the sign
-# ---------------------------------------------------------------------------------------
-#
-# The noise floor is a fact about the HARNESS -- whether a difference of this size is resolvable
-# here at all -- and it was the only gate a direction had to pass. Repetitions that disagree
-# produce a geometric mean well clear of the floor anyway: 0.7, 0.7, 1.2, 1.2 averages to 0.917
-# with a CI of 0.700-1.200, and the table said "improved" over it, in the one column anybody
-# quotes. The CI was computed, printed, and never consulted.
+# A CI that contains 1.0 is not a result: the pairs have to agree on the sign. The noise floor is a
+# fact about the HARNESS, whether a difference of this size is resolvable at all, and it was the
+# only gate a direction had to pass. Repetitions that disagree produce a geometric mean well clear
+# of the floor anyway: 0.7, 0.7, 1.2, 1.2 averages to 0.917 with a CI of 0.700-1.200, and the
+# table said 'improved' over it in the one column anybody quotes. The CI was computed, printed,
+# and never consulted.
+# ---------------------------------------------------------------------------------------
def _split_pairs(metric: str, ratios: list[float]) -> list[Pair]:
"""One metric, one ratio per rung, so the pairs can be made to disagree on the sign."""
out = []
@@ -635,22 +628,16 @@ def test_the_table_does_not_print_a_direction_it_could_not_resolve():
assert "contains it" in text # the interval straddles 1.0, as opposed to being absent
-# ---------------------------------------------------------------------------------------
-# A measured zero is a reading: sub-floor arms bound the ratio rather than voiding the pair
-# ---------------------------------------------------------------------------------------
-#
-# `Pair.usable` required both values to be strictly positive. `time_in_jank_pct` and `jank_index`
-# are 0.0 on any arm smooth enough to have no over-budget frames, which is the ordinary state of a
-# healthy base, so a treatment that introduced jank over a zero-jank base had its pair dropped:
-# the table said `no reading` about two arms that had both read, the regression was absent from
-# `regressions` and from the headline, and as a null control it neither voided nor reached the
-# noise floor derived from it.
-#
-# The rule applied here is the one `score.py` has always applied to the same reading: a sub-floor
-# value is at least as good as the floor, so the floor is what enters the ratio. What that must
-# NOT do is admit a reading that was never taken, which is the distinction `frames.py` protects
-# when it refuses to score an unscheduled rAF loop as zero jank.
+# A measured zero is a reading: sub-floor arms bound the ratio rather than voiding the pair.
+# `Pair.usable` required both values to be strictly positive, but `time_in_jank_pct` and
+# `jank_index` are 0.0 on any arm smooth enough to have no over-budget frames, so a treatment that
+# introduced jank over a zero-jank base had its pair dropped: the table said `no reading` about
+# two arms that had both read. The rule applied here is the one `score.py` has always applied, a
+# sub-floor value is at least as good as the floor so the floor enters the ratio, without
+# admitting a reading that was never taken, the distinction `frames.py` protects when it refuses
+# to score an unscheduled rAF loop as zero jank.
+# ---------------------------------------------------------------------------------------
JANK_FLOOR = 0.1
SMOOTH = Measure.read(0.0, "%", floor = JANK_FLOOR)
JANKY = Measure.read(5.0, "%", floor = JANK_FLOOR)
@@ -665,8 +652,8 @@ def test_a_zero_jank_base_still_pairs_against_a_treatment_that_introduced_jank()
assert pair.base.has_reading and pair.treatment.has_reading
assert pair.usable is True
- # The floor stands in for the sub-floor arm, so the ratio is a LOWER bound: the true
- # magnitude is larger, never smaller.
+ # The floor stands in for the sub-floor arm, so the ratio is a LOWER bound: the true magnitude is
+ # larger, never smaller.
assert pair.ratio == 50.0
assert pair.bounded is True
assert pair.to_json()["bounded"] is True
diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py
index b5328ee2e68..fe28a10bb73 100644
--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py
+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py
@@ -368,9 +368,9 @@ def test_an_action_running_during_generation_does_not_set_the_worst_streaming_fr
m = _stream_measures(windows)
assert m["stream_max_frame_ms"].value == pytest.approx(286.0)
# And its SSE task chains are excluded from the targeted numerator too. Measured on a
- # standard-tier 10K null, an `action:keystroke` chain cost 23.77 ms per burst against 1.69 ms
- # in the gap windows either side: the chain runs until the event loop drains, so the typing
- # lands inside it and is billed to the stream.
+ # standard-tier 10K null, an `action:keystroke` chain cost 23.77 ms per burst against 1.69 ms in
+ # the gap windows either side: the chain runs until the event loop drains, so the typing lands
+ # inside it and is billed to the stream.
assert "1 unaided streaming window(s)" in m["stream_delta_cost_ms_per_kchar"].note
@@ -473,8 +473,8 @@ def test_a_recorder_that_died_partway_poisons_every_streaming_metric():
m["stream_cost_ms_per_kchar"].note or ""
)
- # The giveaway: without this the crashed cell was byte-identical to the one that never
- # crashed, so nothing downstream could tell a truncated run from a complete one.
+ # The giveaway: without this the crashed cell was byte-identical to the one that never crashed, so
+ # nothing downstream could tell a truncated run from a complete one.
assert _stream_measures([good])["stream_cost_ms_per_kchar"].value is not None
diff --git a/tests/studio/studiobench/sweep/floor_table.py b/tests/studio/studiobench/sweep/floor_table.py
index 34c42d2790c..3d814067dc7 100644
--- a/tests/studio/studiobench/sweep/floor_table.py
+++ b/tests/studio/studiobench/sweep/floor_table.py
@@ -35,9 +35,8 @@ import sys
from pathlib import Path
if __package__ in (None, ""): # pragma: no cover
- # Running the file directly rather than as a module. Supported because the first thing a new
- # contributor does with a script is run it by path, and failing there with an import error is a
- # bad first minute.
+ # Running the file directly rather than as a module, because the first thing a new contributor
+ # does with a script is run it by path.
sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
from tests.studio.studiobench.runtime.ab import failed_invalidating_gates # noqa: E402
@@ -56,28 +55,16 @@ from tests.studio.studiobench.scoring import payload_rules # noqa: E402
METRICS = tuple(ACTION_SOURCES) + FRAME_METRICS + STREAM_METRICS
-# The metrics compared by DIFFERENCE rather than by ratio, because zero is their CLEAN reading.
-#
-# Everything else here is a duration, and a duration is zero only when nothing happened. Jank is
-# the opposite: a stream that never dropped a frame reports exactly 0.0, and that is the reading a
-# good build is supposed to produce. Measured on this repository's recorded payloads,
-# `stream_time_in_jank_pct` is exactly 0.0 on 765 of 1,438 scored cells, and 349 of 668 base-vs-
-# treatment pairs across 117 payload files have a zero base arm.
-#
-# Paired as a ratio those pairs cannot survive: `t / 0.0` is undefined, so `paired` dropped them,
-# and dropping them is not a conservative choice. It removes exactly the comparisons where the base
-# was CLEAN -- the ones a treatment that introduces jank shows up in -- so a regression from 0.0%
-# to 12.0% left no row at all, and a run where only some repetitions had a zero base kept the rest
-# under the same `n`, pooling a metric over a set whose membership depends on the base arm's value.
-#
-# A difference is defined from zero, so these are pooled as `treatment - base` in the metric's own
-# unit, for EVERY pair rather than only the zero ones: mixing the two within one mean would be a
-# second unstable pool. The null control's floor is computed the same way, so the gates compare
-# like with like, and the table marks these rows so the column is not read as a percentage change.
-#
-# Scoped to the two streaming metrics this file has just started harvesting. `time_in_jank_pct`
-# has the same shape and loses 3% of its pairs, but it is already published against ratio-based
-# floors, and re-basing it would silently re-score every table taken so far.
+# The metrics compared by DIFFERENCE rather than ratio, because zero is their CLEAN reading:
+# `stream_time_in_jank_pct` is exactly 0.0 on 765 of 1,438 scored cells. Paired as a ratio
+# `t / 0.0` is undefined, so `paired` dropped exactly the comparisons where the base was
+# clean, the ones a treatment that introduces jank shows up in. Pooled as `treatment - base`
+# for EVERY pair rather than only the zero ones, since mixing the two in one mean would be a
+# second unstable pool; the null control's floor is computed the same way and the table marks
+# these rows. Scoped to the two streaming metrics: `time_in_jank_pct` has the same shape but
+# is already published against ratio-based floors.
+# And 349 of 668 pairs have a zero base arm.
+# Across 117 payload files.
DIFFERENCE_METRICS: frozenset[str] = frozenset({"stream_time_in_jank_pct", "stream_jank_index"})
@@ -106,15 +93,11 @@ def _action_timings(records: list[dict], cid: str) -> dict[str, float]:
for key, value in (row.get("timings") or {}).items():
if isinstance(value, (int, float)) and not isinstance(value, bool):
out[f"{name}.{key}"] = float(value)
- # Correctness invariants, harvested alongside the timings and named apart from them.
- #
- # Same paired arithmetic, opposite meaning. A timing falling is the result a change is
- # trying to produce; a count falling is a regression, and one that no timing can reveal.
- # `select_all_copy.count.selected_chars` is the case this exists for: the selection is
- # taken over the viewport's DOM, so anything that stops mounting the whole thread
- # truncates the clipboard while every timing improves and the action still reports
- # `expect_ok`. Reading it against the other arm needs no calibration, because both arms
- # seed a byte-identical thread.
+ # Correctness invariants, harvested alongside the timings and named apart from them: same
+ # paired arithmetic, opposite meaning, because a count falling is a regression no timing can
+ # reveal. `select_all_copy.count.selected_chars` is the case this exists for: the selection is
+ # taken over the viewport's DOM, so anything that stops mounting the whole thread truncates
+ # the clipboard while every timing improves and the action still reports `expect_ok`.
for key, value in (row.get("counts") or {}).items():
if isinstance(value, (int, float)) and not isinstance(value, bool):
out[f"{name}.count.{key}"] = float(value)
@@ -148,33 +131,20 @@ def refuse_collisions(records: list[dict]) -> None:
if not collided:
return
# A SEQUENTIAL RE-RUN IS NOT A COLLISION, and refusing it rejected every resumed A/B.
- #
- # `ab.skippable_cells` skips a repetition only when EVERY arm of it completed, and an A/B with
- # any work left re-runs EVERY pair in one new session -- deliberately, because a lone arm
- # measured in a new session can never be paired and a partial table publishes a verdict over
- # whichever rungs survived. So a resumed comparison legitimately writes a SECOND completed row
- # under the same deterministic `cell_id`, and the writer's own docstring says what happens
- # next: "The old attempt stays in the payload and `latest_attempt_rows` supersedes it."
- # Nothing here superseded anything, so a resumed run could not be scored at all.
- #
- # WHAT IS ASKED IS WHETHER THE TWO SESSIONS WERE RUNNING AT ONCE, which is the property that
- # makes the readings contended, and it is recorded: each session's `run_meta` carries
- # `started_at` and every row carries `ts_ms` from that session's own clock. On the real
- # two-launcher payload of defect 9 the two sessions occupy 23:50:24-00:10:03 and
- # 23:53:45-00:12:43, sixteen minutes of overlap. A resume starts after its predecessor stopped.
- #
- # SILENCE IS NOT A LICENCE. A payload that does not carry the evidence -- no `started_at`, no
- # `ts_ms` -- is refused exactly as before, because "no proof they overlapped" is not "proof
- # they did not", and this is the guard that stands between a reader and a blend of two
- # contending runs. Only a payload that can SHOW its sessions were sequential is superseded.
+ # `ab.skippable_cells` re-runs EVERY pair of an unfinished A/B in one new session, so a
+ # resumed comparison legitimately writes a SECOND completed row under the same `cell_id` for
+ # `latest_attempt_rows` to supersede. What is asked is whether the two sessions were running
+ # AT ONCE, which is recorded: each `run_meta` carries `started_at` and every row a `ts_ms` (on
+ # the real two-launcher payload of defect 9 the sessions overlap by sixteen minutes). Silence
+ # is not a licence: a payload carrying no such evidence is refused exactly as before.
guilty: set[str] = set()
for sessions in collided.values():
guilty |= sessions
verdict, both = concurrent_sessions(records, only = guilty)
if verdict == "sequential":
- # The caller supersedes through `latest_attempt_rows`, which keeps the LAST attempt that
- # wrote anything -- so a resume that was itself hard-killed leaves an incomplete cell and
- # drops out, rather than resurrecting the older completed reading.
+ # The caller supersedes through `latest_attempt_rows`, which keeps the LAST attempt that wrote
+ # anything, so a resume that was itself hard-killed drops out rather than resurrecting the
+ # older completed reading.
return
if verdict == "overlap":
why = (
@@ -333,63 +303,31 @@ def cell_metrics(records: list[dict], session: str | None = None) -> dict[str, d
"""
if session is None:
refuse_collisions(records)
- # SUPERSEDED ATTEMPTS ARE DROPPED, because a resumed A/B re-runs a repetition that had
- # already completed and both copies are in the file. Keyed on the LAST attempt that wrote
- # anything rather than the last that FINISHED: a resume killed inside a cell has flushed
- # its action rows and never reaches the cell row its `finally` would write, and treating
- # the older completed attempt as current would hand a stale reading to a floor. Dropped
- # here, that repetition has no completed cell and falls out of the table instead.
- #
- # ONLY WHEN NO SESSION WAS NAMED. `session=` is the escape hatch this module's own refusal
- # points a reader at -- "pass `session=` to score one of them" -- and superseding inside it
- # would empty the older of two concurrent sessions and take the hatch away. A caller who
- # names a session is asking for that session, not for the payload's latest view of it.
+ # SUPERSEDED ATTEMPTS ARE DROPPED, because a resumed A/B re-runs a completed repetition and
+ # both copies are in the file. Keyed on the LAST attempt that wrote anything rather than the
+ # last that FINISHED: a resume killed inside a cell has flushed its action rows and never
+ # reaches the cell row, and treating the older completed attempt as current would hand a stale
+ # reading to a floor. ONLY WHEN NO SESSION WAS NAMED: `session=` is the escape hatch this
+ # module's refusal points a reader at, and superseding inside it would empty the older of two
+ # concurrent sessions.
records = list(latest_attempt_rows(records))
# A CELL THAT FAILED AN INVALIDATING GATE IS NOT A READING, HERE EITHER. Both gates are
- # advisory where they are emitted, so such a cell arrives `completed=True` with a full set of
- # timings that are CHEAPER than a correct cell's, and this table is where a floor is built and
- # where a result is scored against it. `paired` divides treatment by base per pair, so a
- # one-sided failure is not cancelled by the null control being same-build: it lands in
- # `delta_pct` and `spread_pct` directly. On the result side it is worse, because a gate-failed
- # treatment cell pairs against a clean base cell and prints as `faster`.
- #
- # This is not hypothetical. The virtualization negative result recorded in CONTRIBUTING-perf.md
- # was measured through this workflow and contains both facts at once: the treatment arm's
- # census going from 12 mounted messages to 0 and never recovering, and a headline of 28.2%
- # faster weighted. Nothing between the two noticed.
- #
- # Scoped by hand, because a gate row is not one of the row types `latest_attempt_rows` covers:
- # that reduction has already dropped the superseded attempt's cells, actions and windows above,
- # and a `gate` row carries no attempt of its own to supersede.
+ # advisory where they are emitted, so such a cell arrives `completed=True` with timings
+ # CHEAPER than a correct cell's, and `paired` divides treatment by base per pair, so a
+ # one-sided failure lands in `delta_pct` and `spread_pct` directly. Worse on the result side,
+ # where a gate-failed treatment cell pairs against a clean base and prints as `faster`. Not
+ # hypothetical: the virtualization negative result in CONTRIBUTING-perf.md has the treatment's
+ # census going from 12 mounted messages to 0 alongside a 28.2% faster headline. Scoped by
+ # hand, because a `gate` row is not a row type `latest_attempt_rows` covers.
gate_failures = failed_invalidating_gates(records)
- # A SUPERSEDED ATTEMPT'S COMPLETION IS NOT THIS CELL'S READING, and the session scoping below
- # cannot say so: it answers "which rows belong to this completed cell row", not "is this
- # completed cell row still the cell". `ab.skippable_cells` re-runs an A/B pair WHOLE, so a
- # resume with any work left re-attempts cells that had already completed, and a retry that
- # dies leaves the older completed row as the only one carrying a full set of timings.
- #
- # Left raw, that row is admitted -- and the guard above it disagrees about which attempt is
- # being scored. `failed_invalidating_gates` names the winner from the LAST cell row, so a
- # retry that crashed and wrote its `completed=False` row supersedes the dead attempt's FAILED
- # gate while this loop keeps that same attempt's numbers. A cell refused for losing its
- # thread's middle before the resume was published after it, pairing its cheaper timings
- # against a clean base arm and printing as `faster`: the 28.2% failure described above,
- # arriving through the resume. Same rows, same rule, one lens -- as `runtime/ab.readings_by_arm`
- # and `sweep/ui_parity.collect` already read them.
- #
- # THE REDUCTION THAT DOES THIS NOW SITS AT THE TOP OF THE FUNCTION, and it is CONDITIONAL --
- # skipped when the caller named a `session`. A second unconditional call here would have been
- # a no-op on the default path and a silent regression on that one: it would supersede inside
- # the escape hatch, empty the older of two concurrent sessions, and take away the very thing
- # this module's collision refusal tells the reader to reach for. So the guard above and this
- # loop still read one lens, and which lens that is now depends on the argument, deliberately.
- #
- # Left open on purpose: with `session=` named, `failed_invalidating_gates` still resolves its
- # winner across every session in the file, so a caller who names the OLDER of two sessions can
- # have its gate discarded by the newer one while this loop admits its cells. Scoping the gate
- # search to the named session is the fix, and it is a change to the escape hatch's semantics
- # rather than to this merge.
+ # A SUPERSEDED ATTEMPT'S COMPLETION IS NOT THIS CELL'S READING, and session scoping cannot say
+ # so. Left raw, the older completed row is admitted while `failed_invalidating_gates` names
+ # its winner from the LAST cell row, so a crashed retry supersedes the dead attempt's FAILED
+ # gate while this loop keeps that attempt's numbers: the 28.2% failure above, arriving through
+ # the resume. The reduction that fixes it sits at the top of the function and is CONDITIONAL,
+ # skipped when the caller named a `session`, so it cannot empty the escape hatch. Left open on
+ # purpose: with `session=` named, `failed_invalidating_gates` still resolves across the file.
out: dict[str, dict[str, float]] = {}
if session is not None:
@@ -402,16 +340,15 @@ def cell_metrics(records: list[dict], session: str | None = None) -> dict[str, d
if str(row.get("cell_id")) in gate_failures:
continue
cid = row["cell_id"]
- # Scoped to this cell's OWN attempt: `--resume` re-runs a died cell under a new
- # session_id into the same file, and pooling both attempts mixes two measurements.
+ # Scoped to this cell's OWN attempt: `--resume` re-runs a died cell under a new session_id into
+ # the same file, and pooling both attempts mixes two measurements.
sid = row.get("session_id")
own = [r for r in records if r.get("cell_id") == cid and r.get("session_id") == sid]
vals: dict[str, float] = _action_timings(own, cid)
- # `UNSCORED_WINDOW_KINDS` is idle plus setup, dropped here as in `measures_from_records`.
- # The setup window is the composer click that starts the film, which is mostly
- # Playwright's injected actionability script blocking the page's own main thread; pooled
- # in, an 11 s driver stall would set this table's `max_frame_ms` floor and hide every real
- # regression under it.
+ # `UNSCORED_WINDOW_KINDS` is idle plus setup, dropped here as in `measures_from_records`: the
+ # setup window is the composer click, mostly Playwright's injected actionability script
+ # blocking the main thread, so pooled in an 11 s driver stall would set this table's
+ # `max_frame_ms` floor.
windows = [
w
for w in own
@@ -421,10 +358,9 @@ def cell_metrics(records: list[dict], session: str | None = None) -> dict[str, d
for key, m in _frame_measures(windows).items():
if m.value is not None:
vals[key] = float(m.value)
- # The streaming phase on its own, per streamed character. Kept separate from the pooled
- # frame metrics rather than replacing them: the pooled ones answer "was this film janky",
- # which is still worth asking, and these answer "what did streaming one character cost",
- # which nothing answered before.
+ # The streaming phase on its own, per streamed character. Kept separate from the pooled frame
+ # metrics rather than replacing them: those answer "was this film janky", these answer "what
+ # did streaming one character cost".
for key, m in _stream_measures(windows).items():
if m.value is not None:
vals[key] = float(m.value)
@@ -475,21 +411,15 @@ def paired(records: list[dict], shard: str = "") -> dict[str, list[tuple[float,
A payload recorded before session ids existed has `""` on both arms and pairs exactly as before.
"""
- # THE SESSION IS PART OF THE KEY, for the same reason the shard is. Two sessions in one
- # payload both produce `rep0`, and pairing on the repetition alone matches one session's base
- # against the other's treatment -- which is a comparison between two different runs under two
- # different machine loads, reported as a repetition.
- #
- # Looped per session because several sessions in one payload is ORDINARY -- a sharded or
- # resumed run appends legitimately, and each session's repetitions must pair within themselves.
- # What is never ordinary is one cell completing twice, so that is refused first, here rather
- # than only inside `cell_metrics`: this loop always passes a session, so the refusal there was
- # unreachable from the shipped path and a payload of two concurrent runs pooled straight
- # through it.
+ # THE SESSION IS PART OF THE KEY, for the same reason the shard is: two sessions in one payload
+ # both produce `rep0`, and pairing on the repetition alone matches one session's base against
+ # the other's treatment. Looped per session because several sessions in one payload is
+ # ORDINARY; what is never ordinary is one cell completing twice, so that is refused first,
+ # here rather than only inside `cell_metrics`, where the shipped path made it unreachable.
refuse_collisions(records)
# The same superseding `cell_metrics` applies, done once here so the session list below is the
# set of sessions that still own a completed cell. Without it a finished ladder re-run pairs
- # ONCE PER SESSION and one repetition is pooled twice, under two different sets of numbers.
+ # ONCE PER SESSION and one repetition is pooled twice.
records = list(latest_attempt_rows(records))
by_key: dict[tuple[str, str, str, str], dict[str, dict[str, float]]] = collections.defaultdict(
dict
@@ -504,9 +434,9 @@ def paired(records: list[dict], shard: str = "") -> dict[str, list[tuple[float,
continue
for metric in set(sides["base"]) & set(sides["treatment"]):
b, t = sides["base"][metric], sides["treatment"][metric]
- # `if b` is a ratio's precondition, not a validity test: a zero base is a real reading
- # for the jank metrics and only an undefined DENOMINATOR. Those are compared by
- # difference (see DIFFERENCE_METRICS), so they are kept whenever both sides are finite.
+ # `if b` is a ratio's precondition, not a validity test: a zero base is a real reading for the
+ # jank metrics and only an undefined DENOMINATOR, and those are compared by difference (see
+ # DIFFERENCE_METRICS).
if metric in DIFFERENCE_METRICS:
if math.isfinite(b) and math.isfinite(t):
out[metric].append((b, t))
@@ -578,12 +508,10 @@ def load(paths: list[Path]) -> tuple[dict[str, list[tuple[float, float]]], set[s
corpora: set[str] = set()
for path in paths:
records = read_rows(path)
- # REFUSED HERE, not warned about. This is the same class of refusal as the tier and
- # corpus checks below, one step earlier: those two say "these are different films", and
- # this one says "this film was shot with the camera in the shot". There is no flag to
- # override it, because the only correct response is to re-run without the probe. The
- # check itself lives in the scoring layer so that the A/B table and `--report` refuse on
- # exactly the same evidence rather than on a second copy of the rule.
+ # REFUSED HERE, not warned about: the tier and corpus checks below say "these are different
+ # films", and this one says "this film was shot with the camera in the shot". No flag
+ # overrides it, because the only correct response is to re-run without the probe. The check
+ # lives in the scoring layer so the A/B table and `--report` refuse on the same evidence.
refuse_if_probed(records, str(path))
tiers |= tiers_of(records)
corpora.add(corpus_of(records))
@@ -595,10 +523,9 @@ def load(paths: list[Path]) -> tuple[dict[str, list[tuple[float, float]]], set[s
f"fast-tier film and a standard-tier film are different measurements of "
f"the same action, not repetitions of one."
)
- # Same rule one level down. The tier fixes how long the film runs; the corpus hash fixes what
- # is IN it, and it covers the generator's parameters as well as every unit's bytes. Corpus v2
- # added math, so a v1 payload and a v2 payload measure two different documents under one name,
- # and pooling them would read the corpus change as a performance change.
+ # Same rule one level down: the tier fixes how long the film runs, the corpus hash fixes what
+ # is IN it, covering the generator's parameters and every unit's bytes. Corpus v2 added math,
+ # so pooling a v1 and a v2 payload would read the corpus change as a performance change.
if len(corpora) > 1:
raise SystemExit(
f"refusing to pool payloads built on different corpora: "
@@ -634,21 +561,12 @@ def partial_censoring(paths: list[Path]) -> dict[str, str]:
everything: list[dict] = []
for path in paths:
# SUPERSEDED ATTEMPTS DROPPED, exactly as `paired` drops them, because this decides whether
- # the number `paired` produced may be quoted. A resume that died mid-cell leaves an action
- # row whose `expect_ok` is False, and `censored_metrics` counts that as a censored cell --
- # so reading the raw file here would mark a metric partially censored on the strength of an
- # attempt that contributed nothing to the mean. The two readers have to see one payload.
- #
- # NAMESPACED BY SHARD, for the same reason `paired` keys on it: sharding restarts the
- # repetition counter, so `r500K.base.rep0` names a DIFFERENT cell in each shard. Merged on
- # the bare id, the completed-cell filter below stops separating them -- one shard's failed
- # cell and another shard's successful one become a single id that is both censored and
- # completed, and a metric nothing censored anywhere is refused as partially censored on
- # the strength of a cell that contributed nothing to the mean. That is the very case the
- # paragraph above drops superseded attempts to avoid, arriving one file over.
- #
- # SUFFIXED rather than prefixed, so `refuse_partial_censoring` still reads the rung off
- # `cell_id.split(".", 1)[0]` and its message names rungs rather than directories.
+ # `paired`'s number may be quoted: a resume that died mid-cell leaves an action row whose
+ # `expect_ok` is False, which `censored_metrics` counts as a censored cell. NAMESPACED BY
+ # SHARD, for the same reason `paired` keys on it: sharding restarts the repetition counter, so
+ # merged on the bare id one shard's failed cell and another's successful one become a single
+ # id that is both censored and completed. SUFFIXED rather than prefixed, so
+ # `refuse_partial_censoring` still reads the rung off `cell_id.split(".", 1)[0]`.
shard = str(path.parent.name)
for row in latest_attempt_rows(read_rows(path)):
row = dict(row)
@@ -674,29 +592,18 @@ def summarise(paths: list[Path]) -> dict[str, dict[str, float]]:
censoring = partial_censoring(paths)
for metric, rows in pooled.items():
if metric in DIFFERENCE_METRICS:
- # Same three quantities, same three gates, measured in the metric's own unit because a
- # ratio from a clean zero base does not exist. `delta_pct` and `spread_pct` are then
- # percentage POINTS (or ms), which is why `render` marks the row.
+ # Same three quantities and gates, measured in the metric's own unit because a ratio from a
+ # clean zero base does not exist; `delta_pct` and `spread_pct` are then percentage POINTS,
+ # which is why `render` marks the row.
diffs = [t - b for b, t in rows]
- # A TIE IS NOT A DISAGREEMENT, AND HERE THE TIE IS THE MODAL READING.
- #
- # The ratio branch below asks `all(r > 1) or all(r < 1)`, and that is sound there
- # because a paired ratio of exactly 1.0 is vanishingly rare for a continuous timing.
- # Reused verbatim on differences it stops being sound, because the tie value is 0.0
- # and 0.0 is what a CLEAN reading of these metrics is: `stream_time_in_jank_pct` is
- # exactly 0.0 on 765 of 1,438 scored cells, so a repetition where neither arm dropped
- # a frame contributes a difference of exactly 0.0.
- #
- # A group of `[0.0, 12.0]` then failed both halves of the strict test and was reported
- # as VOID (pairs disagree on sign), which is not what happened: nothing moved the other
- # way, some repetitions were unchanged and the rest regressed together. Measured over
- # the recorded payloads, that voided 34 of 250 pooled groups, 13.6%, and every one of
- # the 34 came from a tie rather than from a real disagreement. Intermittent and
- # load-dependent jank is exactly the shape that produces ties, so the predicate was
- # discarding the regressions it was most needed for.
- #
- # Direction is therefore "no pair went the other way", with at least one that moved,
- # so an all-zero group stays inconsistent rather than being scored as a finding.
+ # A TIE IS NOT A DISAGREEMENT, AND HERE THE TIE IS THE MODAL READING. The ratio branch below
+ # asks `all(r > 1) or all(r < 1)`, which is sound for a continuous timing but not for
+ # differences, where the tie value 0.0 is what a CLEAN reading is. A group of `[0.0, 12.0]`
+ # then failed both halves and was reported VOID though nothing moved the other way: over the
+ # recorded payloads that voided 34 of 250 pooled groups, every one from a tie rather than a
+ # real disagreement, and intermittent jank is exactly the shape that produces ties. Direction
+ # is therefore "no pair went the other way", with at least one that moved, so an all-zero
+ # group stays inconsistent.
nonzero = [d for d in diffs if d != 0.0]
out[metric] = {
"n": len(rows),
@@ -715,21 +622,19 @@ def summarise(paths: list[Path]) -> dict[str, dict[str, float]]:
"base": statistics.fmean(b for b, _ in rows),
"treat": statistics.fmean(t for _, t in rows),
"delta_pct": (statistics.fmean(ratios) - 1.0) * 100.0,
- # GATE 2. Do the pairs even agree on the SIGN? A metric whose four paired ratios
- # straddle 1.0 has no direction, however far its mean happens to land from the floor.
- # This is what separates "the change moved it" from "the metric is thrashing".
+ # GATE 2: do the pairs even agree on the SIGN? A metric whose four paired ratios straddle 1.0
+ # has no direction however far its mean lands from the floor, which is what separates "the
+ # change moved it" from "the metric is thrashing".
"consistent": all(r > 1.0 for r in ratios) or all(r < 1.0 for r in ratios),
- # The spread of the paired ratios. From a null control it is the detection floor; from
- # a real comparison it is the effect's own scatter, which is GATE 3.
+ # The spread of the paired ratios: from a null control it is the detection floor, from a real
+ # comparison the effect's own scatter, which is GATE 3.
"spread_pct": (max(ratios) - min(ratios)) * 100.0,
"difference": False,
}
- # LABELLED, NOT DROPPED AND NOT RAISED. Raising would be the defect-10 mistake: `open_ms`
- # is censored above 100K on EVERY standard and full run, so a refusal that exited would
- # abort the whole table on the normal case and the guard would be removed within a day.
- # Dropping the row silently would delete a reading somebody may still want at the one rung
- # that produced it. So the row survives, carries the rungs it actually covers, and is
- # denied a verdict below -- it can no longer clear a gate or be counted as a survivor.
+ # LABELLED, NOT DROPPED AND NOT RAISED. Raising would be the defect-10 mistake, since `open_ms`
+ # is censored above 100K on EVERY standard run, so a refusal would abort the whole table on
+ # the normal case; dropping silently would delete a reading somebody may still want. The row
+ # survives, carries the rungs it covers, and is denied a verdict below.
if metric in censoring:
out[metric]["poolable"] = False
out[metric]["censoring"] = censoring[metric]
@@ -764,9 +669,8 @@ def verdict_for(
if stat["n"] > 1 and stat["spread_pct"] > abs(stat["delta_pct"]):
return f, "VOID (effect under its own scatter)"
if is_count:
- # A count is an invariant, so the sign means the opposite of what it means for a timing.
- # Less of the conversation copied is not an improvement, and calling it "faster" because
- # the number went down is exactly the kind of misreading this table exists to prevent.
+ # A count is an invariant, so the sign means the opposite of what it means for a timing: less
+ # of the conversation copied is not an improvement.
return f, ("LOST (invariant fell)" if stat["delta_pct"] < 0 else "gained")
return f, ("faster" if stat["delta_pct"] < 0 else "SLOWER")
@@ -815,19 +719,12 @@ def render(
f"different corpus is a different film. Re-run the null control."
)
# THE REST OF THE COMPARABILITY IDENTITY, which tier and corpus are only two axes of.
- #
- # `--compare` refuses two payloads whose `comparability_key`s differ, and this path -- the one
- # that actually certifies a number -- checked two of the eleven fields that key covers. So an
- # 0.1.0 floor could score an 0.2.0 payload on the same corpus and tier, and this commit is
- # precisely why that matters: it redefines `reasoning_toggle.open_ms` to terminate on a settled
- # DOM rather than on the `data-state` flip, so the older floor measures a different quantity
- # under the same name. `host`, `engine`, `cadence`, `headed`, `rungs` and the injection and
- # probe axes are the same class of difference, and `--compare` is a separate command nobody is
- # obliged to run first.
- #
- # Checked through `explain_incomparable` rather than against a second list of fields, so a
- # field added to `comparability_fields` is enforced here without being added here too. A guard
- # kept in step by hand is the drift `payload_rules` exists to argue against.
+ # `--compare` refuses two payloads whose `comparability_key`s differ, while this path, the one
+ # that certifies a number, checked two of the eleven fields, so an 0.1.0 floor could score an
+ # 0.2.0 payload: this commit redefines `reasoning_toggle.open_ms` to terminate on a settled
+ # DOM, so the older floor measures a different quantity under the same name. Checked through
+ # `explain_incomparable` rather than a second list of fields, so a field added to
+ # `comparability_fields` is enforced without being added here too.
if floor_meta is not None:
meta, conflicts = merged_meta(paths)
if meta is None:
@@ -865,41 +762,31 @@ def render(
marked = False
for metric in sorted(stats, key = lambda m: (m in METRICS, m)):
s = stats[metric]
- # TWO INDEPENDENT CAVEATS, BOTH CARRIED IN THE NAME COLUMN, and neither allowed to hide
- # the other. They answer different questions -- `(abs)` says what the delta MEANS, `[*]`
- # says which rungs it covers -- and a metric can need both at once.
- #
- # A metric censored at some rungs and measured at others is marked so the caveat cannot be
- # separated from the number when somebody copies one row out of this table into prose.
- # That is the route every withdrawn figure in this harness travelled.
+ # TWO INDEPENDENT CAVEATS, BOTH CARRIED IN THE NAME COLUMN, neither allowed to hide the other:
+ # `(abs)` says what the delta MEANS, `[*]` says which rungs it covers, and a metric can need
+ # both. A metric censored at some rungs and measured at others is marked so the caveat cannot
+ # be separated from the number when somebody copies one row into prose.
partial = s.get("poolable") is False
- # A difference row is named as one. Its delta is in the metric's own unit, and printing it
- # under a "delta %" heading would be the same class of mistake this file exists to stop.
- #
- # ORTHOGONAL TO THE TWO CENSORING MARKS BELOW, and applied first so it survives either of
- # them: `(abs)` says what the delta MEANS, `[*]` and `[f]` say who can judge it. A metric
- # can need `(abs)` and a censoring mark at once.
+ # A difference row is named as one: its delta is in the metric's own unit, so printing it under
+ # a "delta %" heading would be the same class of mistake this file exists to stop. Applied
+ # first so it survives either censoring mark.
label = metric
if s.get("difference"):
label, marked = f"{metric} (abs)", True
# THE FLOOR IS A MEASUREMENT TOO, and it is censored by the same rule on its own cells.
- #
# `summarise` marks the null control's entry `poolable = False` when the metric answered on
- # some of its cells and was censored on others, and this loop then read only the RESULT's
- # flag -- so a result measured in full was scored against a floor built from whichever
- # repetitions of the null happened to survive. Censoring is decided against a fixed budget,
- # so the repetitions it removes are the slow ones, and `spread_pct` is `max - min` over the
- # survivors: removing pairs can only narrow it. Observed on a real null control at 100K
- # plus 500K, `reasoning_toggle.close_ms` censored above 100K, the applied floor came out at
- # 17.1% from the four surviving 100K pairs while the censored repetitions of that same null
- # ran 1429-2149 ms against 498-681 ms and paired as far apart as 1.50 on identical builds.
- # A real result of -24.8% printed `faster` against the 17.1% and is not distinguishable
- # from the null's own scatter.
+ # some cells and was censored on others, and this loop read only the RESULT's flag, so a
+ # result measured in full was scored against a floor built from whichever repetitions of the
+ # null survived. Censoring removes the slow ones and `spread_pct` is `max - min` over the
+ # survivors, so it can only narrow: on a real null at 100K plus 500K,
+ # `reasoning_toggle.close_ms` gave an applied floor of 17.1% from four surviving pairs while
+ # the censored repetitions of that same null paired as far apart as 1.50 on identical builds,
+ # and a real result of -24.8% printed `faster` against it.
+ # They ran 1429-2149 ms against the survivors' 498-681 ms.
floor_stat = None if floors is None else floors.get(metric)
floor_partial = floor_stat is not None and floor_stat.get("poolable") is False
- # MARKED APART FROM `[*]`, because the two say different things. `[*]` means this row's own
- # number is not a ladder number; `[f]` means the number is fine and the null control cannot
- # judge it. Collapsing them would tell a reader to distrust a figure that is sound.
+ # MARKED APART FROM `[*]`, because the two say different things: `[*]` means this row's own
+ # number is not a ladder number, `[f]` means the number is fine and the null cannot judge it.
shown = f"{label} [*]" if partial else (f"{label} [f]" if floor_partial else label)
line = (
f" {shown:<28}{s['n']:>3}{s['base']:>11.1f}{s['treat']:>11.1f}"
@@ -907,13 +794,12 @@ def render(
)
if floors is not None:
if partial:
- # DENIED A VERDICT. Passing three gates on the rungs that could answer is not
- # passing them on the ladder the row is labelled with.
+ # DENIED A VERDICT: passing three gates on the rungs that could answer is not passing them on
+ # the ladder the row is labelled with.
line += f"{'--':>13} not pooled"
elif floor_partial:
- # DENIED A VERDICT ON THE OTHER SIDE. The three gates are all measured against
- # this floor, so a floor that is itself a survivor sample cannot certify anything,
- # however clean the result is.
+ # DENIED A VERDICT ON THE OTHER SIDE: the three gates are all measured against this floor, so a
+ # floor that is itself a survivor sample cannot certify anything.
line += f"{'--':>13} no poolable floor"
else:
f, verdict = verdict_for(s, floor_stat, is_count_metric(metric))
@@ -979,9 +865,8 @@ def main(argv: list[str] | None = None) -> int:
floor_rows = read_rows(floor_paths[0])
floor_tier = tier_of(floor_rows)
floor_corpus = corpus_of(floor_rows)
- # The floor's own comparability identity, read across every shard and every header. A null
- # whose shards disagree with each other is not one null control, so it gets the same
- # refusal a result payload gets rather than a token averaged over the disagreement.
+ # The floor's own comparability identity, read across every shard and header: a null whose
+ # shards disagree with each other is not one null control.
floor_meta, floor_conflicts = merged_meta(floor_paths)
if floor_conflicts:
print(
diff --git a/tests/studio/studiobench/sweep/parity_null_control.py b/tests/studio/studiobench/sweep/parity_null_control.py
index c5657ee047d..7a78eef49c6 100644
--- a/tests/studio/studiobench/sweep/parity_null_control.py
+++ b/tests/studio/studiobench/sweep/parity_null_control.py
@@ -43,7 +43,7 @@ if __package__ in (None, ""): # pragma: no cover
from tests.studio.studiobench.analysis import parity as P # noqa: E402
from tests.studio.studiobench.sweep.ui_parity import collect, shards_of # noqa: E402
-#: How much context to print either side of a differing run of characters.
+#:How much context to print either side of a differing run of characters.
CONTEXT = 90
diff --git a/tests/studio/studiobench/sweep/parity_shots.py b/tests/studio/studiobench/sweep/parity_shots.py
index 62c37963ed6..1f3f33dee6f 100644
--- a/tests/studio/studiobench/sweep/parity_shots.py
+++ b/tests/studio/studiobench/sweep/parity_shots.py
@@ -135,10 +135,9 @@ def differing_actions(
report's -- excused by the null control, then corroborated at the verdict's own `--min-reps`.
"""
unstable, derived, _checks = unstable_set(null_paths or None)
- # THE SAME EFFECTIVE SET THE VERDICT SCORED WITH. The verdict confines the imported
- # exemptions to what the scored runner reproduces, so reading the raw imported set here would
- # put the artifact back out of step with the job it illustrates -- an action the verdict
- # failed on would have no picture, which is the same defect as publishing none at all.
+ # THE SAME EFFECTIVE SET THE VERDICT SCORED WITH. The verdict confines the imported exemptions to
+ # what the scored runner reproduces, so reading the raw imported set here would put the artifact
+ # out of step with the job it illustrates: an action the verdict failed on would have no picture.
if derived:
unstable, _dropped = confine_to_runner(unstable, *in_arm_repeatability(result_paths))
out = []
@@ -150,9 +149,8 @@ def differing_actions(
if r["verdict"] == P.DIFFER and not is_unstable(unstable, action, cell)
]
# Carrying the DIRECTION as the fifth element, exactly as `report` does, so the artifact
- # illustrates the same set the verdict counted. Without it a direction-reversing pair would
- # be firm here and uncorroborated there, and the evidence would show a regression the job
- # did not fail on.
+ # illustrates the same set the verdict counted. Without it a direction-reversing pair would be
+ # firm here and uncorroborated there.
one_sided = [
(action, shard, cell, [r.get("reason", "")], r.get("one_sided") or None)
for action, shard, cell, r in results
@@ -160,9 +158,9 @@ def differing_actions(
and r.get("one_sided")
and not P.racy_execution(action, r.get("idle_reason") or "")
]
- # The third way `report` returns 1: the action ran on both arms and its own assertion failed
- # on one. Not filtered by the unstable set, for the same reason the verdict does not filter
- # it -- that set measures digest stability, and this is not a digest.
+ # The third way `report` returns 1: the action ran on both arms and its own assertion failed on
+ # one. Not filtered by the unstable set, for the same reason the verdict does not filter it: that
+ # set measures digest stability, and this is not a digest.
expect_bad = [
(action, shard, cell, [r.get("expect_reason", "")], r["expect_regressed"])
for action, shard, cell, r in results
@@ -263,9 +261,9 @@ def build(
missing += 1
print(f" {d['action']:<26} {d['cell']}: shot file absent on disk; not a pair")
continue
- # The shard is in the FILENAME as well as the key. Two shards produce the same
- # `<action>__<rung>_<rep>` stem, so without it the second composite silently overwrites
- # the first and the artifact shows one picture for two findings.
+ # The shard is in the FILENAME as well as the key: two shards produce the same
+ # `<action>__<rung>_<rep>` stem, so without it the second composite silently overwrites the
+ # first.
stem = f"{d['shard']}__{d['action']}__{rung}_{rep}"
ok = composite(
bp,
@@ -319,8 +317,8 @@ def prune(payload_dir: Path, shots_dir: Path) -> int:
for action, _shard, cell, r in results:
if r["verdict"] == P.MATCH:
continue
- # The prune reads one arm's own output dir, but key by the shard it came from anyway so
- # the index has one shape everywhere.
+ # The prune reads one arm's own output dir, but key by the shard it came from anyway so the index
+ # has one shape everywhere.
rung, rep = cell.split(" ", 1)
for arm in ("base", "treatment"):
got = index.get((_shard, f"{rung}.{arm}.{rep}", action, arm))
@@ -372,9 +370,9 @@ def main(argv: list[str] | None = None) -> int:
Path(args.null) if args.null else None,
Path(args.shots),
Path(args.out),
- # FORWARDED, and the omission here was the whole bug: the workflow passes --min-reps 2 to
- # match the verdict, `build` defaulted it back to 1, and the artifact then carried
- # composites of one-repetition flakes beside the change that actually failed the job.
+ # FORWARDED, and the omission here was the whole bug: the workflow passes --min-reps 2 to match the
+ # verdict, `build` defaulted it back to 1, and the artifact carried composites of one-repetition
+ # flakes beside the change that failed the job.
min_reps = args.min_reps,
)
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py
index ff9676d0978..47c1987b744 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py
@@ -45,7 +45,7 @@ from tests.studio.studiobench.runtime.types import Recorder
from tests.studio.studiobench.scoring import payload_rules
from tests.studio.studiobench.sweep import floor_table
-#: The corpus every payload here is recorded on. Real, from `outputs/rp/sbench_T_null`.
+#:The corpus every payload here is recorded on. Real, from `outputs/rp/sbench_T_null`.
CORPUS = "ac9d5d8e37be2a3844deed559fde6070247ad2322377295fb383b60b5eec5a0c"
@@ -129,9 +129,9 @@ def _write(
return out / "payload.jsonl"
-#: Real 100K `close_ms` readings. Base then treatment, `outputs/rp/sbench_T_campaign`.
+#:Real 100K `close_ms` readings. Base then treatment, `outputs/rp/sbench_T_campaign`.
RESULT_100K = ((897.6, 631.4), (665.6, 515.5), (665.6, 498.8), (682.1, 532.3))
-#: Real 100K `close_ms` readings from the null control, `outputs/rp/sbench_T_null`.
+#:Real 100K `close_ms` readings from the null control, `outputs/rp/sbench_T_null`.
NULL_100K = ((681.4, 564.7), (565.7, 515.7), (532.0, 515.1), (498.8, 498.8))
#: Real 500K `close_ms` readings from the null control, `outputs/rp/sbench_C_null`. Every one of
#: these cells is censored: the open settle blew its budget, so the action failed and the harness
@@ -235,8 +235,8 @@ def test_the_result_is_not_labelled_as_the_censored_one(tmp_path, capsys):
def test_the_same_result_scores_normally_against_a_whole_floor(tmp_path, capsys):
"""NOT A REFUSAL OF EVERYTHING. Identical result, a null control that censored nothing."""
floors = floor_table.summarise([_null(tmp_path, censored = False)])
- # The whole-ladder floor is 67.5%, which this -24.8% result does not clear, so the verdict
- # under it is VOID -- itself the point: the censored floor turned a VOID into a `faster`.
+ # The whole-ladder floor is 67.5%, which this -24.8% result does not clear, so the verdict under it
+ # is VOID, itself the point: the censored floor turned a VOID into a `faster`.
floor_table.render([_result(tmp_path)], "t", floors = floors)
row = next(
line for line in capsys.readouterr().out.splitlines() if line.strip().startswith(METRIC)
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py
index 437735c580c..5e850e50467 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py
@@ -28,8 +28,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
from tests.studio.studiobench.analysis import parity as P # noqa: E402
from tests.studio.studiobench.sweep import ui_parity as U # noqa: E402
-# parents[4] is `tests/`, which is what the sibling selftests put on sys.path; the repo root is
-# one further up and is where .github lives.
+# parents[4] is `tests/`, which the sibling selftests put on sys.path; the repo root is one
+# further up and is where .github lives.
REPO_ROOT = Path(__file__).resolve().parents[5]
WORKFLOW = REPO_ROOT / ".github/workflows/studiobench-ui-parity.yml"
@@ -91,9 +91,9 @@ def two_reps(action: str, base: str, treat: str) -> list[tuple]:
def test_a_null_control_that_decided_everything_and_found_nothing_passes(tmp_path):
- # Every action reached two observations and not one of them differed. This is the best null
- # control obtainable and the measured unstable set is empty BECAUSE there was nothing to
- # measure. Requiring a measured entry here would fail the job on its best day.
+ # Every action reached two observations and none differed: the best null control obtainable,
+ # whose measured unstable set is empty BECAUSE there was nothing to measure. Requiring a
+ # measured entry here would fail the job on its best day.
null = null_run(
tmp_path,
"quiet",
@@ -105,17 +105,14 @@ def test_a_null_control_that_decided_everything_and_found_nothing_passes(tmp_pat
assert len(report["decided"]) == 2
assert report["undecided"] == []
- # And the thing the naive gate keyed on really is absent, so this test would not have caught
- # the bug by accident: the derived set emits no `action@rung` entry at all.
unstable, _derived, _checks = U.unstable_set([null])
assert not [e for e in unstable if isinstance(e, tuple)]
def test_a_null_control_with_one_observation_each_is_undecided(tmp_path):
- # The other direction, so the audit cannot pass by never failing. --reps 1 gives one
- # observation per (rung, action); `derive_unstable` needs two, so nothing is decided, the
- # measured set is empty for a completely different reason, and the result would be scored
- # against the declared list.
+ # The other direction, so the audit cannot pass by never failing: --reps 1 gives one
+ # observation per (rung, action) where `derive_unstable` needs two, so the measured set is
+ # empty for a completely different reason.
null = null_run(
tmp_path,
"thin",
@@ -131,9 +128,8 @@ def test_a_null_control_with_one_observation_each_is_undecided(tmp_path):
def test_the_two_empty_measured_sets_are_told_apart(tmp_path):
- # Both of the above derive an EMPTY measured set. The whole point of the audit is that one is
- # the best possible reading and the other is a broken one, so assert they are distinguished
- # rather than merely that each has the right code.
+ # Both of the above derive an EMPTY measured set, so assert the audit distinguishes the best
+ # possible reading from a broken one, not merely that each has the right code.
quiet = null_run(tmp_path, "q2", two_reps("settings", "SAME", "SAME"))
thin = null_run(tmp_path, "t2", [("r100K", "rep0", "settings", "SAME", "SAME")])
for path in (quiet, thin):
@@ -167,9 +163,9 @@ def test_a_noisy_null_control_is_decided_and_reports_what_differed(tmp_path):
def test_an_action_that_never_captured_is_undecided_not_stable(tmp_path):
- # A failed capture is blind. `derive_unstable` refuses to count it as an observation, because
- # an action derived as stable from pairs that never rendered would be trusted forever on the
- # strength of nothing. So it must read as UNDECIDED, never as a quiet pass.
+ # A failed capture is blind, and `derive_unstable` refuses to count it: an action derived as
+ # stable from pairs that never rendered would be trusted forever on nothing. So it must read
+ # UNDECIDED, never a quiet pass.
null = null_run(tmp_path, "blind", two_reps("image_upload", None, None))
rc, report = U.audit_null([null])
assert rc == 1
@@ -177,9 +173,8 @@ def test_an_action_that_never_captured_is_undecided_not_stable(tmp_path):
def test_an_excused_action_is_allowed_but_still_needs_a_decided_one(tmp_path):
- # image_upload is permanently blind on this fixture, so it is excusable. What is NOT excusable
- # is a run in which everything is excused: that measured nothing at all while naming a reason
- # for each blank, and passing it would let the excuse list grow until the audit is vacuous.
+ # image_upload is permanently blind on this fixture, so it is excusable; a run in which
+ # EVERYTHING is excused measured nothing at all while naming a reason for each blank.
only_excused = null_run(tmp_path, "allblind", two_reps("image_upload", None, None))
rc, report = U.audit_null([only_excused], frozenset({"image_upload"}))
assert rc == 1
@@ -232,10 +227,9 @@ def test_the_cli_audits_a_single_repetition_as_a_failure(tmp_path, capsys):
def test_the_parity_workflow_audits_the_null_with_the_tool(tmp_path):
- # Twice now a guard written as a regex over this tool's own printed prose has been wrong: once
- # because a real payload spells its rung `r100K` where a fixture spells it `100K`, and once
- # because the literal `action@rung` appears in the explanatory text whether or not anything
- # was measured. The check belongs in the tool, where it is reachable by the tests above.
+ # Twice now a guard written as a regex over this tool's printed prose has been wrong: a real
+ # payload spells its rung `r100K` where a fixture spells it `100K`, and the literal
+ # `action@rung` appears in the explanatory text whether or not anything was measured.
text = WORKFLOW.read_text(encoding = "utf-8")
assert "--audit-null" in text
assert "--allow-undecided image_upload" in text
@@ -247,8 +241,6 @@ def test_the_parity_workflow_audits_the_null_with_the_tool(tmp_path):
def test_a_difference_in_one_repetition_of_two_is_not_counted(tmp_path):
- # The measured false alarm, in miniature. A build renders the same way every time, so a stable
- # action that differs in rep0 and matches in rep1 is telling you about the run.
mine = null_run(
tmp_path,
"one_rep",
@@ -264,7 +256,7 @@ def test_a_difference_in_one_repetition_of_two_is_not_counted(tmp_path):
def test_a_difference_in_every_repetition_still_fails(tmp_path):
# The other direction, and the one that matters more: an injected element differs on every
# pass, so raising the bar must not make the gate unable to fail. Measured on the real probe
- # -- one <span> added inside the thread root -- which stays red at --min-reps 2.
+ # (one <span> added inside the thread root), which stays red at --min-reps 2.
mine = null_run(
tmp_path,
"both_reps",
@@ -277,8 +269,6 @@ def test_a_difference_in_every_repetition_still_fails(tmp_path):
def test_the_headline_count_is_the_one_the_exit_code_uses(tmp_path, capsys):
- # Printing "stable actions differing: 1" above a verdict of 0 is how a reader decides the
- # tool is lying to them.
mine = null_run(
tmp_path,
"hdr",
@@ -296,7 +286,7 @@ def test_the_headline_count_is_the_one_the_exit_code_uses(tmp_path, capsys):
def test_one_repetition_seen_twice_is_not_two_observations(tmp_path):
# Two shards can carry the same (rung, rep); corroboration counts DISTINCT repetitions, or a
- # single flake recorded in two places would corroborate itself.
+ # single flake recorded twice would corroborate itself.
a = null_run(tmp_path, "sh1", [("r100K", "rep0", "settings", "A", "B")])
b = null_run(tmp_path, "sh2", [("r100K", "rep0", "settings", "A", "B")])
assert U.report([a, b], "t", frozenset(), min_reps = 2) == 0
@@ -326,14 +316,11 @@ def run_with_completion(tmp_path: Path, name: str, specs: list[tuple]) -> Path:
def test_a_null_control_ignores_a_cell_that_never_completed(tmp_path):
- # The half-written film. Its rows look complete and pair like real ones, and counting them
- # narrows the excuse set, which is how a truncated null manufactures false alarms downstream.
null = run_with_completion(
tmp_path,
"truncated",
[("rep0", "A", "B", True), ("rep1", "A", "B", False)],
)
- # Only the completed repetition is an observation, so one observation, so undecided.
rc, report = U.audit_null([null])
assert rc == 1
assert report["decided"] == []
@@ -351,10 +338,10 @@ def test_the_same_two_repetitions_decide_it_once_both_cells_completed(tmp_path):
def test_a_result_still_reports_a_difference_from_a_cell_that_died(tmp_path):
- # The OTHER sign, and the one that matters more. A cell that died is the latest attempt at
- # itself and the difference it saw is real; dropping it would silence a regression, which is
- # the worse direction. `test_an_attempt_that_was_never_re_run_still_carries_its_parity_
- # verdict` holds the same line from the other file.
+ # The OTHER sign, and the one that matters more: a cell that died is the latest attempt at
+ # itself and the difference it saw is real, so dropping it would silence a regression.
+ # test_an_attempt_that_was_never_re_run_still_carries_its_parity_verdict holds the same line
+ # from the other file.
mine = run_with_completion(
tmp_path,
"died",
@@ -365,8 +352,6 @@ def test_a_result_still_reports_a_difference_from_a_cell_that_died(tmp_path):
def test_a_payload_with_no_cell_rows_says_it_could_not_check(tmp_path, capsys):
- # Falling back to the old behaviour is right; falling back silently is how a guard stops
- # guarding without anybody noticing.
old = null_run(tmp_path, "nocells", two_reps("settings", "A", "B"))
U.report([old], "t", frozenset())
assert "NOT GUARDED" in capsys.readouterr().out
@@ -390,15 +375,14 @@ def corpus_run(tmp_path: Path, name: str, corpus: str, rep: str) -> Path:
def test_two_corpora_are_refused_before_their_observations_pool(tmp_path):
# Each film is one repetition and can decide nothing alone. Pooled they satisfy
- # min_observations and report DECIDED from two different threads, and the result is then
- # scored against a set never measured on the corpus it is applied to.
+ # min_observations and report DECIDED from two different threads, so the result is scored
+ # against a set never measured on the corpus it is applied to.
a = corpus_run(tmp_path, "c1", "corpusAAAA", "rep0")
b = corpus_run(tmp_path, "c2", "corpusBBBB", "rep1")
with pytest.raises(SystemExit) as got:
U.one_corpus([a, b], "null control")
assert "more than one corpus" in str(got.value)
- # And the refusal is load-bearing: without it the two DO pool into a decision.
rc, report = U.audit_null([a, b])
assert rc == 0 and report["differed"] == [("r100K", "settings")]
@@ -425,9 +409,8 @@ def not_run_row(cid: str, action: str) -> dict:
def test_a_missed_slot_on_both_arms_cannot_turn_a_clean_verdict_red(tmp_path):
- # The measured claim, in miniature. Over 18 mutations of a real payload at 2 to 24 missed
- # slots, the verdict never gained a red; NOT_EXERCISED is filed under coverage, never under
- # the stable differences the exit code is taken from.
+ # The measured claim in miniature: over 18 mutations of a real payload at 2 to 24 missed slots
+ # the verdict never gained a red, because NOT_EXERCISED is filed under coverage.
rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
for arm in ("base", "treatment"):
@@ -443,8 +426,6 @@ def test_a_missed_slot_on_both_arms_cannot_turn_a_clean_verdict_red(tmp_path):
def test_a_run_that_compared_almost_nothing_does_not_pass(tmp_path, capsys):
- # The other half, and the reason the slot gate could not simply be deleted. Two blank pages
- # have identical digests, so a film that ran nothing is the easiest possible pass.
rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
for arm in ("base", "treatment"):
@@ -466,7 +447,7 @@ def test_a_run_that_compared_almost_nothing_does_not_pass(tmp_path, capsys):
def test_the_audit_ignores_an_action_the_result_never_compared(tmp_path):
# `copy_markdown` is undecided in the null and nothing on the result turns on its verdict, so
# demanding one turns machine speed into a red build. The result DOES need an excuse for
- # `settings`, so the scope is not empty and the audit is not passing vacuously.
+ # `settings`, so the audit is not passing vacuously.
null = null_run(
tmp_path,
"n",
@@ -474,15 +455,13 @@ def test_the_audit_ignores_an_action_the_result_never_compared(tmp_path):
)
result = null_run(tmp_path, "r", two_reps("settings", "AAA", "BBB"))
- assert U.audit_null([null], frozenset())[0] == 1 # unscoped: copy_markdown undecided
+ assert U.audit_null([null], frozenset())[0] == 1
scope = U.actions_needing_an_excuse([result], min_reps = 2)
assert scope == {("r100K", "settings")}
- assert U.audit_null([null], frozenset(), scope)[0] == 0 # scoped: nobody needed it
+ assert U.audit_null([null], frozenset(), scope)[0] == 0
def test_the_audit_still_fails_on_an_action_the_result_did_compare(tmp_path):
- # The other direction, so scoping cannot pass by excusing everything: an action the result
- # put a verdict on, that the null could not decide, is exactly the unscored case.
null = null_run(
tmp_path,
"n2",
@@ -502,30 +481,24 @@ def test_the_audit_still_fails_on_an_action_the_result_did_compare(tmp_path):
def test_a_null_from_another_tier_is_refused_rather_than_applied(tmp_path):
- # fast and standard both walk 100K, so a fast-film race lands on exactly the rung key a
- # standard-film regression would be scored under. The old code warned and scored anyway.
assert U.cross_side_mismatch({"standard"}, {"fast"}, {"c1"}, {"c1"}).startswith(
"the null control was recorded at tier"
)
def test_a_null_from_another_corpus_is_refused_rather_than_applied(tmp_path):
- # The likelier of the two in practice: a corpus revision lands, the result is re-recorded,
- # and last week's null control is still sitting in the directory the workflow globs.
got = U.cross_side_mismatch({"fast"}, {"fast"}, {"c2"}, {"c1"})
assert got.startswith("the null control was recorded at corpus")
def test_matching_sides_and_unrecorded_sides_are_both_allowed():
- # Equal is comparable, and an empty set means the recorder predates the field, so there is
- # nothing to disagree about. Refusing those would reject every legacy payload.
assert U.cross_side_mismatch({"fast"}, {"fast"}, {"c1"}, {"c1"}) == ""
assert U.cross_side_mismatch({"fast"}, set(), {"c1"}, set()) == ""
assert U.cross_side_mismatch(set(), {"fast"}, set(), {"c1"}) == ""
def test_the_refusal_exits_two_not_one_through_the_cli(tmp_path, monkeypatch, capsys):
- # Exit 2 is the tool declining to answer. Exit 1 would read as a parity failure and send
+ # Exit 2 is the tool declining to answer; exit 1 would read as a parity failure and send
# somebody hunting for a UI change that was never measured.
null = null_run(tmp_path, "tier_null", two_reps("settings", "SAME", "SAME"), tier = "fast")
result = null_run(tmp_path, "tier_res", two_reps("settings", "SAME", "SAME"), tier = "standard")
@@ -578,11 +551,10 @@ def one_sided_payload(
def test_an_action_the_head_can_no_longer_perform_fails_the_verdict(tmp_path, capsys):
- # THE HOLE THIS CLOSES. A control that stops opening is the one user-visible regression that
- # leaves no digest to differ: the head arm records `ran: false`, the pair is NOT_EXERCISED,
- # and filing that under coverage meant a button that no longer opens shipped green with 64 of
- # 68 pairs still compared -- comfortably above the workflow's --min-compared 16, so the
- # coverage floor could never catch it either.
+ # THE HOLE THIS CLOSES: a control that stops opening is the one user-visible regression that
+ # leaves no digest to differ, and filing NOT_EXERCISED under coverage meant a button that no
+ # longer opens shipped green with 64 of 68 pairs still compared, comfortably above the
+ # workflow's --min-compared 16.
path = one_sided_payload(tmp_path, "regressed", "settings", ("rep0", "rep1"))
assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 1
printed = capsys.readouterr().out
@@ -591,10 +563,9 @@ def test_an_action_the_head_can_no_longer_perform_fails_the_verdict(tmp_path, ca
def test_one_arms_missed_slot_in_one_repetition_is_still_only_a_warning(tmp_path, capsys):
- # The other half, and the reason this is not simply failed on sight. A contended runner can
- # lose one arm's slot once, and both arms are driven from one script in one session, so the
- # loss is not always symmetric. Held to the SAME corroboration bar as a differing digest: a
- # build that cannot open a control cannot open it on either pass.
+ # The other half, and why this is not failed on sight: a contended runner can lose one arm's
+ # slot once, so it is held to the SAME corroboration bar as a differing digest. A build that
+ # cannot open a control cannot open it on either pass.
path = one_sided_payload(tmp_path, "flake", "settings", ("rep0",))
assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 0
assert "UNCORROBORATED one-arm-only" in capsys.readouterr().out
@@ -602,8 +573,7 @@ def test_one_arms_missed_slot_in_one_repetition_is_still_only_a_warning(tmp_path
def test_an_action_expected_to_vary_is_not_failed_for_reaching_one_arm_only(tmp_path, capsys):
# `stop_generation` runs only while a stream is live, so which arm reached it before the
- # stream ended is the same race its UNSTABLE_ACTIONS entry already describes. Scoring that as
- # a build difference would red the job on stream timing.
+ # stream ended is the race its UNSTABLE_ACTIONS entry already describes.
path = one_sided_payload(
tmp_path,
"racy",
@@ -616,8 +586,6 @@ def test_an_action_expected_to_vary_is_not_failed_for_reaching_one_arm_only(tmp_
def test_both_arms_missing_the_same_action_is_coverage_and_not_a_verdict(tmp_path):
- # Unchanged, and pinned so the fix above cannot creep into the missed-slot case the mutation
- # study measured: NEITHER arm running an action is lost coverage in both builds.
rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
for arm in ("base", "treatment"):
@@ -636,10 +604,9 @@ def test_both_arms_missing_the_same_action_is_coverage_and_not_a_verdict(tmp_pat
def test_an_undecided_action_the_result_only_matched_does_not_fail_the_audit(tmp_path):
- # THE CI FAILURE THIS FIXES. The null lost one arm's slot on `settings`, so it has a single
- # observation and cannot decide it. The result MATCHED on `settings` in both repetitions, and
- # no entry in the unstable set can turn a match into anything, so the null was never going to
- # be asked. Requiring an opinion here is what made the gate unsatisfiable on a shared runner.
+ # THE CI FAILURE THIS FIXES: the null lost one arm's slot on `settings` and cannot decide it,
+ # while the result MATCHED on `settings` in both repetitions, so the null was never going to
+ # be asked. Requiring an opinion made the gate unsatisfiable on a shared runner.
result = null_run(tmp_path, "result", two_reps("settings", "SAME", "SAME"))
null = null_run(tmp_path, "null", [("r100K", "rep0", "settings", "SAME", "SAME")])
scope = U.actions_needing_an_excuse([result], min_reps = 2)
@@ -649,10 +616,9 @@ def test_an_undecided_action_the_result_only_matched_does_not_fail_the_audit(tmp
def test_the_audit_still_fails_on_an_undecided_action_the_result_kept_differing_on(tmp_path):
- # THE SAFETY DIRECTION, and the whole reason the scope is not simply dropped. `settings`
- # differed in BOTH repetitions, so it is scored as a regression unless the unstable set
- # excuses it -- and an undecided null excuses it from the DECLARED list, unmeasured, which is
- # the fiction this job exists to replace. Narrowing the scope must not reach this case.
+ # THE SAFETY DIRECTION, and why the scope is not simply dropped: `settings` differing in BOTH
+ # repetitions is scored as a regression unless the unstable set excuses it, and an undecided
+ # null excuses it from the DECLARED list, unmeasured.
result = null_run(tmp_path, "result", two_reps("settings", "AAA", "BBB"))
null = null_run(tmp_path, "null", [("r100K", "rep0", "settings", "SAME", "SAME")])
scope = U.actions_needing_an_excuse([result], min_reps = 2)
@@ -663,8 +629,6 @@ def test_the_audit_still_fails_on_an_undecided_action_the_result_kept_differing_
def test_a_difference_in_one_repetition_only_needs_no_excuse(tmp_path):
- # `corroborated` already reports this without counting it, so no excuse can move it and the
- # scope is threaded with the verdict's own --min-reps rather than a number of its own.
result = null_run(
tmp_path,
"result",
@@ -677,9 +641,9 @@ def test_a_difference_in_one_repetition_only_needs_no_excuse(tmp_path):
def test_a_corroborated_one_arm_only_action_still_needs_the_null_to_decide_it(tmp_path):
- # The other shape the unstable set is load bearing for: `report` excuses a one-arm-only action
+ # The other shape the unstable set is load-bearing for: `report` excuses a one-arm-only action
# when it is in the unstable set, so an undecided null hands out that excuse on the declared
- # list alone. Scoped in for exactly the same reason a corroborated difference is.
+ # list alone.
result = one_sided_payload(tmp_path, "result", "settings", ("rep0", "rep1"))
null = null_run(tmp_path, "null", [("r100K", "rep0", "settings", "SAME", "SAME")])
scope = U.actions_needing_an_excuse([result], min_reps = 2)
@@ -722,30 +686,24 @@ def missed_slot_payload(tmp_path: Path, name: str, action: str, reps: tuple[str,
def test_one_arm_missing_a_slot_in_every_repetition_is_still_not_a_build_difference(tmp_path):
- # MEASURED, not reasoned: this is the run that went red after the one-arm-only gate landed.
- # `settings` missed a 1200ms budget on the BASE arm at 45700ms in both repetitions, by 1292ms
- # and 1352ms. The schedule is fixed, so a machine slow enough to miss that slot once is slow
- # enough to miss it twice -- the two misses are correlated through the runner and corroboration
- # cannot separate them from a build difference. `slot_missed` can, and the driver already
- # records it. Note which arm it was, too: the BASE arm went idle, so reading it as a
- # regression would have blamed the head build for the old one's timing.
+ # MEASURED, not reasoned: `settings` missed a 1200ms budget on the BASE arm at 45700ms in both
+ # repetitions, by 1292ms and 1352ms. The schedule is fixed, so the two misses are correlated
+ # through the runner and corroboration cannot separate them from a build difference, while
+ # `slot_missed` can. Note which arm too: reading it as a regression would have blamed the head
+ # build for the old one's timing.
path = missed_slot_payload(tmp_path, "slow", "settings", ("rep0", "rep1"))
assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 0
def test_a_control_that_cannot_open_is_still_caught_when_a_slot_is_also_missed_elsewhere(tmp_path):
- # The discriminator must not become a blanket excuse: a genuine precondition failure still
- # fails even though the same run also lost slots.
path = one_sided_payload(tmp_path, "broken", "settings", ("rep0", "rep1"))
assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 1
def test_a_difference_at_one_rung_does_not_demand_the_null_decide_another(tmp_path):
- # The scope is keyed by (rung, action) and not by action alone. Reduced to bare names it
- # re-widened across the ladder: a corroborated difference at 1K pulled the action into scope
- # at EVERY rung, so one missed observation at 100K -- where the result matched and no excuse
- # was ever consulted -- failed the audit again. Instability is a property of the rung, which
- # is why `derive_unstable` decides per rung and `unstable_set` measures per rung.
+ # The scope is keyed by (rung, action), not action alone: reduced to bare names a corroborated
+ # difference at 1K pulled the action into scope at EVERY rung, so one missed observation at
+ # 100K failed the audit again. Instability is a property of the rung.
result = null_run(
tmp_path,
"result",
@@ -756,7 +714,6 @@ def test_a_difference_at_one_rung_does_not_demand_the_null_decide_another(tmp_pa
tmp_path,
"null",
[("r1K", "rep0", "keystroke", "A", "B"), ("r1K", "rep1", "keystroke", "A", "B")]
- # One observation only at 100K, so the null cannot decide it there.
+ [("r100K", "rep0", "keystroke", "S", "S")],
)
scope = U.actions_needing_an_excuse([result], min_reps = 2)
@@ -786,22 +743,18 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_
vary" while the command exits 0.
"""
rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
- # rep0 is a clean, uncontested observation and it MATCHES.
for arm in ("base", "treatment"):
cid = f"r100K.{arm}.rep0"
row = action_row(cid, "settings", "SAME")
row["session_id"] = "s1"
rows.append(row)
rows.append({"row_type": "cell", "cell_id": cid, "completed": True, "session_id": "s1"})
- # rep1 completed in session s1 ...
for arm in ("base", "treatment"):
cid = f"r100K.{arm}.rep1"
row = action_row(cid, "settings", "SAME")
row["session_id"] = "s1"
rows.append(row)
rows.append({"row_type": "cell", "cell_id": cid, "completed": True, "session_id": "s1"})
- # ... and was re-run whole in s2, where the treatment arm died before its cell row. Its
- # digests DIFFER, which is what would make the action look unstable.
for arm, digest in (("base", "SAME"), ("treatment", "OTHER")):
row = action_row(f"r100K.{arm}.rep1", "settings", digest)
row["session_id"] = "s2"
@@ -819,7 +772,6 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_
assert got["incomplete"] == 1, got
keys = {(k[2], k[4]) for k in got["pairs"]}
assert ("rep0", "settings") in keys
- # The dead retry left an unpaired base row; it must not have become an observation.
assert not any(
"base" in sides and "treatment" in sides
for k, sides in got["pairs"].items()
@@ -830,15 +782,13 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_
assert ("r100K", "settings") not in unstable, unstable
-# ── the audit has to run in the modes that ask for it ────────────────
-#
-# The audit block sat below the structural early return while reading nothing from the plan, so a
-# payload with no structural entries returned first and the audit never ran. `--mode visible` and
-# `--mode behaviour` hard-set `structural` to an empty set for every pattern, so those two could
-# never audit at all: the command exited 0 out of the ordinary visible report having silently
-# skipped the option whose promise is to FAIL unless the null decided what the result needs.
+# The audit block sat below the structural early return while reading nothing from the plan,
+# so a payload with no structural entries returned first and it never ran; and `--mode
+# visible` / `--mode behaviour` hard-set `structural` to empty, so those two could never audit
+# at all.
+# ── the audit has to run in the modes that ask for it ────────────────
def test_the_cli_audits_a_single_repetition_under_forced_visible_mode(tmp_path, capsys):
"""THE DEFECT. Same payload and same expectation as the auto-mode failure test above, with
`--mode visible` added: the audit must still run and still fail."""
@@ -884,7 +834,6 @@ def _arm_payload(tmp_path: Path, name: str, regress: str | None, self_race: str
if act == regress and arm == "treatment":
digest = "HEAD_IS_DIFFERENT"
if act == self_race:
- # Racing against itself: side A does not even reproduce between repetitions.
digest = f"RACE_{arm}_{rep}"
rows.append(action_row(cid, act, digest))
rows.append({"row_type": "cell", "cell_id": cid, "completed": True})
@@ -904,8 +853,6 @@ def test_the_other_runners_race_does_not_excuse_a_regression_on_this_one(tmp_pat
it was excused solely by a race on a machine the result never touched -- and a corroborated
head regression there would have shipped green.
"""
- # The null raced on reasoning_toggle. The result's own runner did not: side A renders the
- # same DOM in both repetitions, and head renders a different one in both.
null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")
result = _arm_payload(tmp_path, "result", regress = "reasoning_toggle", self_race = None)
@@ -913,10 +860,8 @@ def test_the_other_runners_race_does_not_excuse_a_regression_on_this_one(tmp_pat
assert ("r100K", "reasoning_toggle") in imported, imported
assert derived
- # Scored against the imported set as it stands, the regression is excused and the job is GREEN.
assert U.report([result], "imported", imported, min_reps = 2, min_compared = 16) == 0
- # Confined to the runner being scored, the exemption does not survive and the job is RED.
local_unstable, local_stable = U.in_arm_repeatability([result])
assert ("r100K", "reasoning_toggle") in local_stable
assert ("r100K", "reasoning_toggle") not in local_unstable
@@ -927,8 +872,6 @@ def test_the_other_runners_race_does_not_excuse_a_regression_on_this_one(tmp_pat
def test_an_exemption_this_runner_reproduces_is_kept(tmp_path):
- # The other direction, and the one that keeps the gate usable. When the scored runner races on
- # the same action, the exemption is doing real work and removing it would red a sound run.
null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")
result = _arm_payload(tmp_path, "result", regress = None, self_race = "reasoning_toggle")
imported, _derived, _ = U.unstable_set([null])
@@ -938,9 +881,8 @@ def test_an_exemption_this_runner_reproduces_is_kept(tmp_path):
def test_an_action_this_runner_could_not_decide_keeps_its_exemption(tmp_path):
- # UNDECIDED IS NOT STABLE. One repetition of side A is one observation, and reading it as
- # "this runner says the action is repeatable" would turn a lost slot into a red job -- the
- # exact direction the false-alarm data says actually happens.
+ # UNDECIDED IS NOT STABLE: one repetition of side A is one observation, and reading it as
+ # "this runner says the action is repeatable" would turn a lost slot into a red job.
null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")
rows = [
json.loads(line)
@@ -948,7 +890,6 @@ def test_an_action_this_runner_could_not_decide_keeps_its_exemption(tmp_path):
.read_text(encoding = "utf-8")
.splitlines()
]
- # Drop side A's reasoning_toggle in rep1, leaving a single observation of it on this runner.
rows = [
r
for r in rows
@@ -974,7 +915,7 @@ def test_an_action_this_runner_could_not_decide_keeps_its_exemption(tmp_path):
def test_a_declared_exemption_is_never_dropped_by_a_runner_measurement(tmp_path):
# The declared list is a standing claim about the app, not a measurement of a machine, so a
- # machine cannot contradict it. Only the `(rung, action)` entries are runner-derived.
+ # machine cannot contradict it; only the `(rung, action)` entries are runner-derived.
null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")
result = _arm_payload(tmp_path, "result", regress = None, self_race = None)
imported, _derived, _ = U.unstable_set([null])
@@ -983,9 +924,8 @@ def test_a_declared_exemption_is_never_dropped_by_a_runner_measurement(tmp_path)
def test_the_verdict_confines_the_imported_set_when_driven_through_main(tmp_path, capsys):
- # Driven through main() rather than report(), because the failure this guards against is not
- # a wrong confinement, it is a correct one that never reaches the verdict. That exact shape --
- # a value computed or parsed and then not forwarded -- has already shipped once in this file's
+ # Driven through main() rather than report(), because the failure guarded against is a correct
+ # confinement that never reaches the verdict: the same shape that shipped once in this file's
# neighbour, where --min-reps was parsed and never passed to build().
null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")
result = _arm_payload(tmp_path, "result", regress = "reasoning_toggle", self_race = None)
@@ -1053,7 +993,6 @@ def test_a_control_that_stopped_working_is_not_excused_by_its_digest_exemption(t
The assertion is not a digest, so the digest exemption does not reach it.
"""
path = _expect_payload(tmp_path, "regressed", "stop_generation", failed_on = "treatment")
- # Scored with stop_generation declared unstable, exactly as the gate scores it.
assert "stop_generation" in U.UNSTABLE_ACTIONS
assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1
printed = capsys.readouterr().out
@@ -1062,22 +1001,20 @@ def test_a_control_that_stopped_working_is_not_excused_by_its_digest_exemption(t
def test_both_arms_failing_the_assertion_is_lost_coverage_not_a_difference(tmp_path):
- # The fixture cannot reach the state on either build. That is worth knowing and it is not a
- # statement about the change under review, so it must not red the job.
path = _expect_payload(tmp_path, "both", "stop_generation", failed_on = None, both = True)
assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 0
def test_an_action_that_asserts_nothing_is_not_an_assertion_failure(tmp_path):
# `expect_ok is None` is "this action makes no claim", which every payload recorded before the
- # field existed also carries. Reading None as False would red every one of them.
+ # field existed also carries, so reading None as False would red every one of them.
path = _expect_payload(
tmp_path, "none", "stop_generation", failed_on = None, expect_ok_value = None
)
assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 0
- # Asserted on the signal as well as the exit code: read through the exit code alone this case
- # is indistinguishable from the both-arms one, so a mutation that turns None into a failure
- # would still pass here. `expect_regressed` is where the distinction actually lives.
+ # Asserted on the signal as well as the exit code: through the exit code alone this case is
+ # indistinguishable from the both-arms one, and `expect_regressed` is where the distinction
+ # lives.
row = {
"ran": True,
"expect_ok": None,
@@ -1097,8 +1034,6 @@ def test_an_action_that_asserts_nothing_is_not_an_assertion_failure(tmp_path):
def test_an_assertion_that_failed_in_one_repetition_of_two_is_not_counted(tmp_path, capsys):
- # Held to the same corroboration bar as everything else: a build that cannot stop generation
- # cannot stop it on either pass.
path = _expect_payload(
tmp_path, "flake", "stop_generation", failed_on = "treatment", reps = ("rep0",)
)
@@ -1153,17 +1088,16 @@ def test_an_assertion_that_blames_a_different_arm_each_time_is_not_corroborated(
def test_an_assertion_that_blames_the_same_arm_twice_still_fails_the_job(tmp_path):
- # The other side of the same key, and the reason the fix is not just "require more". A build
- # that consistently fails the assertion is exactly what this category exists to catch, and it
- # must survive the direction keying.
+ # The other side of the same key, and why the fix is not just "require more": a build that
+ # consistently fails the assertion is what this category exists to catch and must survive the
+ # direction keying.
path = _expect_payload(tmp_path, "consistent", "stop_generation", failed_on = "treatment")
assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1
def test_one_arm_only_that_swaps_arms_between_repetitions_is_not_corroborated(tmp_path, capsys):
- # The same defect on the one-arm-only category, which is the one the item was filed against.
- # A precondition race that stops the treatment arm performing the action in rep0 and the base
- # arm in rep1 says nothing about either build.
+ # The same defect on the one-arm-only category: a precondition race stopping the treatment arm
+ # in rep0 and the base arm in rep1 says nothing about either build.
rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
for arm in ("base", "treatment"):
@@ -1231,15 +1165,12 @@ def test_a_scoped_action_with_no_rows_in_the_null_is_undecided_not_absent(tmp_pa
assert report_["decided"] == [("r100K", "settings")]
assert rc == 1, "an action the null never measured cannot license an excuse"
- # The reason it matters: send_turn carries no measured entry and is excused by name anyway.
unstable, _derived, _checks = U.unstable_set([null])
assert ("r100K", "send_turn") not in unstable
assert U.is_unstable(unstable, "send_turn", "r100K rep0")
def test_a_missing_scoped_action_on_the_waived_list_is_still_waived(tmp_path):
- # `image_upload` has no attachments button on this fixture, and the workflow waives it by
- # name. Whether it produced rows or none at all, the waiver is the same statement.
null = _scope_payload(tmp_path, "null", ("settings",))
result = _scope_payload(tmp_path, "result", ("settings", "image_upload"))
rc, report_ = U.audit_null(
@@ -1252,8 +1183,6 @@ def test_a_missing_scoped_action_on_the_waived_list_is_still_waived(tmp_path):
def test_an_action_outside_the_scope_is_not_required_to_exist(tmp_path):
- # The reconciliation is against the SCOPE, not against the schedule. An action the result
- # matched on needs no excuse, so the null owing it nothing is not a hole.
null = _scope_payload(tmp_path, "null", ("settings",))
result = _scope_payload(tmp_path, "result", ("settings",))
rc, report_ = U.audit_null([null], frozenset(), U.actions_needing_an_excuse([result], 2))
@@ -1281,8 +1210,8 @@ def test_a_broken_control_is_not_excused_because_its_digest_varies(tmp_path, cap
def test_an_action_with_no_not_run_path_is_never_exempt_from_one_arm_only(tmp_path):
- # `scroll_after` has no `not_run` in `scene/actions.py` at all, so a `ran: false` for it
- # cannot be a race under any reading, yet the digest list exempted it.
+ # `scroll_after` has no `not_run` in `scene/actions.py` at all, so a `ran: false` for it cannot
+ # be a race under any reading, yet the digest list exempted it.
path = one_sided_payload(tmp_path, "scroll", "scroll_after", ("rep0", "rep1"))
assert "scroll_after" in U.UNSTABLE_ACTIONS
assert "scroll_after" not in P.RACY_EXECUTION
@@ -1290,15 +1219,13 @@ def test_an_action_with_no_not_run_path_is_never_exempt_from_one_arm_only(tmp_pa
def test_every_racy_execution_entry_states_its_mechanism():
- # The same bar the digest list is held to. An exemption without a stated mechanism is how the
- # nine-action version of this list survived unexamined, and each of these has to name the
- # `not_run` it is excusing.
+ # The same bar the digest list is held to: an exemption without a stated mechanism is how the
+ # nine-action version survived unexamined, and the markers are the operative half, since an
+ # entry matching nothing, or everything, is the failure this keying prevents.
for action, (why, markers) in P.RACY_EXECUTION.items():
assert action in P.UNSTABLE_ACTIONS, action
assert "not_run" in why or "not run" in why, action
assert len(why) > 60, action
- # And the markers are the operative half: an entry that documents a mechanism but matches
- # nothing, or matches everything, is the failure this keying exists to prevent.
assert markers and all(len(m) > 10 for m in markers), action
@@ -1319,16 +1246,14 @@ def test_a_racy_execution_action_is_not_put_in_the_audit_scope(tmp_path):
reason = "nothing was generating and a new turn did not start within 8s",
)
assert "stop_generation" in P.RACY_EXECUTION
- # The verdict does not count it.
assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 0
- # So it must not be in the scope either.
assert ("r100K", "stop_generation") not in U.actions_needing_an_excuse([path], 2)
def test_a_direction_reversing_one_sided_pair_is_not_put_in_the_audit_scope(tmp_path):
- # Same rule, the other axis. `report` keys corroboration on the live arm, so a pair blaming
- # opposite arms is UNCORROBORATED and cannot move the verdict; built here without the
- # direction it corroborated, entered the scope, and an undecided null failed a passing job.
+ # Same rule, the other axis: `report` keys corroboration on the live arm, so a pair blaming
+ # opposite arms cannot move the verdict, yet built without the direction it entered the scope
+ # and an undecided null failed a passing job.
rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
for arm in ("base", "treatment"):
@@ -1352,8 +1277,6 @@ def test_a_direction_reversing_one_sided_pair_is_not_put_in_the_audit_scope(tmp_
def test_a_control_the_head_cannot_open_is_still_in_the_audit_scope(tmp_path):
- # The other side, so the two fixes above cannot quietly empty the scope: an action the
- # verdict DOES fail on is exactly what the null has to have decided.
path = one_sided_payload(tmp_path, "real", "settings", ("rep0", "rep1"))
assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 1
assert ("r100K", "settings") in U.actions_needing_an_excuse([path], 2)
@@ -1376,13 +1299,10 @@ def test_a_removed_stop_button_is_not_exempt_just_because_stop_generation_can_ra
)
assert not P.racy_execution("stop_generation", "the stop button is not present")
assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1
- # And the null is asked about it, because the verdict now turns on it.
assert ("r100K", "stop_generation") in U.actions_needing_an_excuse([path], 2)
def test_a_missing_composer_is_not_exempt_just_because_send_turn_can_be_queued(tmp_path):
- # The same for send_turn: "a reply was still streaming" is the previous turn overrunning,
- # "no composer on the page" is the build having removed the composer.
racy = one_sided_payload(
tmp_path,
"queued",
@@ -1398,28 +1318,18 @@ def test_a_missing_composer_is_not_exempt_just_because_send_turn_can_be_queued(t
assert U.report([gone], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1
-# ── the reply that was still arriving is not a blind capture ──────────
-#
-# WHAT TURNED RED. A wave of `studiobench UI parity` audited UNDECIDED with 13 decided, 0 differed
-# and 2 undecided: `keystroke@r100K` and `send_turn@r100K`, both "never reached min_observations"
-# despite `--min-reps 2` and both having run on both arms in both repetitions. They are the two
-# actions whose slot lands INSIDE a live turn on the packed 100K film, and the payload says so:
-# `streaming: true`, `in_flight: [19]` and `[22]`, the in-flight message 18,277 characters on one
-# arm and 18,253 on the other.
-#
-# WHERE IT CAME FROM. Pre-#9575 that pair scored DIFFER, which is wrong but is an OBSERVATION, so
-# the null decided the action and the audit passed. #9575 correctly demoted it to NOT_COMPARABLE,
-# and `derive_unstable` files every NOT_COMPARABLE under `blind`, which is zero observations.
-# Replayed against the real payload: keystroke scores 2 observations pre-#9575 and 0 after.
-#
-# So the refusal is right and the accounting behind it was not. `SETTLED_MATCH` is the narrow
-# repair: that branch is reached only once the scaffold, overlays, message set and every settled
-# row have agreed, so it is a complete reading of everything the pair could read.
-#
-# Held in both directions, because a fix that makes the audit easier to pass is worth nothing
-# without the control that shows it still fails.
+# WHAT TURNED RED: a wave of `studiobench UI parity` audited UNDECIDED with `keystroke@r100K`
+# and `send_turn@r100K` "never reached min_observations" despite --min-reps 2 and both arms
+# running both repetitions: the two actions whose slot lands INSIDE a live turn on the packed
+# 100K film. Pre-#9575 that pair scored DIFFER, which is wrong but is an OBSERVATION; #9575
+# correctly demoted it to NOT_COMPARABLE, which `derive_unstable` files under `blind` (zero
+# observations). So the refusal is right and the accounting was not: `SETTLED_MATCH` is the
+# narrow repair, reached only once the scaffold, overlays, message set and every settled row
+# have agreed. Held in both directions, since a fix that makes the audit easier to pass is
+# worth nothing without the control showing it still fails.
+# ── the reply that was still arriving is not a blind capture ──────────
def streaming_pair_rows(
cid_base: str,
cid_treat: str,
@@ -1503,9 +1413,9 @@ def streaming_null(tmp_path: Path, name: str, cells: list[list[dict]]) -> Path:
def test_a_mid_stream_tail_is_still_a_refusal_and_never_a_pass():
- # FIRST, the thing that must not change: the verdict stays NOT_COMPARABLE. A treatment build
- # whose live message renders wrongly sits in exactly this shape, so promoting it to MATCH
- # would hide a regression. The repair is in what the NULL counts, not in what the result says.
+ # FIRST, the thing that must not change: the verdict stays NOT_COMPARABLE, because a treatment
+ # build whose live message renders wrongly sits in exactly this shape. The repair is in what
+ # the NULL counts.
rows = streaming_pair_rows(
"r100K.base.rep0",
"r100K.treatment.rep0",
@@ -1517,13 +1427,10 @@ def test_a_mid_stream_tail_is_still_a_refusal_and_never_a_pass():
assert got["verdict"] == P.NOT_COMPARABLE
assert "STILL BEING WRITTEN" in got["reason"]
assert got["moved"] == []
- # And it carries the flag that says the rest of the thread WAS read and did agree.
assert got[P.SETTLED_MATCH] is True
def test_a_refusal_that_read_the_settled_thread_is_an_observation():
- # The unit the audit is built on. Two settled-match refusals are two observations of
- # non-difference, so the action is decided and is NOT called unstable.
rows = streaming_pair_rows(
"r100K.base.rep0",
"r100K.treatment.rep0",
@@ -1542,9 +1449,9 @@ def test_a_refusal_that_read_the_settled_thread_is_an_observation():
def test_an_ordinary_refusal_is_still_blind():
- # THE CONTROL. Without it the change reads as "count NOT_COMPARABLE as an observation", which
- # would derive an action that never rendered at all as stable. A failed capture carries no
- # settled reading and stays at zero.
+ # THE CONTROL: without it the change reads as "count NOT_COMPARABLE as an observation", which
+ # would derive an action that never rendered as stable. A failed capture carries no settled
+ # reading and stays at zero.
blind = {"verdict": P.NOT_COMPARABLE, "reason": "threadRoot is not a function"}
row = P.derive_unstable([("keystroke", blind), ("keystroke", blind)])["keystroke"]
assert row["observations"] == 0
@@ -1554,8 +1461,6 @@ def test_an_ordinary_refusal_is_still_blind():
def test_a_settled_message_that_moved_is_a_difference_not_an_observation():
- # One layer down: the flag is only set when everything OUTSIDE the live message agreed. Move
- # a settled row and the pair is a DIFFER, so real instability is recorded as before.
rows = streaming_pair_rows(
"r100K.base.rep0",
"r100K.treatment.rep0",
@@ -1570,9 +1475,8 @@ def test_a_settled_message_that_moved_is_a_difference_not_an_observation():
def test_a_scaffold_that_moved_is_a_difference_not_an_observation():
- # The scaffold half of the same claim. Both arms agree on streaming and on the composer
- # control, so this is not the generation-disagreement refusal but a scaffolding change on an
- # otherwise identical pair, and it stays a finding.
+ # The scaffold half of the same claim: both arms agree on streaming and on the composer
+ # control, so this is a scaffolding change on an otherwise identical pair and stays a finding.
rows = streaming_pair_rows(
"r100K.base.rep0",
"r100K.treatment.rep0",
@@ -1587,11 +1491,10 @@ def test_a_scaffold_that_moved_is_a_difference_not_an_observation():
def test_a_live_row_whose_role_changed_carries_no_positive_reading():
- # The control `_any_moved` cannot reach, which is what makes it worth asserting. The role
- # sits beside the digest and an in-flight digest is withheld, so `assistant` on one arm and
- # `user` on the other passes the digest comparison untouched and the pair still reads as
- # "everything settled agreed". `settled_messages_moved` calls that a structural change even
- # in flight, so the flag must not fire here.
+ # The control `_any_moved` cannot reach: the role sits beside the digest and an in-flight
+ # digest is withheld, so `assistant` on one arm and `user` on the other passes the digest
+ # comparison untouched, and `settled_messages_moved` calls that a structural change even in
+ # flight.
rows = streaming_pair_rows(
"r100K.base.rep0",
"r100K.treatment.rep0",
@@ -1605,7 +1508,6 @@ def test_a_live_row_whose_role_changed_carries_no_positive_reading():
got = P.compare(base, treat)
assert got["verdict"] == P.NOT_COMPARABLE
assert P.SETTLED_MATCH not in got
- # And so it stays blind: two of them decide nothing and the audit still has to say undecided.
row = P.derive_unstable([("keystroke", got), ("keystroke", got)])["keystroke"]
assert row["observations"] == 0
assert row["not_comparable"] == 2
@@ -1614,10 +1516,9 @@ def test_a_live_row_whose_role_changed_carries_no_positive_reading():
def test_a_settled_match_cannot_supply_the_observation_that_mints_an_exemption():
# DECIDING an action and CLASSIFYING it unstable are different claims, and only the first is
- # this flag's to make. One DIFFER beside one settled-match refusal is one differing comparison,
- # so letting the refusal supply the second observation would derive `(rung, action)` into the
- # MEASURED exemption set from a reading that never saw the live subtree -- the excuse set
- # growing, which `SETTLED_MATCH` promises it cannot do.
+ # this flag's to make: one DIFFER beside one settled-match refusal is one differing
+ # comparison, so letting the refusal supply the second would grow the MEASURED exemption set
+ # from a reading that never saw the live subtree.
rows = streaming_pair_rows(
"r100K.base.rep0",
"r100K.treatment.rep0",
@@ -1632,21 +1533,18 @@ def test_a_settled_match_cannot_supply_the_observation_that_mints_an_exemption()
assert row["observations"] == 2
assert row["differed"] == 1
assert row[P.SETTLED_MATCH] == 1
- # NOT unstable, because only one of the two readings compared anything, and not "decided and
- # stable" either: one differing comparison is the single flake `min_observations` exists for.
- # On the raw total this reached `cross_check` as `declared_stable_in_practice`, whose stated
- # meaning is that the null never saw this action differ.
+ # NOT unstable, because only one reading compared anything, and not "decided and stable"
+ # either: one differing comparison is the single flake `min_observations` exists for. On the
+ # raw total this reached `cross_check` as `declared_stable_in_practice`.
assert row["unstable"] is False
assert row["undetermined"] is True
- # Two complete comparisons still classify exactly as before.
both = P.derive_unstable([("keystroke", differ), ("keystroke", differ)])["keystroke"]
assert both["unstable"] is True
def test_the_audit_decides_an_action_whose_only_leftover_was_the_live_reply(tmp_path):
# END TO END, the case that turned the job red: two repetitions of `keystroke` inside a live
- # turn, the settled thread identical on every arm and only the in-flight tail a chunk apart.
- # Nothing structural moved on this build against itself, and the audit must be able to say so.
+ # turn, the settled thread identical and only the in-flight tail a chunk apart.
null = streaming_null(
tmp_path,
"inflight",
@@ -1672,14 +1570,13 @@ def test_the_audit_decides_an_action_whose_only_leftover_was_the_live_reply(tmp_
assert report["decided"] == [("r100K", "keystroke")]
assert report["undecided"] == []
assert report["differed"] == []
- # Reported as the narrower reading it is, rather than folded in with a plain MATCH.
assert report["decided_on_settled_thread"] == [("r100K", "keystroke")]
def test_the_audit_still_fails_when_the_capture_itself_was_blind(tmp_path):
- # THE PAIRED FAILURE. Same scope and repetitions, but the arms could not place the stream at
- # all (`in_flight_unplaced`: the `data-status` hook went quiet). No settled reading sits
- # behind that refusal, so the audit stays red.
+ # THE PAIRED FAILURE: same scope and repetitions, but the arms could not place the stream at
+ # all (`in_flight_unplaced`), so no settled reading sits behind the refusal and the audit
+ # stays red.
cells = []
for rep, tails in (("rep0", ("T1", "T2")), ("rep1", ("T3", "T4"))):
cell = streaming_pair_rows(
@@ -1700,9 +1597,8 @@ def test_the_audit_still_fails_when_the_capture_itself_was_blind(tmp_path):
def test_a_settled_match_can_never_grow_the_excuse_set(tmp_path):
- # THE SAFETY DIRECTION, asserted rather than argued. The observation is a NON-difference, so
- # the derived excuse set can only narrow: a null made entirely of settled-match refusals
- # derives an EMPTY measured set, so a corroborated difference gets no excuse from it.
+ # THE SAFETY DIRECTION, asserted rather than argued: the observation is a NON-difference, so a
+ # null made entirely of settled-match refusals derives an EMPTY measured set.
null = streaming_null(
tmp_path,
"narrow",
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py
index 50ddc99b165..801e3c54c2b 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py
@@ -104,8 +104,8 @@ def quiet_null(tmp: Path, name: str, actions: list[str]) -> Path:
def test_only_the_stable_differences_are_illustrated(tmp_path, capsys):
shots = tmp_path / "shots"
- # `settings` differs and is stable here; `stop_generation` differs and is DECLARED unstable,
- # so it is noise the reader must not be handed.
+ # `settings` differs and is stable here; `stop_generation` differs and is DECLARED unstable, so
+ # it is noise the reader must not be handed.
result = result_with(
tmp_path,
"result",
@@ -205,8 +205,8 @@ def test_the_two_halves_are_labelled_in_the_image_not_only_the_filename(tmp_path
},
)
got = Image.open(out).convert("RGB")
- # The banner strip must carry ink on both halves. A pair whose sides are distinguished only
- # by filename is one rename away from being misattributed.
+ # The banner strip must carry ink on both halves: a pair whose sides are distinguished only by
+ # filename is one rename away from being misattributed.
left = got.crop((0, 0, 320, S.BANNER)).getcolors(maxcolors = 100000)
right = got.crop((320 + S.GUTTER, 0, 640 + S.GUTTER, S.BANNER)).getcolors(maxcolors = 100000)
assert len(left) > 1, "the BEFORE banner drew nothing"
@@ -227,9 +227,8 @@ def test_shot_index_reads_the_payload_rather_than_globbing(tmp_path):
def test_a_one_repetition_flake_is_not_illustrated_at_the_verdicts_threshold(tmp_path, capsys):
- # `settings` differs on both passes and is what fails the job; `thread_reopen` differs on one
- # and is explicitly uncorroborated. Shipping both leaves the reader unable to tell which is
- # which, which buries the finding the artifact exists to show.
+ # `settings` differs on both passes and fails the job; `thread_reopen` differs on one and is
+ # explicitly uncorroborated. Shipping both leaves the reader unable to tell which is which.
shots = tmp_path / "shots"
rows = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
@@ -249,9 +248,8 @@ def test_a_one_repetition_flake_is_not_illustrated_at_the_verdicts_threshold(tmp
def test_the_workflow_illustrates_at_the_same_threshold_it_scores_at():
- # Two numbers in two steps of one job that must agree, and nothing at runtime would notice
- # if they drifted: the verdict would fail on one set of actions and the artifact would show
- # another.
+ # Two numbers in two steps of one job that must agree, with nothing at runtime to notice if they
+ # drifted: the verdict would fail on one set of actions and the artifact would show another.
text = (
Path(__file__).resolve().parents[5] / ".github/workflows/studiobench-ui-parity.yml"
).read_text(encoding = "utf-8")
@@ -262,10 +260,10 @@ def test_the_workflow_illustrates_at_the_same_threshold_it_scores_at():
def test_the_cli_forwards_the_threshold_it_was_given(tmp_path, capsys):
- # `build` takes min_reps and `main` parses --min-reps, and between the two the value was
- # dropped: the workflow asked for 2, `build` defaulted back to 1, and the artifact carried a
- # one-repetition flake beside the change that actually failed the job. Driven through main()
- # rather than build(), because build() was never the half that was wrong.
+ # `build` takes min_reps and `main` parses --min-reps, and the value was dropped between them:
+ # the workflow asked for 2, `build` defaulted back to 1, and the artifact carried a
+ # one-repetition flake beside the change that failed the job. Driven through main() because
+ # build() was never the half that was wrong.
shots = tmp_path / "shots"
rows = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
@@ -301,10 +299,10 @@ def test_the_cli_forwards_the_threshold_it_was_given(tmp_path, capsys):
def test_the_workflow_actually_runs_the_prune_it_documents():
- # The images are the only large thing this job produces and the payload upload is
- # `if: always()`, so a green run shipped every one of them -- 18 actions x 2 arms x 2
- # repetitions per job -- while the step below it claimed the arms had already deleted the
- # matching ones. A command nobody invokes cannot be the reason a claim is true.
+ # The images are the only large thing this job produces and the payload upload is `if: always()`,
+ # so a green run shipped every one of them (18 actions x 2 arms x 2 repetitions per job) while
+ # the step below claimed the arms had already deleted the matching ones. A command nobody invokes
+ # cannot be the reason a claim is true.
text = (
Path(__file__).resolve().parents[5] / ".github/workflows/studiobench-ui-parity.yml"
).read_text(encoding = "utf-8")
@@ -333,9 +331,9 @@ def _one_sided_rows(action_name: str, reps: tuple[str, ...], shots: Path) -> lis
def test_a_control_that_only_one_arm_can_operate_is_illustrated(tmp_path, capsys):
# The regression shape that leaves NO digest to differ: a control that stops opening records
- # `ran: false`, not a different hash. `ui_parity.report` exits 1 on it, and the arm-side prune
- # keeps its shots for exactly this step -- but the selector only accepted DIFFER, so a red
- # verdict caused by it published "no STABLE differences" and no composite at all.
+ # `ran: false`, not a different hash. `ui_parity.report` exits 1 on it and the arm-side prune
+ # keeps its shots for this step, but the selector only accepted DIFFER, so a red verdict
+ # published 'no STABLE differences' and no composite at all.
shots = tmp_path / "shots"
result = write(tmp_path, "result", _one_sided_rows("settings", ("rep0", "rep1"), shots))
null = quiet_null(tmp_path, "null", ["settings"])
@@ -347,9 +345,8 @@ def test_a_control_that_only_one_arm_can_operate_is_illustrated(tmp_path, capsys
def test_a_one_arm_miss_in_one_repetition_of_two_is_not_illustrated(tmp_path):
- # The same bar the verdict uses. A contended runner can lose one arm's slot once, and that
- # does not red the job -- so putting a picture of it beside the change that did would bury
- # the finding the artifact exists to show.
+ # The same bar the verdict uses. A contended runner can lose one arm's slot once without redding
+ # the job, so a picture of it beside the change that did would bury the finding.
shots = tmp_path / "shots"
result = write(tmp_path, "result", _one_sided_rows("settings", ("rep0",), shots))
null = quiet_null(tmp_path, "null", ["settings"])
@@ -359,12 +356,10 @@ def test_a_one_arm_miss_in_one_repetition_of_two_is_not_illustrated(tmp_path):
def test_a_superseded_attempts_screenshot_is_not_shown_for_the_retrys_verdict(tmp_path):
- # `--resume` re-runs an A/B pair WHOLE, so an arm that already succeeded is re-run under the
- # same deterministic cell id. If the retry captures the differing digest but its screenshot
- # fails, a raw scan of the append-only payload keeps the DEAD attempt's `shot` -- because the
- # newer row carries none -- and the artifact pairs the retry's verdict with a picture of a
- # page that is not the one that turned the gate red. A missing half is a caption `build`
- # already prints; a stale half is a lie.
+ # `--resume` re-runs an A/B pair WHOLE under the same deterministic cell id, so if the retry
+ # captures the differing digest but its screenshot fails, a raw scan of the append-only payload
+ # keeps the DEAD attempt's `shot` and pairs the retry's verdict with a picture of a different
+ # page. A missing half is a caption `build` already prints; a stale half is a lie.
shots = tmp_path / "shots"
png(shots / "old.png", 200, 120, (10, 60, 10))
png(shots / "base.png", 200, 120, (10, 60, 10))
@@ -409,17 +404,17 @@ def test_a_superseded_attempts_screenshot_is_not_shown_for_the_retrys_verdict(tm
def test_the_workflow_runs_the_gate_on_the_routes_the_measurement_depends_on():
# A gate that does not fire on a change that can move what it measures is not measuring it.
- # chat-api.ts sends the scripted turn to /api/inference/chat/completions, and lifecycle.py
- # logs each arm in through /api/auth/login and /api/auth/change-password before any scene
- # renders -- so both can alter, empty, or block the measured DOM on a PR with no frontend file.
+ # chat-api.ts sends the scripted turn to /api/inference/chat/completions, and lifecycle.py logs
+ # each arm in through /api/auth/login and /api/auth/change-password before any scene renders, so
+ # both can alter, empty or block the measured DOM on a PR with no frontend file.
text = (
Path(__file__).resolve().parents[5] / ".github/workflows/studiobench-ui-parity.yml"
).read_text(encoding = "utf-8")
for route in ("inference.py", "auth.py", "chat_history.py", "providers.py"):
assert f"studio/backend/routes/{route}" in text, route
- # A route is not one file. Its response models, its provider lookups and its storage can each
- # change the measured picker without the route module being touched, which is how the same
- # item arrived three rounds running, one file further down each time.
+ # A route is not one file: its response models, provider lookups and storage can each change the
+ # measured picker without the route module being touched, which is how the same item arrived
+ # three rounds running, one file further down each time.
for dep in (
"studio/backend/storage/studio_db.py",
"studio/backend/models/providers.py",
@@ -440,8 +435,8 @@ def test_the_evidence_uses_the_same_confined_set_the_verdict_scored_with(tmp_pat
as publishing no evidence at all, one round after it was fixed the first time.
"""
shots = tmp_path / "shots"
- # The null raced on `settings`. This runner did not: side A is identical across both
- # repetitions, and head differs in both.
+ # The null raced on `settings`. This runner did not: side A is identical across both repetitions,
+ # and head differs in both.
null_rows = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
for arm in ("base", "treatment"):
@@ -468,9 +463,9 @@ def test_the_evidence_uses_the_same_confined_set_the_verdict_scored_with(tmp_pat
def test_a_direction_reversing_pair_is_not_illustrated_either(tmp_path):
- # The verdict does not fail on a pair whose two repetitions blame opposite arms, so the
- # artifact must not present one as the change that turned the job red. The two selections are
- # keyed the same way for the same reason they are corroborated the same way.
+ # The verdict does not fail on a pair whose two repetitions blame opposite arms, so the artifact
+ # must not present one as the change that turned the job red. The two selections are keyed the
+ # same way for the same reason they are corroborated the same way.
shots = tmp_path / "shots"
rows = [{"row_type": "run_meta", "tier": "fast"}]
for rep in ("rep0", "rep1"):
@@ -539,14 +534,12 @@ def test_two_shards_do_not_share_one_screenshot_identity(tmp_path):
# Direct first-party imports of a measured route that are deliberately NOT gated on, each with
-# the reason. This is the other half of the filter: a module is either in the workflow or it is
-# here, and adding an import to a measured route fails the test below until someone decides which.
-#
-# The drip this ends: routes/chat_history.py was listed, then its storage/studio_db.py arrived a
-# round later, then models/providers.py and core/inference/providers.py, then external_provider.py
-# and sse_control_frames.py, then provider_credentials.py. Every one was correct and every one was
-# found by reading the imports by hand. The boundary is a judgement, but it should be a RECORDED
-# judgement rather than one rediscovered each round.
+# the reason. A module is either in the workflow or here, and adding an import to a measured
+# route fails the test below until someone decides which.
+# The drip this ends: routes/chat_history.py was listed, then storage/studio_db.py a round later,
+# then models/providers.py and core/inference/providers.py, then external_provider.py and
+# sse_control_frames.py, then provider_credentials.py, every one correct and every one found by
+# reading imports by hand. The boundary is a judgement, but a RECORDED one.
NOT_GATED: dict[str, str] = {
"studio/backend/auth/authentication.py": "session plumbing shared by every route in the app; "
"routes/auth.py already gates the endpoints studiobench actually calls",
@@ -594,7 +587,7 @@ def test_every_import_of_a_measured_route_is_gated_or_explicitly_waived():
def test_every_waiver_names_a_file_that_exists_and_gives_a_reason():
# A waiver for a file that has been moved or deleted is a stale exemption that reads as a
- # decision. Same bar the RACY_EXECUTION entries are held to.
+ # decision. Same bar as the RACY_EXECUTION entries.
repo = Path(__file__).resolve().parents[5]
for path, why in NOT_GATED.items():
assert (repo / path).exists(), path
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py
index ecf718ed183..95c73d69e81 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py
@@ -56,8 +56,7 @@ def _payload(tmp_path: Path) -> Path:
"completed": True,
}
)
- # A control that is measured at BOTH rungs, so the refusal cannot pass by
- # rejecting everything.
+ # A control that is measured at BOTH rungs, so the refusal cannot pass by rejecting everything.
rows.append(
{
"row_type": "action",
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py
index 01c346802c4..0249e432048 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py
@@ -48,8 +48,8 @@ METRIC = "message_menu.open_close_ms"
#: Wall clock per session. A resume starts after the run it continues has stopped; the real
-#: two-launcher payload of defect 9 overlaps by sixteen minutes, which is what `_at` reproduces
-#: for the concurrent fixtures.
+#: two-launcher payload of defect 9 overlaps by sixteen minutes, which is what `_at` reproduces for
+#: the concurrent fixtures.
_STARTED = {
"s1": "2026-08-23T00:00:00",
"t2": "2026-08-23T02:00:00",
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py
index dfdab1e0406..4fb5571cfb3 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py
@@ -64,8 +64,8 @@ def test_cell_metrics_refuses_to_collapse_two_sessions():
"cell_metrics keyed on cell_id alone and returned the last writer's values. That is how a "
"payload holding two concurrent runs reported a 149.8% regression that does not exist."
)
- # The colliding cell is NAMED, not just counted. A refusal that says only "two sessions" sends
- # the reader back to the payload to find out which reading it was protecting them from.
+ # The colliding cell is NAMED, not just counted: a refusal that says only "two sessions" sends the
+ # reader back to the payload to find out which reading it was protecting them from.
assert "r1M.base.rep0" in message
assert "91c4d6d94da8" in message and "430f0b831dda" in message
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py
index baff730dad0..e9ef3c0d7ea 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py
@@ -88,7 +88,7 @@ def test_a_large_consistent_effect_over_a_tight_floor_passes(tmp_path):
def test_an_effect_under_the_floor_is_void(tmp_path):
- # 3% claimed, against a null control whose own two identical builds land 20% apart.
+ # 3% claimed, against a null control whose two identical builds land 20% apart.
assert (
verdict(
tmp_path,
@@ -100,9 +100,8 @@ def test_an_effect_under_the_floor_is_void(tmp_path):
def test_the_floor_clears_the_null_controls_bias_not_only_its_spread(tmp_path):
- # A null control that is TIGHT but systematically offset: identical builds, yet the treatment
- # label reads 10% faster on every repetition. Spread is nearly zero, so a floor built from
- # spread alone would admit any effect at all. The bar is max(|bias|, spread).
+ # A null control that is tight but systematically offset: spread alone would admit any
+ # effect, so the bar is max(|bias|, spread).
floor_pairs = [(1000.0, 900.0)] * 4
assert verdict(tmp_path, [(1000.0, 950.0)] * 4, floor_pairs) == "VOID (under floor)"
assert verdict(tmp_path, [(1000.0, 700.0)] * 4, floor_pairs) == "faster"
@@ -112,7 +111,7 @@ def test_the_floor_clears_the_null_controls_bias_not_only_its_spread(tmp_path):
def test_pairs_that_disagree_on_sign_are_void_however_large_the_mean(tmp_path):
- # Mean lands well past the floor; two repetitions say faster and two say slower.
+ # Mean past the floor; two repetitions say faster and two say slower.
assert (
verdict(
tmp_path,
@@ -127,8 +126,7 @@ def test_pairs_that_disagree_on_sign_are_void_however_large_the_mean(tmp_path):
def test_an_effect_smaller_than_its_own_scatter_is_void(tmp_path):
- # Every repetition agrees on the direction and the mean clears the floor, but the readings
- # range from 2% to 60% faster. Twelve rows in a 40-comparison audit looked exactly like this.
+ # Every repetition agrees and the mean clears the floor, but readings range from 2% to 60% faster.
assert (
verdict(
tmp_path,
@@ -151,8 +149,7 @@ def test_gate_three_does_not_fire_on_a_tight_large_effect(tmp_path):
def test_gate_three_cannot_fire_on_a_single_pair(tmp_path):
- # With n=1 the spread is 0 by construction, so the gate has nothing to say. It must not
- # silently pass a single reading off as having survived a scatter check.
+ # With n=1 the spread is 0 by construction, so the gate must not pass a single reading off as scatter-checked.
result = F.summarise([payload(tmp_path, "result", [(1000.0, 500.0)])])
assert result["message_menu.open_close_ms"]["n"] == 1
_f, v = F.verdict_for(
@@ -203,14 +200,13 @@ def test_a_count_is_harvested_under_a_name_that_marks_it_as_an_invariant(tmp_pat
def test_a_count_that_fell_reads_as_a_loss_and_never_as_faster(tmp_path):
- # The regression this exists to catch: virtualization truncates select-all from 400k characters
- # to 3k. Every timing improves, `expect_ok` stays true because chars > 0, and the only thing
- # that can say so is this row. Scored as a timing it would read "faster".
+ # The regression this catches: virtualization truncates select-all from 400k chars to 3k.
+ # Every timing improves, expect_ok stays true, and only this row can say so.
stats = F.summarise([count_payload(tmp_path, "r", [(400000.0, 3000.0)] * 4)])
floor = {"delta_pct": 0.0, "spread_pct": 1.0}
_f, v = F.verdict_for(stats[COUNT_METRIC], floor, F.is_count_metric(COUNT_METRIC))
assert v == "LOST (invariant fell)"
- # Same numbers, scored as a timing, would have been reported as an improvement.
+ # Same numbers scored as a timing would read as an improvement.
assert F.verdict_for(stats[COUNT_METRIC], floor, False)[1] == "faster"
@@ -232,7 +228,7 @@ def test_a_lost_invariant_is_printed_and_counted_by_the_table(tmp_path, capsys):
survivors = F.render([result], "t", floors = floors)
out = capsys.readouterr().out
assert "LOST (invariant fell)" in out
- # It has to be counted as a finding, not dropped for not being one of the two timing verdicts.
+ # Counted as a finding, not dropped for not being one of the two timing verdicts.
assert survivors >= 1
@@ -303,9 +299,8 @@ def test_scoring_against_a_floor_from_another_tier_is_refused(tmp_path):
def test_pooling_across_corpora_is_refused(tmp_path):
- # The tier fixes how long the film runs; the corpus hash fixes what is IN it. Corpus v2 added
- # math, so a v1 payload and a v2 payload measure two different documents under one name, and
- # pooling them would read the corpus change as a performance change.
+ # The tier fixes how long the film runs, the corpus hash fixes what is IN it: pooling v1 and
+ # v2 payloads reads a corpus change as a performance change.
one = payload(tmp_path, "one", [(1000.0, 900.0)], corpus = "aaaa1111")
two = payload(tmp_path, "two", [(1000.0, 900.0)], corpus = "bbbb2222")
with pytest.raises(SystemExit) as exc:
@@ -328,8 +323,7 @@ def test_the_same_corpus_on_both_sides_pools_normally(tmp_path):
def test_a_payload_with_no_corpus_hash_is_not_silently_pooled_with_one_that_has_it(tmp_path):
- # An older payload predating the field reads "?", which is a different value, not a wildcard.
- # Treating it as compatible is how a v1 run would end up scored against a v2 floor.
+ # An older payload predating the field reads '?', a different value, not a wildcard.
old = payload(tmp_path, "old", [(1000.0, 900.0)], corpus = None)
new = payload(tmp_path, "new", [(1000.0, 900.0)], corpus = "bbbb2222")
with pytest.raises(SystemExit) as exc:
@@ -338,11 +332,8 @@ def test_a_payload_with_no_corpus_hash_is_not_silently_pooled_with_one_that_has_
def test_a_resumed_payload_carrying_two_corpora_is_refused(tmp_path):
- # The recorder appends, so `--resume` into the same --out leaves the first run's completed
- # cells next to a SECOND run_meta. If the corpus changed in between, that one file holds a
- # base recorded on the old film and a treatment recorded on the new one, and `paired` matches
- # them on (shard, rung, rep) without noticing. Reading only the first header would pass this
- # payload and print the corpus change as a performance change.
+ # `--resume` into the same --out leaves a second run_meta; one file can then hold a base on
+ # the old film and a treatment on the new, and `paired` matches them regardless.
out = tmp_path / "resumed"
out.mkdir(parents = True, exist_ok = True)
rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "corpus_hash": "aaaa1111"}]
@@ -356,14 +347,14 @@ def test_a_resumed_payload_carrying_two_corpora_is_refused(tmp_path):
with pytest.raises(SystemExit) as exc:
F.load([path])
assert "more than one corpus" in str(exc.value)
- # And the floor-vs-result path refuses it too, rather than scoring it under the first hash.
+ # And the floor-vs-result path refuses it too, rather than scoring under the first hash.
with pytest.raises(SystemExit) as exc:
F.render([path], "t", floors = {}, floor_corpus = "aaaa1111")
assert "more than one corpus" in str(exc.value)
def test_a_payload_with_repeated_headers_on_one_corpus_still_loads(tmp_path):
- # A plain resume, nothing changed in between: two headers, one hash, no refusal.
+ # A plain resume, nothing changed: two headers, one hash, no refusal.
out = tmp_path / "plain"
out.mkdir(parents = True, exist_ok = True)
meta = {"row_type": "run_meta", "tier": "standard", "corpus_hash": "aaaa1111"}
@@ -421,8 +412,7 @@ def test_an_incomplete_cell_is_not_measured(tmp_path):
def test_a_shard_pairs_within_itself_and_never_across(tmp_path):
- # Two shards both number their repetitions from rep0. Pairing on the repetition alone would
- # cross them over and silently compare one session's base with another's treatment.
+ # Both shards number repetitions from rep0; pairing on the repetition alone would cross them over.
a = payload(tmp_path / "a", "s0", [(1000.0, 500.0)])
b = payload(tmp_path / "b", "s0", [(2000.0, 1000.0)])
pooled, _ = F.load([a, b])
@@ -431,9 +421,8 @@ def test_a_shard_pairs_within_itself_and_never_across(tmp_path):
def test_an_action_whose_own_assertion_failed_contributes_no_timing(tmp_path):
- # `ran = True, expect_ok = False` is the action that happened and did not do its job. Its p95
- # is lower BECAUSE it failed, so pairing it prints the failure as an improvement. The payload
- # notes layer already says these timings must not be quoted.
+ # `ran=True, expect_ok=False` is the action that happened and failed; its p95 is lower BECAUSE
+ # it failed, so pairing prints the failure as an improvement.
rows = [
{"row_type": "run_meta", "tier": "standard"},
{"row_type": "cell", "cell_id": "100K.base.rep0", "completed": True},
@@ -456,8 +445,8 @@ def test_an_action_whose_own_assertion_failed_contributes_no_timing(tmp_path):
def test_an_action_with_no_expectation_recorded_is_still_harvested(tmp_path):
- # `expect_ok` is None on an action that asserts nothing, and on every payload recorded before
- # the field existed. Only an explicit False is a failed assertion.
+ # `expect_ok` is None on an action that asserts nothing and on pre-field payloads; only
+ # explicit False is a failed assertion.
rows = [
{"row_type": "run_meta", "tier": "standard"},
{"row_type": "cell", "cell_id": "100K.base.rep0", "completed": True},
@@ -570,7 +559,7 @@ def test_a_clean_zero_base_arm_still_pairs_so_a_jank_regression_is_not_lost(tmp_
stats, floors = F.summarise([result]), F.summarise([null])
s = stats["stream_time_in_jank_pct"]
assert s["n"] == 4
- # Compared by DIFFERENCE, in the metric's own unit: 0.0% to 12.0% is +12.0 points.
+ # Compared by DIFFERENCE in the metric's own unit: 0.0% to 12.0% is +12.0 points.
assert s["difference"] is True
assert s["delta_pct"] == pytest.approx(12.0)
assert "stream_time_in_jank_pct" in floors
@@ -594,7 +583,7 @@ def test_an_unchanged_repetition_does_not_read_as_a_pair_disagreeing_on_sign(tmp
something that did not happen is reporting a wrong answer in the right vocabulary, which is
worth three lines on its own; it is not worth relaxing gate 3 to chase.
"""
- # Two repetitions clean on both arms, two that regress. Nothing regresses the other way.
+ # Two repetitions clean on both arms, two that regress, nothing the other way.
result = stream_payload(tmp_path, "result", [(SMOOTH, SMOOTH), (SMOOTH, JANKY)] * 2)
null = stream_payload(tmp_path, "null", [(SMOOTH, SMOOTH)] * 4)
@@ -607,7 +596,7 @@ def test_an_unchanged_repetition_does_not_read_as_a_pair_disagreeing_on_sign(tmp
]
assert s["consistent"] is True, "an unchanged repetition is a tie, not a disagreement"
_f, v = F.verdict_for(s, F.summarise([null])["stream_time_in_jank_pct"])
- # Still refused, and that is right, but no longer refused for a reason that did not occur.
+ # Still refused, and rightly, but no longer for a reason that did not occur.
assert v == "VOID (effect under its own scatter)"
assert v != "VOID (pairs disagree on sign)"
@@ -632,9 +621,7 @@ def test_a_ratio_metric_still_drops_a_zero_base_rather_than_dividing_by_it(tmp_p
def test_the_enforced_idle_window_is_not_pooled_into_the_frame_metrics(tmp_path):
- # Every cell records a 1.5 s `idle:calibrate` window with the frame recorder running. Pooling
- # its quiet into the film halves the jank share, so the column would not be the quantity the
- # rest of the tool prints under that name.
+ # Pooling each cell's 1.5s `idle:calibrate` quiet into the film halves the jank share.
cid = "r100K.base.rep0"
rows = [
{"row_type": "run_meta", "tier": "standard"},
@@ -648,9 +635,8 @@ def test_the_enforced_idle_window_is_not_pooled_into_the_frame_metrics(tmp_path)
def test_a_resumed_cell_is_measured_from_its_own_attempt_only(tmp_path):
- # `--resume` re-runs a cell that died, and the payload is append-only, so the dead attempt's
- # windows sit in the file under the SAME cell id. Pooling them into the retry reports a number
- # that no single run of the film ever produced.
+ # `--resume` re-runs a dead cell into an append-only payload under the SAME cell id; pooling
+ # both attempts reports a number no single run produced.
cid = "r100K.base.rep0"
rows = [
{"row_type": "run_meta", "tier": "standard", "session_id": "s1"},
@@ -666,8 +652,8 @@ def test_a_resumed_cell_is_measured_from_its_own_attempt_only(tmp_path):
def test_one_file_holding_two_tiers_is_refused_like_two_files(tmp_path):
- # The recorder appends, so a second run into the same --out leaves both films in one payload.
- # Reading only the first run_meta let that file through the refusal it was written for.
+ # A second run into the same --out leaves both films in one payload, and reading only the
+ # first run_meta let that through.
rows = [
{"row_type": "run_meta", "tier": "fast"},
*cell("100K", "base", "rep0", {"open_close_ms": 1000.0}),
@@ -736,10 +722,9 @@ def resumed_payload(tmp_path: Path, name: str, retry_session: str) -> Path:
def test_an_arm_resumed_into_a_new_session_is_not_paired_with_the_old_one(tmp_path):
- # `--resume` skips the arm that completed and re-runs the one that died, under a NEW session id
- # in the SAME shard directory. Keyed on the repetition alone the two pair, and the whole 8%
- # session drift is charged to whichever arm was re-run. `scoring/ab.py` refuses exactly this
- # comparison; the sweep's own pairing has to refuse it too.
+ # `--resume` re-runs the dead arm under a NEW session id in the same shard; keyed on the
+ # repetition alone the two pair and the 8% drift is charged to the re-run arm.
+ # `scoring/ab.py` refuses exactly this.
path = resumed_payload(tmp_path, "resumed", "s2")
assert F.cell_metrics(F.read_rows(path))["r100K.treatment.rep0"] == {
"message_menu.open_close_ms": 108.0
@@ -748,8 +733,8 @@ def test_an_arm_resumed_into_a_new_session_is_not_paired_with_the_old_one(tmp_pa
def test_an_arm_resumed_inside_the_same_session_still_pairs(tmp_path):
- # The other direction, so the refusal above cannot pass by rejecting every resumed run. Two
- # attempts in ONE session are still one session, and the retry's own numbers pair normally.
+ # The other direction, so the refusal cannot pass by rejecting every resumed run: two attempts
+ # in ONE session still pair normally.
path = resumed_payload(tmp_path, "same", "s1")
assert F.paired(F.read_rows(path), shard = "same") == {
"message_menu.open_close_ms": [(100.0, 108.0)]
@@ -757,8 +742,8 @@ def test_an_arm_resumed_inside_the_same_session_still_pairs(tmp_path):
def test_a_payload_with_no_session_ids_pairs_exactly_as_before(tmp_path):
- # Payloads recorded before session ids existed carry none, so both arms resolve to "" and the
- # new key term is inert. A refusal that also rejected these would delete every old reading.
+ # Pre-session-id payloads resolve both arms to '', so the new key term is inert; refusing them
+ # would delete every old reading.
path = payload(tmp_path, "legacy", [(1000.0, 900.0)])
assert F.paired(F.read_rows(path), shard = "legacy") == {
"message_menu.open_close_ms": [(1000.0, 900.0)]
@@ -789,8 +774,8 @@ def parity_action(cid: str, action: str, digest: str) -> dict:
def test_a_mismatch_at_a_smaller_rung_survives_the_later_rungs(tmp_path):
- # A standard-tier repetition walks 1K, 10K and 100K. Keyed on the repetition alone, the 100K
- # rows overwrite the 1K ones and a real difference at 1K is reported as a pass.
+ # A standard-tier repetition walks 1K, 10K and 100K; keyed on the repetition alone the 100K
+ # rows overwrite the 1K ones.
rows = [{"row_type": "run_meta", "tier": "standard"}]
for rung, base_digest, treat_digest in (("r1K", "AAA", "BBB"), ("r100K", "CCC", "CCC")):
rows.append(parity_action(f"{rung}.A0.rep0", "settings", base_digest))
@@ -825,12 +810,9 @@ def parity_run(tmp_path: Path, name: str, cells: list[tuple[str, str, str, str]]
def test_instability_measured_at_one_rung_does_not_silence_a_stable_rung(tmp_path):
- # A standard-tier null control at the default --reps 1 walks 1K, 10K and 100K. `settings`
- # races only at 100K, where the mounted thread is largest. Derived over the pooled action
- # name, that single differing observation borrows the other two rungs' observation COUNT,
- # clears min_observations, and marks `settings` unstable everywhere -- so a real DOM
- # regression at 1K, a rung the null control measured clean, prints as expected variation and
- # the command exits 0.
+ # `settings` races only at 100K, but derived over the pooled action name that one observation
+ # borrows the other rungs' COUNT, clears min_observations, and marks it unstable everywhere,
+ # so a real 1K regression prints as expected variation and exits 0.
null = parity_run(
tmp_path,
"null",
@@ -853,10 +835,8 @@ def test_instability_measured_at_one_rung_does_not_silence_a_stable_rung(tmp_pat
def test_instability_measured_at_a_rung_still_silences_that_rung(tmp_path):
- # The other direction, so the scoping above cannot pass by never silencing anything. With two
- # repetitions the null control has the observations to MEAN it at 100K, and a mismatch there
- # is expected variation -- while the same action at 1K, measured clean, still carries a
- # verdict.
+ # The other direction, so the scoping cannot pass by never silencing anything: with two
+ # repetitions the null control can mean it at 100K while 1K still carries a verdict.
null = parity_run(
tmp_path,
"null2",
@@ -878,13 +858,11 @@ def test_instability_measured_at_a_rung_still_silences_that_rung(tmp_path):
def test_a_declared_unstable_action_still_holds_at_every_rung(tmp_path):
- # A declared entry carries a MECHANISM that is a property of the action, so it is not scoped
- # to a rung and a null control is not needed to honour it.
- #
- # NOT 1 is what this test is about: the difference is filed as expected variation rather than
- # as a stable difference. It is 2 and not 0 because the only pair in this payload landed in
- # that bucket, so nothing was compared at all -- see `report`'s `matched == 0` guard, which
- # refuses to exit 0 over a run where the digest never produced a verdict.
+ # A declared entry carries a MECHANISM that is a property of the action, so it is not
+ # rung-scoped and needs no null control.
+ # NOT 1: the difference is filed as expected variation rather than a stable difference. It is
+ # 2 because the only pair landed in that bucket, so `report`'s `matched == 0` guard refuses to
+ # exit 0 over a run that produced no verdict.
rows = [{"row_type": "run_meta", "tier": "standard"}]
rows.append(parity_action("r1K.base.rep0", "stop_generation", "A"))
rows.append(parity_action("r1K.treatment.rep0", "stop_generation", "B"))
@@ -927,11 +905,9 @@ def resumed_parity(tmp_path: Path, name: str, retry_session: str) -> Path:
def test_a_resumed_arm_is_not_paired_with_its_partner_from_the_old_session(tmp_path):
- # `--resume` re-runs the arm that died under a NEW session id in the SAME shard, so the
- # completed partner is left under the old one. Keyed on the repetition alone the two pair, and
- # every session-scoped volatile the normaliser missed reads as a difference between the arms.
- # Two of them at one rung are enough for `derive_unstable` to call `settings` unstable AT THAT
- # RUNG, and a real DOM regression at 100K then prints as expected variation and exits 0.
+ # `--resume` re-runs the dead arm under a NEW session id, so session-scoped volatiles read as
+ # arm differences; two at one rung make `derive_unstable` call `settings` unstable there and a
+ # real 100K regression exits 0.
null = resumed_parity(tmp_path, "null_resumed", "s2")
unstable, _derived, _checks = U.unstable_set([null])
assert ("r100K", "settings") not in unstable
@@ -942,31 +918,27 @@ def test_a_resumed_arm_is_not_paired_with_its_partner_from_the_old_session(tmp_p
def test_a_cross_session_pair_carries_no_verdict_in_either_direction(tmp_path):
- # And it is not silently dropped either. Both arms ran, so the reader is told the surface went
- # unmeasured rather than being shown a pass -- which is the distinction the NOT COMPARABLE
- # outcome exists to make.
+ # And not silently dropped: both arms ran, so the reader is told the surface went unmeasured,
+ # which is what NOT COMPARABLE exists to say.
null = resumed_parity(tmp_path, "blind_resumed", "s2")
verdicts = {r["verdict"] for _a, _s, _c, r in U.compare_all([null])[0]}
assert verdicts == {U.P.NOT_COMPARABLE}
- # NOT 1, because a cross-session pair is not a stable difference, and NOT 0 either: every pair
- # here is NOT COMPARABLE, so the payload carries no verdict in either direction and `report`
- # says so with 2 rather than letting CI go green on a run that compared nothing.
+ # NOT 1 (a cross-session pair is not a stable difference) and NOT 0 either: every pair is NOT
+ # COMPARABLE, so 2 rather than letting CI go green on a run that compared nothing.
assert U.report([null], "t", U.UNSTABLE_ACTIONS) == 2
def test_a_parity_arm_resumed_inside_the_same_session_still_pairs(tmp_path):
- # The other direction, so the refusal above cannot pass by rejecting every resumed payload.
- # Two attempts in ONE session are still one session, the pairs are real, and the instability
- # they show is derived exactly as before.
+ # The other direction: two attempts in ONE session are one session, the pairs are real, and
+ # instability derives as before.
null = resumed_parity(tmp_path, "same_session", "s1")
unstable, _derived, _checks = U.unstable_set([null])
assert ("r100K", "settings") in unstable
def test_a_parity_payload_with_no_session_ids_pairs_exactly_as_before(tmp_path):
- # Payloads recorded before session ids existed carry none, so both arms resolve to "" and the
- # new key term is inert. A refusal that also rejected these would blind the tool to every
- # older run.
+ # Pre-session-id payloads resolve both arms to '', so the term is inert; refusing them would
+ # blind the tool to every older run.
path = parity_run(tmp_path, "legacy_parity", [("r1K", "rep0", "A", "B")])
assert U.report([path], "t", U.UNSTABLE_ACTIONS) == 1
@@ -1019,10 +991,8 @@ def resumed_both_arms(tmp_path: Path, name: str) -> Path:
def test_a_superseded_attempt_is_not_a_second_parity_observation(tmp_path):
- # One repetition ran twice, so the payload holds two comparable pairs for it. Counted as two,
- # the differing dead attempt and the matching retry are exactly `min_observations` and
- # `derive_unstable` marks `settings` unstable at 100K on the strength of a reading the run
- # itself threw away.
+ # One repetition ran twice, so counting the dead attempt gives exactly `min_observations` and
+ # marks `settings` unstable on a reading the run threw away.
null = resumed_both_arms(tmp_path, "null_both_arms")
assert len(U.collect([null])["pairs"]) == 1
@@ -1034,18 +1004,15 @@ def test_a_superseded_attempt_is_not_a_second_parity_observation(tmp_path):
def test_a_superseded_attempt_does_not_silence_a_real_parity_regression(tmp_path):
- # The consequence, end to end. Scored against that unstable set, a genuine DOM difference on
- # `settings` at 100K prints under "expected to vary" and the gate exits 0 on a regression.
+ # The consequence end to end: a genuine 100K difference then prints under 'expected to vary' and the gate exits 0.
unstable, _derived, _checks = U.unstable_set([resumed_both_arms(tmp_path, "null_silencer")])
regressed = parity_run(tmp_path, "mine_both_arms", [("r100K", "rep0", "X", "REGRESSED")])
assert U.report([regressed], "t", unstable) == 1
def test_two_repetitions_of_one_pair_are_still_two_parity_observations(tmp_path):
- # The control: superseding is keyed on the ATTEMPT, not on the cell id, so two repetitions of
- # one rung inside one session remain two independent observations and still derive as
- # unstable. A filter that kept only one reading per cell would pass the two tests above by
- # deleting the evidence they are meant to preserve.
+ # The control: superseding keys on the ATTEMPT, not the cell id, so two repetitions in one
+ # session stay two observations.
rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "session_id": "s1"}]
for rep, treat in (("rep0", "Y"), ("rep1", "Z")):
drained_arm(rows, "base", rep, "s1", "X", True)
@@ -1059,9 +1026,7 @@ def test_two_repetitions_of_one_pair_are_still_two_parity_observations(tmp_path)
def test_an_attempt_that_was_never_re_run_still_carries_its_parity_verdict(tmp_path):
- # The other control: a cell that died and was never resumed is the LATEST attempt at itself,
- # so its rows stay. Dropping every incomplete cell instead would blind the tool to the runs
- # that have the most to say, and would silence the difference below rather than report it.
+ # The other control: a cell that died and was never resumed is the latest attempt at itself, so its rows stay.
rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "session_id": "s1"}]
drained_arm(rows, "base", "rep0", "s1", "STABLE", True)
drained_arm(rows, "treatment", "rep0", "s1", "DIFFERENT", False)
@@ -1090,33 +1055,24 @@ def two_tier_parity(tmp_path: Path, name: str, fast: tuple[str, str], standard:
def test_a_null_control_holding_two_tiers_is_refused_before_the_set_is_derived(tmp_path):
- # THE POOLING THIS USED TO DEMONSTRATE IS GONE, and it was fixed underneath rather than here.
- # The setup assertion used to be `("r100K", "settings") in unstable`: one differing 100K pair
- # from the fast film and one matching 100K pair from the standard film pooled into exactly
- # `min_observations`, marking the action unstable on the strength of two different films.
- #
- # `latest_attempt_rows` now keys an attempt on any attempt-stamped row rather than on the
- # terminal cell row alone, and the two films write the SAME cell ids, so they are read as two
- # attempts of one cell and the superseded one is dropped before pairing. Only the standard
- # film's matching pair survives, so there is one observation and nothing is derived. Asserting
- # the old pooling here would be asserting a bug that no longer exists.
+ # THE POOLING THIS DEMONSTRATED IS GONE, fixed underneath rather than here.
+ # `latest_attempt_rows` now keys on any attempt-stamped row, and the two films write the SAME
+ # cell ids, so the superseded one is dropped before pairing and nothing is derived.
null = two_tier_parity(tmp_path, "null_mixed", ("X", "Y"), ("Q", "Q"))
unstable, _derived, _checks = U.unstable_set([null])
assert ("r100K", "settings") not in unstable
- # The refusal is kept regardless, as the layer that does not depend on the two films colliding
- # on a cell id. Two tiers that walk different ladders need not collide at all, and a set
- # derived across films is wrong whether or not de-duplication happened to absorb it.
+ # The refusal is kept as the layer that does not depend on the two films colliding on a cell
+ # id: a set derived across films is wrong either way.
with pytest.raises(SystemExit) as exc:
U.main([str(tmp_path / "mine_any"), "--null", str(null.parent)])
assert "more than one tier" in str(exc.value)
def test_two_mixed_tier_sets_do_not_pass_by_matching_each_other(tmp_path):
- # The case the tier-mismatch WARNING cannot see: both sides were re-run at the other tier, so
- # both sets are {fast, standard}, they compare EQUAL, and no warning fires. The pooled null
- # control then silences `settings` at 100K and the real 100K regression in the payload prints
- # as expected variation with the command exiting 0.
+ # The case the tier-mismatch WARNING cannot see: both sides re-run at the other tier, so both
+ # sets are {fast, standard}, compare EQUAL, and the pooled null control silences a real 100K
+ # regression.
null = two_tier_parity(tmp_path, "null_both", ("X", "Y"), ("Q", "Q"))
mine = two_tier_parity(tmp_path, "mine_both", ("Q", "Q"), ("Q", "REGRESSED"))
assert U.tier_of([null]) == U.tier_of([mine]) == {"fast", "standard"}
@@ -1127,9 +1083,7 @@ def test_two_mixed_tier_sets_do_not_pass_by_matching_each_other(tmp_path):
def test_a_payload_holding_two_tiers_is_refused_even_with_no_null_control(tmp_path):
- # The declared unstable set is not derived from anything, so nothing is pooled -- but the
- # payload's own 100K pairs still come from two films, and reporting them as repetitions of one
- # measurement is the same misreading.
+ # A declared set is derived from nothing, but the payload's own 100K pairs still come from two films.
mine = two_tier_parity(tmp_path, "mine_alone", ("Q", "Q"), ("Q", "REGRESSED"))
with pytest.raises(SystemExit) as exc:
U.main([str(mine.parent)])
@@ -1137,8 +1091,7 @@ def test_a_payload_holding_two_tiers_is_refused_even_with_no_null_control(tmp_pa
def test_one_tier_on_each_side_still_scores_in_both_directions(tmp_path):
- # The control, so the refusal cannot pass by rejecting every run. A single-tier null control
- # and a single-tier payload score exactly as before, and a real regression still exits 1.
+ # The control: single-tier null control and payload score as before, and a real regression still exits 1.
null = parity_run(tmp_path, "null_one", [("r100K", "rep0", "Q", "Q")])
clean = parity_run(tmp_path, "mine_clean", [("r100K", "rep0", "Q", "Q")])
regressed = parity_run(tmp_path, "mine_bad", [("r100K", "rep0", "Q", "REGRESSED")])
@@ -1147,14 +1100,10 @@ def test_one_tier_on_each_side_still_scores_in_both_directions(tmp_path):
def test_a_tier_mismatch_between_two_single_tier_sets_is_refused(tmp_path, capsys):
- # This used to warn and then score anyway, which is the worst of the two options: the warning
- # said the derived set does not transfer, and then the run was scored against it regardless.
- # A set that does not transfer is not a weaker excuse than a real one, it is an arbitrary one.
- #
- # Exit 2, not 1. The payload below does carry a regression, so the old assertion of exit 1 was
- # passing for a reason unrelated to the tier check. Refusing to answer and reporting a parity
- # failure have to be distinguishable, or a refusal sends somebody hunting for a UI change that
- # was never measured.
+ # This used to warn and score anyway, the worst option: a set that does not transfer is
+ # arbitrary, not merely weaker.
+ # Exit 2, not 1: the payload does carry a regression, so the old exit-1 assertion passed for
+ # an unrelated reason, and refusing to answer must be distinguishable from a parity failure.
null = write(
tmp_path,
"null_fast",
@@ -1170,10 +1119,8 @@ def test_a_tier_mismatch_between_two_single_tier_sets_is_refused(tmp_path, capsy
def test_a_corpus_mismatch_between_two_valid_sides_is_refused(tmp_path, capsys):
- # The likelier of the two in practice, and the one that had no check at all: each side is a
- # perfectly valid single corpus, so `one_corpus` passes on both, and the null's set was still
- # applied to a payload that rendered a different thread. Which actions race is a property of
- # the thread the film drove, so that set describes something the payload never displayed.
+ # The likelier case with no check at all: each side is a valid single corpus, so `one_corpus`
+ # passes on both while the null's set describes a thread the payload never displayed.
def at(name, corpus, cells):
rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "corpus_hash": corpus}]
for rung, rep, base_digest, treat_digest in cells:
@@ -1191,8 +1138,7 @@ def test_a_corpus_mismatch_between_two_valid_sides_is_refused(tmp_path, capsys):
def test_a_side_recorded_before_corpus_hashes_existed_is_still_scored(tmp_path, capsys):
- # An absent hash is not a disagreement. Refusing it would reject every payload recorded before
- # the field existed, which is the whole archive.
+ # An absent hash is not a disagreement; refusing it would reject the whole archive.
rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "corpus_hash": "v1"}]
rows.append(parity_action("r100K.base.rep0", "settings", "Q"))
rows.append(parity_action("r100K.treatment.rep0", "settings", "Q"))
@@ -1203,8 +1149,8 @@ def test_a_side_recorded_before_corpus_hashes_existed_is_still_scored(tmp_path,
def test_main_prints_a_mixed_unstable_set_without_dying(tmp_path, capsys):
- # The set now holds bare action names and (rung, action) pairs together, and `sorted()` over
- # the two raises TypeError. The one place that formats it is the run header.
+ # The set holds bare action names and (rung, action) pairs together, so `sorted()` raises
+ # TypeError; the run header is the one place that formats it.
null = parity_run(tmp_path, "nullm", [("r1K", "rep0", "X", "Y"), ("r1K", "rep1", "X", "Z")])
mine = parity_run(tmp_path, "minem", [("r1K", "rep0", "Q", "Q")])
assert U.main([str(mine.parent), "--null", str(null.parent)]) == 0
@@ -1274,8 +1220,7 @@ def test_a_probe_named_in_a_later_run_meta_is_still_caught(tmp_path):
def test_a_failed_probe_free_gate_is_enough_on_its_own(tmp_path):
- # Two independent records of one fact, so a payload from a version that emits only the gate
- # is still refused.
+ # Two independent records of one fact, so a payload emitting only the gate is still refused.
path = probe_payload(tmp_path, "gated", None)
with path.open("a", encoding = "utf-8") as fh:
fh.write(
@@ -1323,8 +1268,7 @@ def test_the_report_path_refuses_a_probed_payload_too(tmp_path):
def test_a_null_probe_field_is_the_ordinary_scorable_case(tmp_path):
- # Explicit null and absent must both score, or every payload written before the field
- # existed becomes unreadable.
+ # Explicit null and absent must both score, or every pre-field payload becomes unreadable.
pooled, _ = F.load([probe_payload(tmp_path / "explicit", "clean", None)])
assert pooled["message_menu.open_close_ms"]
pooled, _ = F.load([payload(tmp_path / "absent", "clean", [(1000.0, 900.0)] * 4)])
@@ -1376,7 +1320,6 @@ def test_the_composer_click_does_not_set_the_frame_floor(tmp_path):
# ── the documented loop runs liveness before it reads a timing ───────
-
LOOP_DOC = Path(__file__).resolve().parents[2] / "CONTRIBUTING-perf.md"
@@ -1459,11 +1402,10 @@ def test_the_documented_loop_asserts_liveness_before_it_reads_any_timing():
def test_floor_table_prints_a_clean_verdict_from_a_run_that_liveness_voids(tmp_path, capsys):
- # WHY THAT ORDER IS LOAD-BEARING, and the reason the line above is a gate rather than a note.
- # The treatment build is 10% faster on the three repetitions it managed, and on the fourth it
- # was so slow that `message_menu` never reached its slot. `paired` matches on the metrics BOTH
- # arms recorded, so that repetition leaves the table with no trace except a smaller `n`, and
- # the survivors print as a clean win over a tight floor.
+ # WHY THAT ORDER IS LOAD-BEARING. The treatment is 10% faster on the three repetitions it
+ # managed; on the fourth `message_menu` never reached its slot, and `paired` matches only
+ # metrics BOTH arms recorded, so that repetition leaves no trace but a smaller `n` and the
+ # survivors print as a clean win.
mine = liveness_payload(
tmp_path, "mine", [(1000.0, 900.0), (1000.0, 900.0), (1000.0, 900.0), (1000.0, None)]
)
@@ -1481,9 +1423,8 @@ def test_floor_table_prints_a_clean_verdict_from_a_run_that_liveness_voids(tmp_p
def test_the_repetition_that_missed_its_slot_is_what_the_verdict_turns_on(tmp_path, capsys):
- # The control for the test above. With the fourth repetition's timing present, the same run is
- # VOID on the same floor: the clean win is an artefact of the reading that went missing, not a
- # property of the change. Every other number in the payload is identical.
+ # The control: with the fourth timing present the same run is VOID on the same floor, so the
+ # clean win is an artefact of the missing reading. Every other number is identical.
mine = liveness_payload(
tmp_path, "mine", [(1000.0, 900.0), (1000.0, 900.0), (1000.0, 900.0), (1000.0, 2500.0)]
)
@@ -1499,11 +1440,10 @@ def test_the_repetition_that_missed_its_slot_is_what_the_verdict_turns_on(tmp_pa
assert cli_main(["--assert-liveness", str(mine)]) == 0
+# A null control is base against base, so its four paired ratios are this machine's noise. The
+# fourth repetition hiccupped.
# ── the null control needs the same liveness gate the treatment run gets ───────
-
-# A null control is base against base, so its four paired ratios are the noise this machine puts
-# between two identical builds. The fourth repetition is the one that hiccupped.
NULL_WITH_A_NOISY_REPETITION = [
(1000.0, 1000.0),
(1010.0, 1012.0),
@@ -1511,12 +1451,12 @@ NULL_WITH_A_NOISY_REPETITION = [
(1000.0, 1300.0),
]
-# The same null control on a run where that fourth repetition was slow enough to miss the slot, so
-# the reading that carried the noise is the reading that never happened.
+# The same null control where that fourth repetition missed its slot, so the reading that
+# carried the noise never happened.
NULL_THAT_MISSED_THE_SLOT = NULL_WITH_A_NOISY_REPETITION[:3] + [(1000.0, None)]
-# A treatment payload that is 10% faster, consistently, with a spread well inside its own effect:
-# it clears gates 2 and 3 outright, so the only thing standing between it and `faster` is gate 1.
+# A consistent 10% with a spread well inside its own effect: it clears gates 2 and 3, so only
+# gate 1 stands between it and `faster`.
MINE_TEN_PERCENT_FASTER = [
(1000.0, 900.0),
(1000.0, 905.0),
@@ -1552,9 +1492,8 @@ def test_the_documented_loop_asserts_liveness_on_the_null_control_too():
def test_the_doc_says_which_commands_in_the_loop_need_a_studio():
- # The loop is now four offline commands and three that drive a browser, so "every command
- # above except the last" bills `--assert-liveness`, `floor_table` and `ui_parity` for
- # credentials none of them read. All three run against a payload file on disk.
+ # Four offline commands and three that drive a browser, so 'every command above except the
+ # last' bills three payload-file readers for credentials they never use.
prose = credentials_prose()
for offline in ("--assert-liveness", "floor_table", "ui_parity", "--report"):
assert offline in prose, (
@@ -1564,27 +1503,24 @@ def test_the_doc_says_which_commands_in_the_loop_need_a_studio():
def test_a_missed_slot_in_the_null_control_prints_noise_as_a_result(tmp_path, capsys):
- # WHY THE NULL CONTROL IS GATED TOO. The floor is `max(|null delta|, null spread)` over the
- # repetitions that survived pairing, and `paired` keys on the metrics BOTH arms recorded, so a
- # null repetition that missed its slot leaves no trace either. The repetition a run drops is
- # the one that was slow enough to miss a budget, which is the noisiest one it had, so the loss
- # is not symmetric: it can only tighten the floor.
+ # WHY THE NULL CONTROL IS GATED TOO. The floor is max(|null delta|, null spread) over paired
+ # repetitions, and a null repetition that missed its slot leaves no trace either: the dropped
+ # one is the noisiest it had, so the loss can only tighten the floor.
mine = liveness_payload(tmp_path, "mine", MINE_TEN_PERCENT_FASTER)
null = liveness_payload(tmp_path, "null", NULL_THAT_MISSED_THE_SLOT)
assert F.main(["--floor", str(null.parent), str(mine.parent)]) == 0
table = capsys.readouterr().out
assert "0.3 faster" in table
assert "1 metric(s) cleared all three gates." in table
- # And the gate the loop used to run only on `outputs/mine` has nothing to say about it: the
- # contributor's own payload is whole. The hole is in the run that set the bar.
+ # And a gate run only on `outputs/mine` has nothing to say: the contributor's payload is
+ # whole, the hole is in the run that set the bar.
assert cli_main(["--assert-liveness", str(mine)]) == 0
assert cli_main(["--assert-liveness", str(null)]) == 1
def test_the_null_repetition_that_kept_its_reading_voids_the_same_result(tmp_path, capsys):
- # The control for the test above, and the reading the missed slot removed. The treatment
- # payload is byte-identical; only the null control differs, by the one repetition it managed to
- # finish. Its floor is 30.1% rather than 0.3%, and the same 10% is noise.
+ # The control: the treatment payload is byte-identical, only the null differs by its one
+ # finished repetition, and its floor of 30.1% rather than 0.3% makes the same 10% noise.
mine = liveness_payload(tmp_path, "mine", MINE_TEN_PERCENT_FASTER)
null = liveness_payload(tmp_path, "null", NULL_WITH_A_NOISY_REPETITION)
assert F.main(["--floor", str(null.parent), str(mine.parent)]) == 0
@@ -1613,7 +1549,7 @@ def test_a_gate_from_an_attempt_that_never_closed_still_refuses_its_cell(tmp_pat
# The first attempt completed cleanly and closed itself.
rows.append(in_session(parity_action(cid, "settings", "STABLE"), "s1"))
rows.append(parity_cell(cid, "treatment", "s1", "rep0", True))
- # The resume re-ran the same cell, lost messages, and was killed before its `cell` row.
+ # The resume re-ran the cell, lost messages, and died before its `cell` row.
rows.append({"row_type": "run_meta", "tier": "standard", "session_id": "s2"})
rows.append(in_session(parity_action(cid, "settings", "LOST"), "s2"))
rows.append(
@@ -1647,8 +1583,7 @@ def gated_then_resumed(tmp_path: Path, name: str) -> Path:
{"row_type": "run_meta", "tier": "standard", "session_id": "s1"},
{"row_type": "cell", "cell_id": "r100K.base.rep0", "session_id": "s1", "completed": True},
timed_action("r100K.base.rep0", "s1", 100.0),
- # The treatment arm lost its thread's middle. It renders fewer rows, so its timing is
- # CHEAPER than a correct cell's, in the direction that flatters the arm.
+ # The treatment arm lost its thread's middle, so it renders fewer rows and times CHEAPER, flattering the arm.
{
"row_type": "cell",
"cell_id": "r100K.treatment.rep0",
@@ -1678,8 +1613,7 @@ def gated_then_resumed(tmp_path: Path, name: str) -> Path:
"completed": False,
},
]
- # The refusal has to hold on the payload BEFORE the resume, or the test below passes on a
- # payload that was never quotable for some other reason.
+ # The refusal must hold on the payload BEFORE the resume, or the test below passes for another reason.
assert F.paired(F.read_rows(before), shard = "b") == {}
return write(tmp_path, name, rows)
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py
index 212699602a0..5b4c0cecf08 100644
--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py
@@ -381,8 +381,6 @@ def test_a_timed_out_rebuild_leaves_the_behavioural_run_with_no_verdict(tmp_path
assert "NOTHING WAS COMPARED" in out, out
assert "ASSERTION FAILED ON ONE ARM" in out, out
assert code == 1, out
- # THE TWO MODES AGREE ON THIS PAYLOAD, checked rather than asserted in prose. The structural
- # report is the one that already had this precedence rule written down, so it is the reference.
assert U.report([shard], "structural on the same payload", frozenset()) == 1
@@ -433,13 +431,12 @@ def test_every_named_first_to_break_action_has_an_invariant():
assert action in B.INVARIANTS, f"{action} has no behavioural invariant declared"
-# ── the four false greens the first review round found ──────────────
-#
-# Every one of these returned SUCCESS before the fix, which is the only reason they are grouped:
-# they are four different routes to a UI verdict of "fine" over a comparison that either found
-# nothing or declined to look.
+# The four false greens the first review round found. Every one returned SUCCESS before the fix:
+# four different routes to a UI verdict of 'fine' over a comparison that either found nothing or
+# declined to look.
+# ── the four false greens the first review round found ──────────────
def test_two_full_mounts_of_different_lengths_is_a_difference_not_an_excuse():
"""THE MOST SERIOUS ONE. Neither arm is windowing, so neither is holding anything back on
purpose, and the treatment renders fewer messages than the base -- a user-visible loss of
@@ -1404,17 +1401,17 @@ def test_a_capture_that_saw_no_thread_at_all_falls_back_on_the_declaration(tmp_p
assert all(mode == U.WINDOWED for mode, _why in U.decide_modes([shard]).values())
-# ── a cell that failed its own completeness gate carries no UI verdict ──
-#
-# `probe_thread_completeness` runs before the film and `record_completeness_gate` writes the
-# verdict against the cell, so a windowed arm that kept its first page and its last one and lost
-# everything between them says so in its own payload. `report/payload.py::excluded_from_rows`
-# drops that cell from the PERFORMANCE score. `ui_parity.py` read no gate row except the windowed
-# declaration, so the same cell's eighteen action rows were still scored for UI parity -- and the
-# visible region is a window on the END of the thread, which such a store still fills, so the
-# pairs matched and `--mode auto` exited 0 over a payload that had already recorded the loss.
+# A cell that failed its own completeness gate carries no UI verdict. `probe_thread_completeness`
+# runs before the film and `record_completeness_gate` writes the verdict against the cell, so a
+# windowed arm that kept its first and last page and lost the middle says so in its own payload,
+# and `report/payload.py::excluded_from_rows` drops it from the PERFORMANCE score. `ui_parity.py`
+# read no gate row except the windowed declaration, so the same cell's eighteen action rows were
+# still scored, and the visible region is a window on the END of the thread, which such a store
+# still fills, so the pairs matched and `--mode auto` exited 0 over a payload that had already
+# recorded the loss.
+# ── a cell that failed its own completeness gate carries no UI verdict ──
def _completeness_gate(
cell_id,
passed,
@@ -2091,8 +2088,8 @@ def test_the_coverage_floor_sums_the_windowed_and_structural_halves(tmp_path, ca
assert "TOO LITTLE COMPARED" not in out, out
# 1 is the digest regression the fixture carries at the 1K rung, so the floor did not mask it.
assert code == 1, out
- # And one more than the run can reach refuses it, which is what proves the count above is 2
- # rather than the 3 a sum of the three reports would have produced.
+ # And one more than the run can reach refuses it, which proves the count above is 2 rather than the
+ # 3 a sum of the three reports would have produced.
code = U.main([str(tmp_path / "sums"), "--min-compared", "3"])
out = capsys.readouterr().out
assert "TOO LITTLE COMPARED: 2 of 2" in out, out
diff --git a/tests/studio/studiobench/sweep/ui_parity.py b/tests/studio/studiobench/sweep/ui_parity.py
index b5ddf493b66..590a0a6478b 100644
--- a/tests/studio/studiobench/sweep/ui_parity.py
+++ b/tests/studio/studiobench/sweep/ui_parity.py
@@ -52,7 +52,6 @@ from pathlib import Path
from typing import Any, Optional
if __package__ in (None, ""): # pragma: no cover
- # Running the file directly rather than as a module, which is the first thing anyone tries.
sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
from tests.studio.studiobench.analysis import behaviour as B # noqa: E402
@@ -64,57 +63,33 @@ from tests.studio.studiobench.scoring.from_payload import ( # noqa: E402
)
# The DECLARED unstable set, each entry carrying its mechanism. It lives in the studiobench
-# package rather than here so that a test can require a mechanism for every entry, and so that
-# this script and the null control cannot drift into two different opinions about which actions
-# are trusted.
-#
-# Declared is not the same as true. `--null` replaces it with the set MEASURED from a base-vs-base
-# run and reports every disagreement in both directions, which is the only way an entry here gets
-# audited rather than inherited.
+# package so a test can require a mechanism per entry and this script cannot drift from the null
+# control. Declared is not true: `--null` replaces it with the MEASURED set.
UNSTABLE_ACTIONS = frozenset(P.UNSTABLE_ACTIONS)
-#: THE RUN'S OWN DECLARATION that an arm mounts a window, as `__main__.py` records it: the gate row
-#: `windowed_readiness:{arm}` written when `--windowed-arm` names that side, and `readiness.mode` on
-#: every cell row. Matched as literal strings rather than imported from `runtime/readiness.py`
-#: because this script reads payloads written by other checkouts of the tool, where what has to be
-#: matched is the string in the file and not the constant in this working tree.
+#: THE RUN'S OWN DECLARATION that an arm mounts a window, as `__main__.py` records it. Matched as
+#: literal strings rather than imported, because this script reads payloads from other checkouts.
+# From `runtime/readiness.py`.
WINDOWED_GATE = "windowed_readiness:"
MODE_WINDOWED = "windowed"
-#: THE CELL'S OWN VERDICT THAT IT LOST MESSAGES. `runtime/session.py::record_completeness_gate`
-#: writes this row, against the cell, when the traversal to the top of the thread could not
-#: establish that the arm still holds the whole conversation -- the head marker never mounted, or
-#: the ordinals recorded on the way up have a hole in them. `report/payload.py::excluded_from_rows`
-#: reads it as a failed gate and drops the cell from the PERFORMANCE score; this file read no gate
-#: rows at all except the windowed declaration, so the same cell's action rows were still scored
-#: for UI parity. An arm that keeps the first page and the last one and has lost the middle then
-#: retains its first and last visible windows, and its bottom rows, its scroll extent and an
-#: action-specific invariant such as `select_text` can all match: `--mode auto` printed a visible
-#: pass and a behavioural pass and exited 0 over a payload that already recorded the loss.
-#: Matched as a literal string for the same reason `WINDOWED_GATE` is.
+#: THE CELL'S OWN VERDICT THAT IT LOST MESSAGES, written by `record_completeness_gate` when the
+#: traversal could not establish the arm still holds the whole conversation. `excluded_from_rows`
+#: drops such a cell from the PERFORMANCE score, but this file read no gate rows except the
+#: windowed declaration, so an arm that lost the middle still printed a pass and exited 0.
COMPLETENESS_GATE = "thread_complete"
-#: THE CELL'S OWN VERDICT THAT IT STOPPED FOLLOWING THE REPLY IT WAS MEASURING. `runtime/session`
-#: writes this row, against the cell, when the thread did not stay pinned for enough of the
-#: streaming phase or too little of that phase had the stream attached at all. It belongs beside
-#: the completeness gate rather than in a category of its own, because it invalidates the same
-#: thing: a reply that scrolled out of the viewport and was unmounted by a windowed arm stops
-#: costing anything to render, and the action rows measured around it are a reading of an arm that
-#: was not showing the thing under test. `select_text` counts and bottom rows can still match
-#: across such a pair, so `--mode auto` printed a behavioural pass and exited 0 over a payload
-#: that had already recorded the arm losing the stream. Matched as a literal string for the same
-#: reason the two gates above it are.
+#: THE CELL'S OWN VERDICT THAT IT STOPPED FOLLOWING THE REPLY IT WAS MEASURING, written when the
+#: thread did not stay pinned for enough of the streaming phase. It invalidates the same thing as
+#: the completeness gate: a reply scrolled out and unmounted stops costing anything to render.
FOLLOW_GATE = "follows_the_stream"
#: The per-cell gates whose failure means this cell's action rows are not a reading of the build.
-#: A gate that only qualifies one column -- `timer_clamp`, whose failure leaves every column but
-#: `busy_pct` standing -- is deliberately NOT here; see `runtime/ab.py::INVALIDATING_CELL_GATES`,
-#: which is the same list for the performance side.
+#: A gate that only qualifies one column (`timer_clamp`) is deliberately NOT here.
INVALIDATING_GATES: tuple[str, ...] = (COMPLETENESS_GATE, FOLLOW_GATE)
-#: How each is named and what its failure means, for the refusal a reader actually sees. The gate
-#: is described rather than spelled with its row name, because this string is read by somebody
-#: deciding whether a run is usable, not by a parser.
+#: How each is named and what its failure means, for the refusal a reader actually sees. Described
+#: rather than spelled with its row name, because a person reads this deciding if a run is usable.
_GATE_LABEL: dict[str, str] = {
COMPLETENESS_GATE: "completeness",
FOLLOW_GATE: "stream-follow",
@@ -124,17 +99,15 @@ _GATE_REASON: dict[str, str] = {
FOLLOW_GATE: "the arm stopped following the streamed reply",
}
-#: How one action pair is scored. PER PAIR, never per payload: the readiness gate deliberately
-#: permits a payload to hold fully mounted small rungs and windowed large ones, so a payload-wide
-#: decision lets one windowed 100K capture suppress the structural digest on every fully mounted 1K
-#: pair beside it, and an ordinary DOM regression at those rungs is then never looked for.
+#: How one action pair is scored. PER PAIR, never per payload: the readiness gate permits fully
+#: mounted small rungs beside windowed large ones, so a payload-wide decision lets one windowed
+#: 100K capture suppress the structural digest on every 1K pair.
WINDOWED = "windowed"
STRUCTURAL = "structural"
-#: The arms a pair is expected to have. Consulted BY NAME rather than by walking the rows that are
-#: present, because the case that matters is the arm with NO row: an arm that died before it emitted
-#: an action row leaves `sides` holding only the other one, and reading the declaration off the rows
-#: present asks the surviving arm whether the missing one was windowed.
+#: The arms a pair is expected to have, consulted BY NAME rather than by walking the rows present:
+#: an arm that died before emitting an action row leaves `sides` holding only the other one, and
+#: reading off the rows asks the surviving arm whether the missing one was windowed.
ARMS = ("base", "treatment")
@@ -312,19 +285,14 @@ def incomplete_cells(paths: list[Path]) -> dict[str, str]:
if keep is not None and r.get("session_id") not in (None, keep):
continue
detail = r.get("detail") if isinstance(r.get("detail"), dict) else {}
- # NOT MEASURED IS NOT FAILED, and this job draws the line in exactly the same place
- # the performance side does: `gate_detail_is_unmeasured` is THE definition, imported
- # rather than restated. The predicate had been copied here, which is the drift
- # `INVALIDATING_CELL_GATES` was centralised to prevent, arriving one level down.
- #
- # It matters most for the stream-coverage clause: that shortfall is a property of the
- # scene schedule, identical on both arms, and this job compares DOM digests the
- # shortfall does not touch at all. It refused 32 of 32 pairs here, the null control
- # included.
+ # NOT MEASURED IS NOT FAILED, drawn where the performance side draws it:
+ # `gate_detail_is_unmeasured` is THE definition, imported rather than restated. It matters most
+ # for the stream-coverage clause, identical on both arms and untouched by a DOM digest, which
+ # refused 32 of 32 pairs here including the null control.
if gate_detail_is_unmeasured(detail):
continue
- # FIRST FAILURE WINS, so a cell that failed both is not relabelled by whichever row
- # happens to come second in the file.
+ # FIRST FAILURE WINS, so a cell that failed both is not relabelled by whichever row comes second
+ # in the file.
if cid in out:
continue
out[cid] = (
@@ -397,59 +365,35 @@ def collect(
"""
out: dict[tuple, dict] = collections.defaultdict(dict)
attempted = missing = 0
- # TWO DIFFERENT "INCOMPLETE"S MEET HERE, and they are kept apart on purpose. `gate_failures`
- # is per cell and comes from a gate the payload FAILED, and those rows are stamped and kept.
- # `dropped_incomplete` counts action rows discarded because their cell never wrote a `cell`
- # row at all, which only happens under `require_complete`. Sharing one name between them
- # would make the first silently overwrite the second on the first shard scanned.
+ # TWO DIFFERENT "INCOMPLETE"S MEET HERE: `gate_failures` is per cell from a gate the payload
+ # FAILED and those rows are kept, while `dropped_incomplete` counts action rows discarded because
+ # their cell never wrote a `cell` row. One shared name would let the first overwrite the second.
+ # Action rows come from `latest_attempt_rows`.
+ # Which only happens under `require_complete`.
dropped_incomplete = 0
no_cell_rows = 0
for path in paths:
shard = path.parent.name
gate_failures = incomplete_cells([path])
raw_rows = rows(path)
- # A CELL THAT DID NOT FINISH IS NOT AN OBSERVATION -- ON THE NULL CONTROL ONLY, and the
- # asymmetry is the point rather than an omission.
- #
- # Action rows are emitted as the film runs; the `cell` row is written when it ends. So an
- # interrupted or in-flight cell leaves a complete-looking set of action rows that nothing
- # owns, and they pair and compare exactly like real ones.
- #
- # ON THE NULL they must be dropped. The null's job is to say which actions are unstable,
- # and an under-observed action reads as stable or undetermined instead -- which NARROWS
- # the excuse set, so the result's ordinary noise then has nothing to hide behind. Scoring
- # this sweep's own half-written film as a null control turned 0 false alarms into 5 of 30
- # for exactly that reason. `floor_table.cell_metrics` refuses the same rows on the timing
- # side.
- #
- # ON THE RESULT they must be kept, and `test_an_attempt_that_was_never_re_run_still_
- # carries_its_parity_verdict` holds that: a cell that died is the latest attempt at
- # itself, and a difference it observed is still a difference. Dropping it would silence a
- # regression, which is the worse direction of the two. So the conservative choice has
- # opposite signs on the two sides, because "conservative" means a different thing on each.
- #
- # READ THROUGH THE SAME FILTER THE ACTION ROWS ARE READ THROUGH. Taking `completed` from
- # the raw stream while taking the action rows from `latest_attempt_rows` is the guard
- # answering about a different attempt than the one it is guarding. `_resume_set` names the
- # path that gets there without anybody doing anything unusual: an A/B pair is re-run WHOLE
- # (`ab.skippable_cells`), so a resume re-runs an arm that had already succeeded, and if
- # that retry is interrupted the payload holds a completed row and a LATER, unfinished row
- # under the same deterministic `cell_id`. Read raw, the dead retry's action rows are
- # admitted on the strength of the completion the run before them earned. On the null that
- # is the worst direction: one valid repetition plus this one reaches `min_observations`,
- # the action is called unstable, and that excuse hides a real result difference.
- # Per file: the payload is append-only within one shard, and a cell id is reused across
- # shards, so superseding has to be resolved inside the stream that appended it.
+ # A CELL THAT DID NOT FINISH IS NOT AN OBSERVATION -- ON THE NULL CONTROL ONLY. Action rows are
+ # emitted as the film runs and the `cell` row when it ends, so an interrupted cell leaves a
+ # complete-looking set nothing owns. ON THE NULL they must be dropped: an under-observed action
+ # reads as stable and NARROWS the excuse set (0 false alarms became 5 of 30). ON THE RESULT they
+ # must be kept, because a difference a dead cell observed is still a difference. Read through the
+ # SAME filter as the action rows, or the guard answers about a different attempt. Per file, since
+ # a cell id is reused across shards, so superseding has to be resolved inside the stream that
+ # appended it. The keep-on-the-result direction is held by
+ # test_an_attempt_that_was_never_re_run_still_carries_its_parity_verdict.
kept_rows = latest_attempt_rows(raw_rows)
completed = {
r.get("cell_id")
for r in kept_rows
if r.get("row_type") == "cell" and r.get("completed")
}
- # A payload with no `cell` rows at all predates them, or is a fixture. Falling back is
- # right; falling back SILENTLY is how a guard stops guarding, so it is counted and said.
- # Asked of the raw stream on purpose: the question is whether this recorder ever wrote
- # cell rows, not whether the surviving attempt happened to reach one.
+ # A payload with no `cell` rows predates them, or is a fixture. Falling back is right; falling
+ # back SILENTLY is how a guard stops guarding, so it is counted and said. Asked of the raw
+ # stream: the question is whether this recorder ever wrote cell rows.
has_cell_rows = any(r.get("row_type") == "cell" for r in raw_rows)
if not has_cell_rows:
no_cell_rows += 1
@@ -463,8 +407,8 @@ def collect(
# BEFORE the tally, so a selected report's counts are counts of the pairs it scored.
if select is not None and key not in select:
continue
- # And before the tally for the same reason, but after the selection: a row this
- # report never asked for is not an incomplete observation of it.
+ # And before the tally for the same reason, but after the selection: a row this report never
+ # asked for is not an incomplete observation of it.
if require_complete and has_cell_rows and r.get("cell_id") not in completed:
dropped_incomplete += 1
continue
@@ -473,9 +417,8 @@ def collect(
attempted += 1
else:
missing += 1
- # STAMPED, NOT DROPPED, for the same reason a failed capture is kept: the comparison
- # layer has to be able to say that this pair carries no verdict, and a row deleted here
- # would leave the pair looking like an action that simply never ran.
+ # STAMPED, NOT DROPPED: the comparison layer has to be able to say this pair carries no verdict,
+ # and a deleted row would leave the pair looking like an action that never ran.
if cid in gate_failures:
r = dict(r)
r["_incomplete"] = gate_failures[cid]
@@ -600,17 +543,11 @@ def decide_modes(paths: list[Path]) -> dict[tuple, tuple[str, str]]:
if len(sides) == 2 and all(_mount_measured(r.get("parity")) for r in sides.values()):
decided[key] = (STRUCTURAL, "")
continue
- # EITHER EXPECTED ARM, INCLUDING ONE THAT HAS NO ROW AT ALL.
- #
- # The declaration fallback used to be read off the rows this pair actually has, which
- # covers a declared-windowed arm whose captures failed and misses the case one step worse:
- # the arm failed before it emitted an action row, so `sides` holds only the other side, the
- # loop never asks about the arm that is missing, and the pair is scored structurally. In a
- # mixed-rung payload the fully mounted pairs then supply `matched > 0`, the missing windowed
- # pair is filed as structurally NOT COMPARABLE, and the command exits 0 without ever running
- # a windowed report for the rung that has no verdict at all.
- #
- # AND ONLY THIS PAIR'S OWN SHARD DECLARES IT. See `declared_windowed`.
+ # EITHER EXPECTED ARM, INCLUDING ONE WITH NO ROW AT ALL. Reading the declaration off the rows a
+ # pair has misses the case one step worse: the arm failed before emitting an action row, so the
+ # loop never asks about it, the pair is scored structurally, and the command exits 0 with no
+ # verdict for that rung. And only this pair's own shard declares it.
+ # See `declared_windowed`.
why = ""
shard = key[0]
for label in ARMS:
@@ -763,10 +700,10 @@ def behaviour_report(
)
continue
out = B.compare_behaviour(sides["base"], sides["treatment"])
- # THE SAME TWO QUESTIONS THE STRUCTURAL PATH ASKS, off the same functions; see
- # `compare_all_with`. `compare_behaviour` cannot see either shape: an action not performed
- # on one arm records no quantities to disagree about, and one that ran and failed its own
- # assertion records perfectly ordinary ones.
+ # THE SAME TWO QUESTIONS THE STRUCTURAL PATH ASKS, off the same functions: `compare_behaviour`
+ # cannot see either shape, since an action not performed on one arm records no quantities to
+ # disagree about and one that failed its own assertion records ordinary ones.
+ # `compare_all_with` is the other half.
idle_ = P.execution_verdict(sides["base"], sides["treatment"])
out["one_sided"] = (idle_ or {}).get("one_sided") or ""
out["idle_reason"] = (idle_ or {}).get("idle_reason") or ""
@@ -779,14 +716,11 @@ def behaviour_report(
print(f"\n{label} (BEHAVIOURAL MODE)")
print(f" CLAIM: {P.CLAIM_BEHAVIOURAL}.")
# Through the helper, not straight off the table: this mode's line names the coverage band it
- # enforces, and the band lives in `behaviour`. Printing the raw template would emit the
- # placeholders themselves and quietly stop reporting the numbers being applied.
+ # enforces, and printing the raw template would emit the placeholders instead of the numbers.
print(f" POLICY: {P.behaviour_policy(B.MIN_CLIPBOARD_COVERAGE, B.MAX_CLIPBOARD_COVERAGE)}.")
- # WHICH REASON, and only if it is true. Forced behavioural mode on a payload that mounts its
- # whole thread -- which is exactly how a NULL CONTROL is scored on the same scale as the
- # windowed arm it is the control for -- used to print "one arm of this payload mounts a window
- # of the thread" about a payload where neither arm does. A control that misdescribes itself in
- # its own heading is not a small thing when the heading is what gets read.
+ # WHICH REASON, and only if it is true: forced behavioural mode on a fully mounted payload --
+ # which is how a NULL CONTROL is scored -- used to print "one arm mounts a window of the thread"
+ # about a payload where neither does.
if windowed:
print(
" One arm of this payload mounts a window of the thread, so the structural DOM digest"
@@ -858,36 +792,23 @@ def behaviour_report(
names = sorted({a for a, _s, _c, _v in idle})
print(f"\n NOT EXERCISED: {', '.join(names)}")
- # `not broken` IS THE PRECONDITION THE PARAGRAPH BELOW ARGUES FROM, and it used to be assumed
- # rather than checked. An all-BROKEN payload reaches `matched == 0` too -- the two counters are
- # independent -- so a run that had just listed its broken invariants went on to say that not
- # one invariant was evaluated and that it carried no failure, immediately before returning 1.
- # The artifact contradicted its own results and sent the reader off to find out why actions
- # that had run did not run. `report` (`elif decided == 0`) and `visible_report` (`and not
- # differing`) both already gate this banner on their own failure bucket; this is that gate.
- #
- # NOT GATED ON `build_failed`, which is the other half of the return condition and is NOT the
- # same case: those two shapes are collected from pairs whose invariants were unreadable, so
- # "not one behavioural invariant was evaluated" stays literally true when one fires, and the
- # run is a failure for a directional reason rather than an invariant one. See
- # `test_a_timed_out_rebuild_leaves_the_behavioural_run_with_no_verdict`, which asserts exactly
- # that pairing -- NOTHING WAS COMPARED together with exit 1 -- on purpose.
+ # `not broken` IS THE PRECONDITION THE PARAGRAPH BELOW ARGUES FROM: an all-BROKEN payload reaches
+ # `matched == 0` too, so a run that had just listed its broken invariants went on to say none was
+ # evaluated and it carried no failure, immediately before returning 1. NOT gated on
+ # `build_failed`, which is collected from pairs whose invariants were unreadable.
+ # test_a_timed_out_rebuild_leaves_the_behavioural_run_with_no_verdict asserts that pairing.
if matched == 0 and not broken:
- # NOTHING WAS VALIDATED, which is not the same as nothing being wrong. With every pair
- # unchecked, not comparable or never exercised, `broken` is empty and the block above has
- # just printed "Every declared behavioural invariant held on both arms" -- a sentence that
- # is technically true of an empty set and reads as a pass. This is the same false green
- # the digest path returns 2 for, and it is the more dangerous of the two here, because
- # behavioural mode is what REPLACES the digest on a windowed arm: if it silently validates
- # nothing then a windowed arm has no UI verdict at all while appearing to have passed one.
+ # NOTHING WAS VALIDATED is not the same as nothing being wrong: with every pair unchecked,
+ # `broken` is empty and the block above has just printed a sentence that is true of an empty set
+ # and reads as a pass. More dangerous here, because behavioural mode REPLACES the digest.
print(
"\n NOTHING WAS COMPARED. Not one behavioural invariant was evaluated on any pair, "
"so this run carries no UI verdict -- neither a pass nor a failure. Treat it as an "
"absent result and find out why the actions did not run."
)
- # AFTER the diagnosis prints and BEFORE the "could not tell" code, the ordering `report` states:
- # a run can find a real regression while deciding nothing (the two shapes `build_differences`
- # collects survive a pair whose invariants were unreadable), and "it failed" is more specific.
+ # AFTER the diagnosis prints and BEFORE the "could not tell" code: a run can find a real
+ # regression while deciding nothing, and "it failed" is more specific.
+ # The two shapes are what `build_differences` returns.
if broken or build_failed:
return Outcome(1, frozenset(compared), _keys(results))
if matched == 0:
@@ -932,23 +853,17 @@ def visible_unstable_set(null_paths: list[Path] | None) -> frozenset[tuple[str,
"""
if not null_paths:
return frozenset()
- # AND ONLY FROM CELLS THAT FINISHED, which is the same admission rule `unstable_set` applies
- # to the structural floor and `audit_null` to the audit. `collect` states the asymmetry: action
- # rows are written as the film runs and the `cell` row when it ends, so a null-control cell
- # that died mid-film leaves a complete-looking set of captures that nothing owns, and they pair
- # and compare like real ones. Read raw here, one DIFFERING unfinished observation plus one
- # matching completed one is exactly `min_observations`, `derive_unstable` calls that (rung,
- # action) unstable, and `visible_report` then files a real visible difference at the same key
- # under "differ against an identical build" and exits 0. The floor is the one place an
- # under-observed action must read as undetermined rather than as an excuse.
+ # AND ONLY FROM CELLS THAT FINISHED, the same admission rule `unstable_set` and `audit_null`
+ # apply. A null-control cell that died mid-film leaves captures nothing owns, and read raw one
+ # DIFFERING unfinished observation plus one matching completed one is exactly `min_observations`,
+ # so a real visible difference is filed under "differ against an identical build" and exits 0.
results, _got = compare_all_with(
null_paths, P.compare_visible, "visible", require_complete = True
)
by_rung: dict[str, list[tuple[str, dict]]] = collections.defaultdict(list)
for action, _shard, cell, r in results:
- # A pair whose action never ran on both arms is not an observation of anything, in either
- # direction. `derive_unstable` refuses to count a verdict it cannot read; this refuses to
- # hand it one it should not read.
+ # A pair whose action never ran on both arms is not an observation in either direction:
+ # `derive_unstable` refuses to count a verdict it cannot read, and this refuses to hand it one.
if r.get("_ran"):
by_rung[rung_of_cell(cell)].append((action, r))
out: set[tuple[str, str]] = set()
@@ -994,12 +909,10 @@ def visible_report(
found = build_differences(results, min_reps)
compared: set[tuple[str, str, str]] = set()
differing, unstable_bad, blind, idle, matched = [], [], [], [], 0
- # THE RESIDUE, PRINTED. A message that was on screen during the action and had been unmounted
- # again by the time the capture ran cannot be digested, and `compare_visible` refuses the pair
- # for it. That refusal used to be invisible: the pair returned MATCH with the ordinals tucked
- # into `not_digested` and nothing here read the key, so a rendering difference in the missing
- # message left no trace anywhere in the output. It is collected across every verdict, because a
- # DIFFER pair with a residue is also a pair whose report is incomplete.
+ # THE RESIDUE, PRINTED. A message on screen during the action but unmounted by capture time
+ # cannot be digested and `compare_visible` refuses the pair for it, which used to be invisible:
+ # the pair returned MATCH with the ordinals in `not_digested` and nothing read the key. Collected
+ # across every verdict, because a DIFFER pair with a residue is also incomplete.
residue = []
for action, shard, cell, r in results:
if r.get("not_digested"):
@@ -1009,13 +922,9 @@ def visible_report(
elif r["verdict"] == P.NOT_COMPARABLE:
blind.append((action, shard, cell, r))
elif r["verdict"] == P.DIFFER:
- # A SEVERE difference is never routed into the noise floor. See compare_visible: an
- # action can be in the derived unstable set for an unrelated attribute and still be
- # the action on which one arm lost the whole thread.
- #
- # AT THIS PAIR'S OWN RUNG. An action that differs against an identical build at 100K
- # says nothing about the same action at 1K, where the thread is a fraction of the size
- # and the film's slots land somewhere else entirely.
+ # A SEVERE difference is never routed into the noise floor: an action can be in the derived
+ # unstable set for an unrelated attribute and still be the action on which one arm lost the whole
+ # thread. AT THIS PAIR'S OWN RUNG, since an action differing at 100K says nothing about 1K.
noise = (rung_of_cell(cell), action) in unstable and not r.get("severe")
(unstable_bad if noise else differing).append((action, shard, cell, r))
compared.add((action, shard, cell))
@@ -1079,8 +988,7 @@ def visible_report(
)
else:
# WHICH RUNG EACH FLOOR ENTRY WAS MEASURED AT, printed, because it is also the only rung it
- # silences anything at. A floor that reads as a list of action names would look like it
- # covers the whole payload.
+ # silences anything at; a bare list of action names would look like it covers the payload.
keys = sorted(unstable)
shown = ", ".join(f"{rung or '?'} {action}" for rung, action in keys[:12])
print(
@@ -1092,13 +1000,13 @@ def visible_report(
build_failed = print_build_differences(found, min_reps)
if matched == 0 and not differing:
- # The same false green every other mode here has already been fixed for: nothing compared
- # is not the same as nothing wrong, and it must not exit 0.
+ # The same false green every other mode here has been fixed for: nothing compared is not nothing
+ # wrong, and it must not exit 0.
print(
"\n NOTHING WAS COMPARED. Not one action pair yielded a visible-region verdict, so\n"
" this run carries no UI verdict at all -- neither a pass nor a failure."
)
- # See the same three lines in `behaviour_report` and in `report`: the diagnosis prints, then a
+ # See the same three lines in `behaviour_report` and `report`: the diagnosis prints, then a
# failure outranks a refusal.
if differing or build_failed:
return Outcome(1, frozenset(compared), _keys(results))
@@ -1167,8 +1075,8 @@ def compare_all_with(
out = compare(sides["base"].get(key), sides["treatment"].get(key))
out["_ran"] = ran
idle = P.execution_verdict(sides["base"], sides["treatment"])
- # WHICH ARM, not merely that one of them was idle. `execution_verdict` returns `one_sided`
- # empty for a missed slot and for the symmetric case, and the arm that DID run otherwise.
+ # WHICH ARM, not merely that one was idle: `execution_verdict` returns `one_sided` empty for a
+ # missed slot and for the symmetric case, and the arm that DID run otherwise.
out["one_sided"] = (idle or {}).get("one_sided") or ""
out["idle_reason"] = (idle or {}).get("idle_reason") or ""
out["idle_detail"] = (idle or {}).get("reason") or ""
@@ -1195,12 +1103,9 @@ def compare_all(
for (shard, rung, rep, sid, action), sides in sorted(got["pairs"].items()):
cell = f"{rung} {rep}"
if "base" not in sides or "treatment" not in sides:
- # One arm never produced this row at all. Recorded rather than skipped: an action that
- # ran on one arm and not the other is itself a difference between the arms.
- #
- # A resumed run lands here too, with the two arms under two session ids, and the
- # session is named so the reader can tell that case from a surface one build never
- # rendered. Both carry NOT_COMPARABLE, which is the honest verdict for either.
+ # One arm never produced this row. Recorded rather than skipped: an action that ran on one arm
+ # and not the other is itself a difference. A resumed run lands here too with two session ids, so
+ # the session is named; both carry NOT_COMPARABLE, which is honest for either.
results.append(
(
action,
@@ -1383,16 +1288,15 @@ def audit_null(
by_rung[rung_of_cell(cell)].append((action, r))
decided, undecided, differed, excused, out_of_scope = [], [], [], [], []
- # Decided entries whose every observation came from a settled-match refusal. Kept apart from
+ # Decided entries whose every observation came from a settled-match refusal, kept apart from
# `decided` because the live tail was never read at all.
on_settled = []
for rung, pairs in sorted(by_rung.items()):
for action, row in sorted(P.derive_unstable(pairs).items()):
entry = (rung, action)
- # OUT OF SCOPE IS NOT AN EXCUSE, and conflating the two is what made the empty scope
- # read as a vacuous audit below. An excused action is one the fixture cannot perform
- # and it is a HOLE in a question that was asked. An out-of-scope action is a question
- # nobody asked, because no excuse could have changed the result's verdict for it.
+ # OUT OF SCOPE IS NOT AN EXCUSE: an excused action is a HOLE in a question that was asked, while
+ # an out-of-scope one is a question nobody asked, and conflating them made the empty scope read
+ # as a vacuous audit.
if scope is not None and entry not in scope:
out_of_scope.append(entry)
continue
@@ -1405,25 +1309,13 @@ def audit_null(
if row["unstable"]:
differed.append(entry)
- # AN ACTION THE NULL NEVER MEASURED AT ALL IS UNDECIDED, not absent. The loop above can only
- # classify what `derive_unstable` produced, so a scoped (rung, action) with NO rows in the
- # null payload fell into neither list and the audit never asked about it. One other scoped
- # action being decided was then enough to return 0.
- #
- # That is the whole question this audit exists to put, answered by default. `unstable_set`
- # unions the DECLARED names back in, so an action like `send_turn` that the null never
- # observed is still excused by name, and a corroborated result difference on it passes.
- # Reproduced end to end: scope {send_turn, settings}, null carrying no send_turn rows,
- # audit 0, verdict 0, the regression printed under "expected to vary".
- #
- # Rows vanish entirely more easily than they look like they should: the null is collected with
- # `require_complete = True`, so a cell that never finished takes every one of its action rows
- # with it, and the action stops existing rather than becoming undetermined.
- #
- # Counted as MISSING as well as undecided, because the two are not the same reading. Measured
- # and inconclusive means the null tried; never measured means it did not, and a reader chasing
- # a failed audit needs to know which. `allow_undecided` still applies -- `image_upload` has no
- # attachments button on this fixture and is waived by name whether it produced rows or not.
+ # AN ACTION THE NULL NEVER MEASURED AT ALL IS UNDECIDED, not absent: the loop can only classify
+ # what `derive_unstable` produced, so a scoped (rung, action) with no rows fell into neither list
+ # and one other decided action was enough to return 0 -- with `unstable_set` unioning the DECLARED
+ # names back in, a corroborated result difference on it then passed. Reproduced end to end: scope
+ # {send_turn, settings}, no send_turn rows, audit 0, verdict 0, regression printed under
+ # "expected to vary". Counted as MISSING as well as undecided, because measured-and-inconclusive
+ # is a different reading from never measured.
missing = []
if scope is not None:
seen = set(decided) | set(undecided) | set(excused)
@@ -1440,16 +1332,14 @@ def audit_null(
"missing": missing,
"decided_on_settled_thread": on_settled,
}
- # NOTHING TO DECIDE IS A PASS, and only when a scope said so. A result whose every action
- # matched asks the null for no excuses at all, and there is no way to fail a question that
- # was never put. This is not the vacuous case guarded below: the scope was computed from the
- # result, so an empty one is a statement about the result rather than about the null.
+ # NOTHING TO DECIDE IS A PASS, and only when a scope said so: a result whose every action matched
+ # asks the null for no excuses, and there is no way to fail a question never put. The scope is
+ # computed from the result, so an empty one is a statement about the result.
if scope is not None and not scope:
report_["reason"] = "the result needs no excuse from this null control"
return 0, report_
- # Everything excused is not a decided null control, it is a null control that measured
- # nothing while naming a reason for each blank. Passing it would let the excuse list grow
- # until the audit is vacuous.
+ # Everything excused is not a decided null control but one that measured nothing while naming a
+ # reason for each blank; passing it would let the excuse list grow until the audit is vacuous.
if not decided:
report_["reason"] = "no (rung, action) reached min_observations"
return 1, report_
@@ -1488,8 +1378,8 @@ def print_null_audit(rc: int, report_: dict, allow_undecided: frozenset) -> None
f"({', '.join(sorted(allow_undecided))}) -- each one a hole"
)
if report_.get("out_of_scope"):
- # Counted apart from the excused, because it is not a hole: no excuse could have moved
- # the result's verdict on these, so the null was never asked about them.
+ # Counted apart from the excused, because it is not a hole: no excuse could have moved the
+ # result's verdict on these.
print(
f" not audited, out of scope: {len(report_['out_of_scope'])} "
"(the result's verdict for these does not turn on an excuse)"
@@ -1514,9 +1404,8 @@ def print_null_audit(rc: int, report_: dict, allow_undecided: frozenset) -> None
print("\n These (rung, action) pairs never reached min_observations:")
_missing = set(report_.get("missing") or ())
for rung, action in report_["undecided"][:12]:
- # Said apart from the rest: measured and inconclusive means the null tried and
- # could not decide; NO ROWS means it never measured the action at all, which is a
- # different thing to go and fix and used to be invisible here.
+ # Said apart from the rest: measured and inconclusive means the null tried and could not decide,
+ # while NO ROWS means it never measured the action at all.
tail = " -- NO ROWS in the null at all" if (rung, action) in _missing else ""
print(f" {action}@{rung}{tail}")
print(
@@ -1561,15 +1450,14 @@ def unstable_set(paths: list[Path] | None) -> tuple[frozenset, dict, dict]:
derived[f"{action}@{rung}"] = row
if row["unstable"]:
measured.add((rung, action))
- # The cross-check stays pooled ON PURPOSE. It audits the DECLARED list, whose entries claim to
- # hold at every rung, so the question it answers -- did this run ever see this action differ --
- # is an action-level question. It is advisory output and carries no verdict.
+ # The cross-check stays pooled ON PURPOSE: it audits the DECLARED list, whose entries claim to
+ # hold at every rung, so the question is action-level. Advisory output, carrying no verdict.
checks = P.cross_check(
P.derive_unstable([(a, r) for a, _s, _c, r in results]), UNSTABLE_ACTIONS
)
- # UNION, not replacement. An action the null control could not reach -- `image_upload` has no
- # visible attachments button on this fixture -- would otherwise silently move from "declared
- # unstable" to "stable" on the strength of a measurement that never happened.
+ # UNION, not replacement: an action the null could not reach (`image_upload` has no attachments
+ # button on this fixture) would otherwise move from "declared unstable" to "stable" on a
+ # measurement that never happened.
return frozenset(measured) | frozenset(UNSTABLE_ACTIONS), derived, checks
@@ -1599,7 +1487,6 @@ def in_arm_repeatability(paths: list[Path]) -> tuple[set, set]:
not what the measurement is picking up.
"""
got = collect(paths, require_complete = True)
- # (shard, rung, session, action) -> {rep: side A row}
side_a: dict[tuple, dict] = collections.defaultdict(dict)
for (shard, rung, rep, sid, action), sides in got["pairs"].items():
row = sides.get("base")
@@ -1690,14 +1577,11 @@ def corroborated(entries: list[tuple], min_reps: int) -> tuple[list[tuple], list
by_action[(entry[0], rung_of_cell(entry[2]), direction)].append(entry)
firm, weak = [], []
for group in by_action.values():
- # DISTINCT repetitions, not rows: one repetition seen twice is one observation.
- # DELIBERATELY NOT KEYED ON THE SHARD, and the shard is available in `e[1]`. Two shards
- # carrying the same (rung, rep) cannot be told apart from ONE film recorded twice -- the
- # duplicate-session case, where both copies are the same observation and counting them as
- # two lets a single flake corroborate itself. Keying on the cell alone can only
- # under-count, which costs a finding; keying on the shard can manufacture corroboration
- # out of duplicated provenance, which is the failure this whole instrument is about.
- # `test_one_repetition_seen_twice_is_not_two_observations` holds this direction.
+ # DISTINCT repetitions, not rows: one repetition seen twice is one observation. DELIBERATELY NOT
+ # KEYED ON THE SHARD -- two shards carrying the same (rung, rep) cannot be told from one film
+ # recorded twice, where counting both lets a single flake corroborate itself. Keying on the cell
+ # alone can only under-count; keying on the shard manufactures corroboration.
+ # test_one_repetition_seen_twice_is_not_two_observations holds this direction.
reps = {e[2] for e in group}
(firm if len(reps) >= min_reps else weak).extend(group)
return firm, weak
@@ -1722,8 +1606,8 @@ def report(
"""
results, got = compare_all(paths, select)
if not results:
- # An empty result is reported as an empty result. "No mismatches found" when nothing was
- # ever compared is the exact shape of a check that silently does nothing.
+ # An empty result is reported as an empty result: "No mismatches found" when nothing was compared
+ # is the exact shape of a check that silently does nothing.
print(
f"\n{label}: NO PARITY DATA in {len(paths)} payload(s). "
f"{got['missing']} action rows carried no digest. "
@@ -1738,69 +1622,43 @@ def report(
compared: set[tuple[str, str, str]] = set()
matched = 0
for action, shard, cell, r in results:
- # COLLECTED BEFORE THE VERDICT BRANCHES AND OUTSIDE THE EXEMPTION, because it is not a
- # digest. An action can run on both arms, produce two digests the instability exemption
- # then excuses, and still have failed its own assertion on one arm only -- which is the
- # two builds behaving differently and the one shape `ran` cannot see. `stop_generation`
- # is the live example: `ran = True, expect_ok = stopped_ms is not None`, and it is on the
- # declared unstable list, so a head that no longer stops generation was excused twice
- # over. The exemption was measured for digest stability; applying it here would be
- # applying it to a different quantity that happens to share an action name.
- # The fifth element is the DIRECTION, and `corroborated` keys on it. Which arm's
- # assertion failed is part of what is being claimed: a treatment failure in one
- # repetition and a base failure in the next is a race that landed on either side, not
- # one finding seen twice.
+ # COLLECTED BEFORE THE VERDICT BRANCHES AND OUTSIDE THE EXEMPTION, because it is not a digest: an
+ # action can produce two digests the instability exemption excuses and still have failed its own
+ # assertion on one arm only, which is the one shape `ran` cannot see. `stop_generation` is the
+ # live example: `ran = True, expect_ok = stopped_ms is not None`, and it is on the declared
+ # unstable list, so a head that no longer stops generation was excused twice over.
+ # The fifth element is the DIRECTION, and `corroborated` keys on it: a treatment
+ # failure in one repetition and a base failure in the next is a race, not one finding twice.
if r.get("expect_regressed"):
expect_bad.append(
(action, shard, cell, [r.get("expect_reason", "")], r["expect_regressed"])
)
if r["verdict"] == P.NOT_APPLICABLE:
- # NEITHER A PASS NOR A FAIL. The digest is the wrong question for this pair; see
- # analysis/parity.applicability. It is bucketed separately so it cannot be summed into
- # either column, and `--mode behaviour` is what answers it instead.
- #
- # BELOW the assertion collection above, and that ordering is the merge's one real
- # decision. This branch `continue`s, so putting it first would drop a failed assertion
- # whenever the pair also happened to be a windowed mount -- exactly what the comment
- # above means by collecting outside the verdict branches. A windowed arm that stops
- # stopping generation is the case that would have gone silent.
+ # NEITHER A PASS NOR A FAIL: the digest is the wrong question for this pair, so it is bucketed
+ # separately and `--mode behaviour` answers it instead. BELOW the assertion collection above,
+ # and that ordering is the merge's one real decision: this branch `continue`s, so putting it
+ # first would drop a failed assertion whenever the pair was also a windowed mount.
inapplicable.append((action, shard, cell, [r.get("reason", "")]))
continue
if r["verdict"] == P.NOT_EXERCISED:
# Same, for which arm stayed live: `one_sided` names the arm that DID run.
entry = (action, shard, cell, [r.get("reason", "")], r.get("one_sided") or None)
if r.get("one_sided"):
- # ONE ARM RAN IT AND THE OTHER COULD NOT, which is not the missed-slot case.
- # A missed slot is a runner losing a race and it costs coverage; an action that
- # RUNS on one build and cannot be performed on the other is the two builds
- # behaving differently, and it is the one regression shape that leaves no digest
- # to differ. A control that stopped opening produces exactly this and nothing
- # else, so filing it under coverage is how "the button no longer works" ships
- # green. Held to the SAME corroboration bar as a differing digest below rather
- # than failed on sight, because a contended runner can lose one arm's slot once.
- #
- # EXEMPTED BY `RACY_EXECUTION`, NOT BY THE DIGEST SET, and the distinction is the
- # same one `expect_regressed` rests on. Every UNSTABLE_ACTIONS mechanism describes
- # what makes the CAPTURE move; none of them says the action can fail to happen.
- # Keyed on that list, nine of sixteen actions were permanently exempt from the one
- # regression shape that leaves no digest to differ, so a broken composer taking
- # `keystroke` down on the treatment arm in both repetitions exited 0. `scroll_after`
- # was exempt without even having a `not_run` path to reach.
- # ON THE RECORDED REASON, not the action name. Each of the three has non-racy
- # not_run paths too -- send_turn's "no composer on the page", stop_generation's
- # "the stop button is not present" -- and a treatment build that REMOVES either
- # control records exactly the regression this exists to catch.
+ # ONE ARM RAN IT AND THE OTHER COULD NOT, which is not the missed-slot case: a missed slot costs
+ # coverage, while an action that runs on one build and cannot be performed on the other is the
+ # one regression shape that leaves no digest to differ. Held to the SAME corroboration bar as a
+ # differing digest. EXEMPTED BY `RACY_EXECUTION`, NOT BY THE DIGEST SET: every UNSTABLE_ACTIONS
+ # mechanism describes what makes the CAPTURE move, so keyed on that list nine of sixteen actions
+ # were permanently exempt. ON THE RECORDED REASON, not the action name.
_racy = P.racy_execution(action, r.get("idle_reason") or "")
(one_sided_unstable if _racy else one_sided).append(entry)
else:
idle.append(entry)
continue
- # THE STYLE VERDICT IS COLLECTED BEFORE THE STRUCTURAL REFUSAL IS BUCKETED, because it is
- # an INDEPENDENT reading and the refusal is not about it. The bounded computed-style probe
- # is the only thing here that sees `display`, `visibility` or `pointer-events`, and a pair
- # refused for landing at two points in one stream can still carry a real CSS regression on
- # a settled surface. Sitting below the `continue`, that verdict was computed, attached to
- # the row, and then dropped on the floor for exactly the pairs this PR creates most of.
+ # THE STYLE VERDICT IS COLLECTED BEFORE THE STRUCTURAL REFUSAL IS BUCKETED, because it is an
+ # INDEPENDENT reading: the bounded computed-style probe is the only thing here that sees
+ # `display`, `visibility` or `pointer-events`, and a pair refused for landing at two points in
+ # one stream can still carry a real CSS regression.
if r.get("style_verdict") == P.DIFFER:
style_bad.append((action, shard, cell, [r.get("style_reason", "")]))
if r["verdict"] == P.NOT_COMPARABLE:
@@ -1814,9 +1672,9 @@ def report(
compared.add((action, shard, cell))
(unstable_bad if is_unstable(unstable, action, cell) else stable_bad).append(entry)
- # SPLIT BEFORE THE COUNTS ARE PRINTED. Printing "stable actions differing: 1" above a verdict
- # of 0 is how a reader concludes the tool is lying to them; the headline number has to be the
- # one the exit code is taken from.
+ # SPLIT BEFORE THE COUNTS ARE PRINTED: "stable actions differing: 1" above a verdict of 0 is how
+ # a reader concludes the tool is lying, so the headline number must be the one the exit code
+ # comes from.
stable_bad, uncorroborated = corroborated(stable_bad, min_reps)
one_sided, one_sided_weak = corroborated(one_sided, min_reps)
expect_bad, expect_weak = corroborated(expect_bad, min_reps)
@@ -1873,10 +1731,8 @@ def report(
f"pointer-events)"
)
- # THE SAME SET, COUNTED TWICE, checked rather than assumed: `compared` is what the coverage
- # floor is applied to, `scored` is the count this report has always printed, and `stable_bad`
- # splits into itself plus `uncorroborated` above. The assert is here so a later edit to either
- # branch cannot make the floor answer about a different set than the summary does.
+ # THE SAME SET, COUNTED TWICE, checked rather than assumed, so a later edit to either branch
+ # cannot make the coverage floor answer about a different set than the summary does.
scored = matched + len(stable_bad) + len(uncorroborated) + len(unstable_bad)
assert len(compared) == scored, (len(compared), scored)
shortfall = coverage_shortfall(scored, len(results), min_compared)
@@ -1925,28 +1781,18 @@ def report(
for action, shard, cell, why, *_dir in one_sided_unstable[:8]:
print(f" {action:<26} {shard} {cell}: {why[0]}")
- # A DEMOTED DIFFERENCE IS STILL A COMPARISON. `corroborated` moves a difference seen in fewer
- # than `min_reps` repetitions out of `stable_bad`, and that must not make the pair look like
- # one that never yielded a verdict: the digest was taken on both arms, it differed, and the
- # corroboration bar is the only reason it is not counted as a regression. Reading `matched`
- # alone, a payload whose every difference was uncorroborated exited 2 "NOTHING WAS COMPARED"
- # over a run that compared everything it had -- which is the opposite of the false green this
- # guard exists to catch, and would make `--min-reps 2` fail runs it was meant to quieten.
+ # A DEMOTED DIFFERENCE IS STILL A COMPARISON: `corroborated` moves a sub-`min_reps` difference out
+ # of `stable_bad`, and reading `matched` alone a payload whose every difference was uncorroborated
+ # exited 2 "NOTHING WAS COMPARED" over a run that compared everything it had.
decided = matched + len(uncorroborated) + len(one_sided_weak)
if stable_bad:
print("\n UI PARITY DIFFERENCES ON STABLE ACTIONS -- these need explaining:")
for action, shard, cell, moved in stable_bad:
print(f" {action:<26} {shard} {cell}: {', '.join(moved[:4])}")
elif decided == 0:
- # NOT A PASS, and it must not print like one. Not one pair produced a digest verdict, so
- # "no stable action rendered differently" would be true, reassuring and about nothing --
- # the exact shape of a check that silently does nothing, which is what the NOT COMPARABLE
- # bucket exists to prevent elsewhere in this file.
- #
- # THE GUARD USED TO REQUIRE `inapplicable`, so it only caught the windowed-mount route to
- # an empty comparison. A run whose parity probes all failed, or whose slots were all
- # missed, produced nothing but NOT COMPARABLE and NOT EXERCISED rows, no inapplicable ones,
- # and exited 0 under a heading that reads as a clean structural pass.
+ # NOT A PASS, and it must not print like one: with no pair producing a digest verdict, "no stable
+ # action rendered differently" is true, reassuring and about nothing. The guard used to require
+ # `inapplicable`, so a run whose parity probes all failed exited 0 under a clean-looking heading.
print(
f"\n NOTHING WAS COMPARED. Not one of the {len(results)} pair(s) here yielded a "
f"structural verdict ({len(inapplicable)} windowed, {len(blind)} not comparable, "
@@ -1966,9 +1812,8 @@ def report(
)
if uncorroborated:
- # Printed in full. A build renders the same way every time, so one repetition out of two
- # is evidence of the run rather than of the build -- but it is still a reading, and a
- # reader chasing an intermittent regression needs to see it rather than be told nothing.
+ # Printed in full: one repetition out of two is evidence of the run rather than of the build, but
+ # it is still a reading a reader chasing an intermittent regression needs to see.
print(
f"\n UNCORROBORATED -- differed in only one repetition, so not counted as a "
f"regression:"
@@ -1982,8 +1827,8 @@ def report(
print(f" {action:<26} {shard} {cell}: {why[0]}")
if idle:
- # Named surfaces, deduplicated: what matters is WHICH actions this run never opened, not
- # that it failed to open one of them sixteen separate times.
+ # Named surfaces, deduplicated: what matters is WHICH actions this run never opened, not that it
+ # failed to open one of them sixteen times.
names = sorted({entry[0] for entry in idle})
print(
f"\n NOT EXERCISED -- {len(idle)} pair(s) over {len(names)} action(s) that did not "
@@ -2003,19 +1848,13 @@ def report(
for action, shard, cell, moved in unstable_bad[:8]:
print(f" {action:<26} {shard} {cell}: {', '.join(moved[:3])}")
# A one-sided action is a failure on the same footing as a differing digest: the two builds
- # behaved differently and the difference happens to leave no digest to compare. `expect_bad`
- # joins them for the reason given where it is collected: an assertion that failed on one arm
- # only is the two builds behaving differently, and it is the one shape a digest cannot show.
+ # behaved differently and the difference leaves no digest to compare.
+ # `expect_bad` is collected ahead of the verdict branches.
if stable_bad or one_sided or expect_bad:
return Outcome(1, frozenset(compared), _keys(results))
- # 2, the same code the empty-payload path uses, and for the same reason: the tool was asked a
- # question it could not answer. Exiting 0 here would let CI go green on a run where the parity
- # check was structurally incapable of firing -- whether because every pair was a windowed mount
- # or because every capture failed. `decided`, not `matched`, for the reason given above it.
- #
- # AFTER the failure return, not before it: a run can find a real regression while deciding
- # nothing -- `expect_bad` is collected ahead of the verdict branches and survives a pair that
- # `continue`s -- and "it failed" is the more specific answer than "it could not tell".
+ # 2, the same code the empty-payload path uses: the tool was asked a question it could not
+ # answer, and exiting 0 would let CI go green on a run structurally incapable of firing.
+ # `decided`, not `matched`. AFTER the failure return, since "it failed" is more specific.
if decided == 0:
return Outcome(2, frozenset(compared), _keys(results))
return Outcome(0, frozenset(compared), _keys(results))
@@ -2169,10 +2008,9 @@ def main(argv: list[str] | None = None) -> int:
)
ap.add_argument(
"--mode",
- # `structural` and `behavior` are ALIASES, not extra modes. The report header prints
- # "(STRUCTURAL MODE)" and the pull request template asks for "the structural digest", so
- # a reader who follows either and types `--mode structural` should not be told it is not
- # a choice. Same for the American spelling of behaviour.
+ # `structural` and `behavior` are ALIASES, not extra modes: the report header prints
+ # "(STRUCTURAL MODE)" and the PR template asks for "the structural digest", so a reader who types
+ # either should not be told it is not a choice.
choices = ("auto", "digest", "structural", "visible", "behaviour", "behavior"),
default = "auto",
help = (
@@ -2233,28 +2071,16 @@ def main(argv: list[str] | None = None) -> int:
args = ap.parse_args(argv)
args.mode = {"structural": "digest", "behavior": "behaviour"}.get(args.mode, args.mode)
- # THE MODE DECISION FIRST, before the unstable set is even derived. Deriving an unstable set
- # from a null control and then not using it because the payload is windowed would print a page
- # of scoring apparatus that has no bearing on the report underneath it.
- #
- # PER ACTION PAIR, and per payload and per invocation were both too coarse:
- #
- # per invocation `ui_parity normal_run windowed_run` scored the NORMAL run behaviourally
- # too, so an ordinary DOM regression in a fully mounted arm went unreported
- # because an unrelated payload on the same command line was windowed.
- # per payload one payload holds several rungs, and the readiness gate deliberately
- # permits an arm to mount everything at 1K and a window at 100K. One windowed
- # large-rung capture then suppressed the structural digest on every fully
- # mounted pair beside it, at the rungs where that digest is exactly the right
- # question.
- #
- # `--mode visible|behaviour|digest` still forces every pair, because that is what forcing means.
+ # THE MODE DECISION FIRST, before the unstable set is derived, or a windowed payload prints a
+ # page of scoring apparatus with no bearing on the report. PER ACTION PAIR: per invocation scored
+ # a normal run behaviourally because an unrelated payload on the same command line was windowed,
+ # and per payload let one windowed 100K capture suppress the digest on every mounted pair.
plan: list[dict] = []
for pattern in args.payloads:
paths = shards_of(pattern)
if not paths:
- # Kept with no pairs: the structural loop below is what reports a pattern that matched
- # no payload, and it exits 2 for it.
+ # Kept with no pairs: the structural loop below is what reports a pattern that matched no
+ # payload, and it exits 2 for it.
plan.append({"pattern": pattern, "paths": [], "windowed": set(), "structural": None})
continue
if args.mode == "digest":
@@ -2279,8 +2105,8 @@ def main(argv: list[str] | None = None) -> int:
"pattern": pattern,
"paths": paths,
"windowed": windowed,
- # None, not an empty set, when the payload holds no action pairs at all: `report`
- # is what says NO PARITY DATA and exits 2, and it has to be reached to say it.
+ # None, not an empty set, when the payload holds no action pairs at all: `report` is what says
+ # NO PARITY DATA and exits 2, and it has to be reached to say it.
"structural": (set(decided) - windowed) if decided else None,
"why": next((w for _k, (m, w) in sorted(decided.items()) if m == WINDOWED), ""),
}
@@ -2288,10 +2114,9 @@ def main(argv: list[str] | None = None) -> int:
worst = 0
scored_windowed = 0
- # THE COVERAGE FLOOR'S OWN BOOKKEEPING, kept per payload pattern and across every mode that
- # pattern was scored in. A SET rather than a running total because `--mode auto` scores each
- # windowed pair twice, once on the visible region and once on the invariants; see `Outcome`.
- # Keyed by pattern because the floor is a question about one film; see `_floored`.
+ # THE COVERAGE FLOOR'S OWN BOOKKEEPING, per payload pattern and across every mode. A SET rather
+ # than a running total because `--mode auto` scores each windowed pair twice; keyed by pattern
+ # because the floor is a question about one film.
compared_pairs: dict[str, set[tuple]] = {}
seen_pairs: dict[str, set[tuple]] = {}
@@ -2320,48 +2145,35 @@ def main(argv: list[str] | None = None) -> int:
f"{len(struct)} fully mounted pair(s) are scored structurally further down."
)
for key in sorted(win)[:8]:
- # shard, rung, rep and the action. The session term of the key is left out of
- # the line: it is part of a pair's identity, not part of naming it to a reader.
+ # shard, rung, rep and the action. The session term is left out of the line: it is part of a
+ # pair's identity, not part of naming it to a reader.
print(f" windowed: {key[0]} {key[1]} {key[2]} {key[4]}")
if len(win) > 8:
print(f" ... and {len(win) - 8} more windowed pair(s)")
scored_windowed += len(win)
- # Per pattern, and reset here rather than outside the loop: two payload patterns are two
- # films and must not pool their coverage, which is the rule `_floored` already states.
+ # Per pattern, and reset here rather than outside the loop: two payload patterns are two films
+ # and must not pool their coverage.
windowed_compared: Optional[frozenset[tuple]] = None
windowed_seen: set[tuple] = set()
- # BOTH, and in this order. Visible-region parity is the verdict the off-screen exemption
- # asks for and it is the one that can FAIL a windowed arm for something a user would see.
- # The behavioural invariants are the complement: they catch what a viewport comparison
- # cannot, such as a clipboard that no longer carries the thread. Neither subsumes the other,
- # so a windowed pair gets both and the run fails if either does.
+ # BOTH, and in this order. Visible-region parity is the verdict the off-screen exemption asks for
+ # and can FAIL a windowed arm for something a user would see; the behavioural invariants catch
+ # what a viewport comparison cannot, such as a clipboard that no longer carries the thread.
if args.mode in ("auto", "visible"):
if vis_unstable is None:
- # The floor, derived from the null control the caller passed. Without it an
- # identical pair of builds outscores the arm under test. Derived once: it is a
- # property of the null control, not of the payload being scored against it.
+ # The floor, derived from the null control the caller passed; without it an identical pair of
+ # builds outscores the arm under test. Derived once: it is a property of the null control.
vis_null: list[Path] = []
for pat in args.null:
vis_null.extend(shards_of(pat))
vis_unstable = visible_unstable_set(vis_null)
vis_null_tiers = one_tier(vis_null, "null control")
vis_null_corpora = one_corpus(vis_null, "null control")
- # A FLOOR FROM ANOTHER FILM IS NOT THIS PAYLOAD'S FLOOR, ON EITHER AXIS. `tier_of`
- # states the mechanism for the film: the slot spacing decides which actions land
- # inside a live stream and so differ against themselves, and `--tier fast` then
- # `--tier standard` is the documented way to work, which is exactly how a stale fast
- # null control ends up on the command line of a standard run. `one_corpus` states the
- # same mechanism one level down for the thread the film drove, and that axis is the
- # quieter of the two: a corpus revision lands, the payload is re-recorded against it,
- # and an older null control is still sitting in the directory the workflow globs.
- # `one_tier` and `one_corpus` above refuse a null control that is internally mixed;
- # `cross_side_mismatch` refuses one that is internally consistent but belongs to a
- # different film or a different thread. The structural section further down says both
- # out loud, and an ALL-WINDOWED payload -- which is every payload under
- # `--mode visible` -- returns before it ever reaches that warning, so a floor measured
- # on the wrong film or the wrong corpus silenced a real visible difference and the
- # command exited 0 without a word about it. Refused rather than applied: an unfloored
- # report says what it is missing, a wrongly floored one does not.
+ # A FLOOR FROM ANOTHER FILM IS NOT THIS PAYLOAD'S FLOOR, ON EITHER AXIS. Slot spacing decides
+ # which actions land inside a live stream, and `--tier fast` then `--tier standard` is the
+ # documented way to work, which is how a stale fast null control reaches a standard run; the
+ # corpus axis is quieter, since a re-recorded payload leaves an older null in the globbed
+ # directory. `one_tier`/`one_corpus` refuse an internally mixed null, `cross_side_mismatch`
+ # refuses a consistent one belonging to a different film. Refused rather than applied.
floor = vis_unstable
mismatch = cross_side_mismatch(
one_tier(paths, "payload"),
@@ -2391,54 +2203,25 @@ def main(argv: list[str] | None = None) -> int:
min_reps = args.min_reps,
)
worst = max(worst, int(beh))
- # Only where behavioural mode is the ONLY mode, i.e. `--mode behaviour`. Under
- # `--mode auto` it adds nothing to the numerator; see below.
+ # Only where behavioural mode is the ONLY mode, i.e. `--mode behaviour`. Under `--mode auto` it
+ # adds nothing to the numerator; see below.
if windowed_compared is None:
windowed_compared = frozenset(beh.compared)
windowed_seen |= set(beh.seen)
- # NOT UNIONED ACROSS THE TWO WINDOWED MODES. The union is right for the structural set
- # below, which is a DISJOINT set of pairs -- coverage there really does add up. It is wrong
- # here: under `--mode auto` these two are not two slices of the film, they are two verdicts
- # on the SAME pairs, which is what the comment above says when it says neither subsumes the
- # other. Unioned, one stood in for the other: behavioural mode reaching all 16 pairs while
- # visible capture succeeded on 1 and returned NOT COMPARABLE for the other 15 still put 16
- # keys in the set, cleared `--min-compared 16`, and exited 0 with essentially the whole
- # appearance surface unmeasured.
- #
- # AND THE TWO ARE NOT INTERCHANGEABLE, so this is not an intersection either. The modes are
- # asymmetric in what they even claim to reach: visible-region parity applies to EVERY
- # windowed pair, while a behavioural invariant only exists for the handful of actions that
- # declare one -- every other pair is reported UNCHECKED, which is explicitly not a pass and
- # equally not a shortfall. Intersecting would therefore fail a run for pairs behavioural
- # mode never had an opinion about, which is a false red measured at 1 of 4 on the coverage
- # fixture here.
- #
- # So in `auto` the numerator is the visible verdict, the one that is owed on every pair.
- # Behavioural coverage cannot vouch for the appearance of a pair the viewport comparison
- # could not read, which is exactly the substitution above.
- #
- # `seen` stays a union: it is the denominator, the pairs the reports were OFFERED, and a
- # pair offered to either was offered. Shrinking it alongside the numerator would hide the
- # shortfall it exists to express.
+ # NOT UNIONED ACROSS THE TWO WINDOWED MODES. The union is right for the structural set below,
+ # which is DISJOINT; here the two are two verdicts on the SAME pairs, and unioned one stood in
+ # for the other -- behavioural mode reaching all 16 pairs while visible capture succeeded on 1
+ # still cleared `--min-compared 16`. Not an intersection either: a behavioural invariant only
+ # exists for the few actions that declare one, so intersecting fails a run for pairs it had no
+ # opinion about. So in `auto` the numerator is the visible verdict; `seen` stays a union.
if windowed_compared is not None:
compared_pairs.setdefault(pattern, set()).update(windowed_compared)
seen_pairs.setdefault(pattern, set()).update(windowed_seen)
- # AHEAD OF THE STRUCTURAL EARLY RETURN, because this mode does not read the plan at all.
- #
- # It takes its shards from `args.payloads` directly, audits each one AS a null control, resets
- # `worst` and returns unconditionally -- so it shares nothing with the windowed/structural
- # split it used to sit under. Below the return it was simply unreachable whenever `remaining`
- # came out empty, and that is not an exotic payload: `--mode visible` and `--mode behaviour`
- # both hard-set `structural` to an empty set for EVERY pattern, so `--audit-null` with either
- # of them could never run the audit at all, and auto mode joined them the moment every pair
- # classified as windowed. The command then exited 0 out of the ordinary visible report, having
- # silently skipped the audit the caller explicitly asked for and the `--compared-in`
- # enforcement that goes with it -- an option whose whole promise is to FAIL unless the null
- # decided the actions the result needs an excuse for.
- #
- # Moved here rather than duplicated above the return: one call site, and the auto-mode path
- # that already worked reaches it after exactly the same windowed reports as before.
+ # AHEAD OF THE STRUCTURAL EARLY RETURN, because this mode does not read the plan at all: it takes
+ # shards from `args.payloads`, audits each AS a null control and returns unconditionally. Below
+ # the return it was unreachable whenever `remaining` came out empty, which `--mode visible` and
+ # `--mode behaviour` guarantee, so `--audit-null` with either silently skipped the audit.
if args.audit_null:
allow = frozenset(a.strip() for a in args.allow_undecided.split(",") if a.strip())
scope = None
@@ -2471,15 +2254,11 @@ def main(argv: list[str] | None = None) -> int:
worst = max(worst, rc)
return worst
- # THE STRUCTURAL EARLY RETURN, and the reason there is now a floor check on both sides of it.
- # Everything below is the structural half, and a payload with no fully mounted pair has none, so
- # returning here is right for all of it. `--min-compared` was not one of those things: it is a
- # COVERAGE floor asking whether the film ran, and a windowed run's film runs exactly as much.
- #
- # This return is the FOURTH cross-cutting enforcement found sitting below it -- the null
- # control's tier axis, then its corpus axis, then `--audit-null`, then this. The first three
- # were hoisted one at a time; so the floor goes through `coverage_shortfall` on BOTH exits, over
- # the pairs every mode compared, with the reports handing those pairs back.
+ # THE STRUCTURAL EARLY RETURN, and why there is now a floor check on both sides of it. Everything
+ # below is the structural half, which a payload with no fully mounted pair has none of, but
+ # `--min-compared` is a COVERAGE floor asking whether the film ran, and a windowed run's film runs
+ # exactly as much. The FOURTH cross-cutting enforcement found sitting below it, so the floor goes
+ # through `coverage_shortfall` on BOTH exits.
remaining = [e for e in plan if e["structural"] is None or e["structural"]]
if not remaining:
return _floored(worst, compared_pairs, seen_pairs, args.min_compared)
@@ -2512,10 +2291,9 @@ def main(argv: list[str] | None = None) -> int:
)
print(" pass --null OUTDIR of a base-vs-base run to derive it instead.")
- # `null_tiers` is the set `one_tier` already refused a mixed null control for, above. NOT
- # reassigned from `tier_of` here: that would be the same set without the refusal in front of it,
- # and `worst` is NOT reset either -- the windowed reports above have already run and a
- # behavioural or visible failure there may not be dropped by the structural pass below.
+ # `null_tiers` is the set `one_tier` already refused a mixed null control for. NOT reassigned
+ # from `tier_of` here, which would be the same set without the refusal in front of it, and `worst`
+ # is NOT reset either: a windowed failure above may not be dropped by the structural pass.
scored_structural = 0
for entry in remaining:
pattern, paths, select = entry["pattern"], entry["paths"], entry["structural"]
@@ -2524,21 +2302,18 @@ def main(argv: list[str] | None = None) -> int:
worst = max(worst, 2)
continue
tiers = one_tier(paths, "payload")
- # REFUSED, NOT WARNED. A tier or corpus the null control was not recorded against makes
- # its unstable set inapplicable to this payload, and a warning printed above a verdict is
- # read as a verdict. `cross_side_mismatch` covers the tier this used to warn about and the
- # corpus it did not check at all.
+ # REFUSED, NOT WARNED: a tier or corpus the null was not recorded against makes its unstable set
+ # inapplicable, and a warning printed above a verdict is read as a verdict. `cross_side_mismatch`
+ # covers the tier this used to warn about and the corpus it did not check at all.
corpora = one_corpus(paths, "payload")
mismatch = cross_side_mismatch(tiers, null_tiers, corpora, null_corpora)
if mismatch:
print(f"\n REFUSING to score {pattern}: {mismatch}")
worst = max(worst, 2)
continue
- # CONFINED TO THE RUNNER BEING SCORED, and only ever downward. The exemptions above were
- # measured in the other matrix job, on another machine, minutes away; side A of THIS
- # payload is the same build in every repetition, so it can say whether this machine
- # reproduces them. One that it positively contradicts is dropped, one it could not decide
- # is kept.
+ # CONFINED TO THE RUNNER BEING SCORED, and only ever downward: the exemptions above were measured
+ # in another matrix job on another machine, while side A of THIS payload is the same build in
+ # every repetition. One it positively contradicts is dropped; one it could not decide is kept.
effective = unstable
if derived:
local_unstable, local_stable = in_arm_repeatability(paths)
@@ -2552,30 +2327,25 @@ def main(argv: list[str] | None = None) -> int:
)
label = f"UI PARITY: {pattern}"
if select is not None and entry["windowed"]:
- # WHICH PAIRS THESE ARE, in the heading. A structural section that silently covered
- # part of a payload would read as a verdict on all of it.
+ # WHICH PAIRS THESE ARE, in the heading: a structural section that silently covered part of a
+ # payload would read as a verdict on all of it.
label += (
f" ({len(select)} fully mounted pair(s) of "
f"{len(select) + len(entry['windowed'])})"
)
scored_structural += len(select) if select is not None else 0
- # BOTH SIDES OF THE MERGE, and the argument list is why this one had to be read rather
- # than chosen. `select` is the FOURTH positional here; the incoming call omitted it,
- # because on that branch this function had no windowed split to select from. Taken as
- # written it would have bound `args.min_reps` to `select` and `args.min_compared` to
- # `min_reps`, leaving the coverage floor at its default of 0 -- a silently disabled guard
- # and a structural pass scoring against an integer, with nothing failing to say so.
- # `min_compared` IS NOT PASSED DOWN ANY MORE: that is the change, not an omission. A floor
- # applied inside each report is checked separately per report, so a run that compared nine
- # windowed pairs and nine structural ones would fail a floor of sixteen twice over having
- # compared eighteen. `report` keeps the parameter (the selftests call it directly and it is
- # a complete report on its own); `main` collects the pairs and applies the floor below.
+ # BOTH SIDES OF THE MERGE, and the argument list is why this had to be read rather than chosen:
+ # `select` is the FOURTH positional and the incoming call omitted it, which would have bound
+ # `args.min_reps` to `select` and left the coverage floor at 0. `min_compared` is deliberately NOT
+ # passed down any more: a floor applied inside each report is checked per report, so a run
+ # comparing nine windowed and nine structural pairs would fail a floor of sixteen twice over.
+ # `args.min_compared` was the argument that went missing.
+ # `report` (`elif decided == 0`) and `visible_report` (`and not not_run`) each check their own.
struct = report(paths, label, effective, select, args.min_reps)
worst = max(worst, int(struct))
note(pattern, struct)
- # COMBINED, and the worst outcome wins. A behavioural failure on one payload is not cancelled
- # by a structural pass on another, and a structural failure at the fully mounted rungs is not
- # cancelled by the windowed rungs passing their own two modes.
+ # COMBINED, and the worst outcome wins: a behavioural failure on one payload is not cancelled by
+ # a structural pass on another.
if scored_windowed and scored_structural:
print(
f"\nCOMBINED EXIT STATUS {worst}: {scored_windowed} pair(s) scored on the visible "
diff --git a/unsloth-cli.py b/unsloth-cli.py
index 6ed73d10447..0e1db20a287 100644
--- a/unsloth-cli.py
+++ b/unsloth-cli.py
@@ -109,8 +109,7 @@ def _save_or_push_model(model, tokenizer, args, is_mlx):
print("Warning: The model is not saved!")
return
- # Enter the GGUF branch when saving or pushing GGUF, so --push_gguf works
- # without --save_gguf (the local save is guarded separately below).
+ # Enter the GGUF branch when saving or pushing GGUF, so --push_gguf works without --save_gguf.
if args.save_gguf or args.push_gguf:
if not args.save_gguf:
print("Warning: --save_gguf not set, pushing GGUF to hub without saving locally.")
@@ -193,7 +192,6 @@ def run(args):
if args.use_dora and is_mlx:
raise NotImplementedError("DoRA is not supported for MLX training yet.")
- # Load model and tokenizer
device_map, distributed = _prepare_device_map(is_mlx)
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = args.model_name,
@@ -203,7 +201,6 @@ def run(args):
device_map = device_map,
)
- # Configure PEFT model
model = FastLanguageModel.get_peft_model(
model,
r = args.r,
@@ -237,7 +234,7 @@ def run(args):
### Response:
{}"""
- EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN
+ EOS_TOKEN = tokenizer.eos_token
def formatting_prompts_func(examples):
instructions = examples["instruction"]
@@ -257,7 +254,6 @@ def run(args):
)
dataset = loader.load_from_file(args.raw_text_file)
elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")):
- # Auto-detect local raw text files
loader = _raw_text_loader_for_backend(RawTextDataLoader, tokenizer, is_mlx)
dataset = loader.load_from_file(args.dataset)
else:
@@ -268,20 +264,16 @@ def run(args):
else:
dataset = load_dataset(args.dataset, split = "train")
- # Format structured datasets
dataset = dataset.map(formatting_prompts_func, batched = True)
return dataset
- # Load dataset using smart loader
dataset = load_dataset_smart(args)
print("Data is formatted and ready!")
- # Configure training arguments
training_args = _build_sft_config(SFTConfig, args, is_mlx, is_bfloat16_supported())
if distributed:
training_args.ddp_find_unused_parameters = False
- # Initialize trainer
trainer = SFTTrainer(
model = model,
processing_class = tokenizer,
diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.py
index 1029a100e19..fc601c4eeb2 100644
--- a/unsloth_cli/__init__.py
+++ b/unsloth_cli/__init__.py
@@ -4,9 +4,7 @@
import os as _os
import sys as _sys
-# Are we the `unsloth` console script, rather than a library import? Both the
-# stream guard below and the `-np<N>` rewrite further down are entry-point
-# behaviour and must not reach into a host application that imports us.
+# Entry-point-only behaviour (stream guard, -np<N> rewrite): must not reach a host that imports us.
_entry_base = _os.path.basename(_sys.argv[0]).lower() if _sys.argv else ""
_is_entry_point = _entry_base in {"unsloth", "unsloth.exe"}
_windows_studio_mutation_entry = (
@@ -57,14 +55,9 @@ if _is_entry_point:
from unsloth_cli._system_dir_guard import check_working_directory as _check_working_directory
-# Running from System32 or any subdir WILL cause errors if not prevented. A
-# command the folder cannot affect (the ones Unsloth Desktop spawns, issue #8510)
-# moves out of it; everything else stops in the callback below.
-#
-# Before the command imports, since unsloth_cli.commands.studio resolves
-# STUDIO_HOME at import time and a relative UNSLOTH_STUDIO_HOME would otherwise
-# be pinned to the folder we are leaving. The message waits for typer to render
-# it. A library import reaches the same check from the callback instead.
+# Running from System32 or a subdir breaks commands; move out before the command imports, since
+# commands.studio resolves STUDIO_HOME at import time (issue #8510).
+# A relative UNSLOTH_STUDIO_HOME would otherwise resolve against System32.
_startup_guard = (
_check_working_directory(_sys.argv[1:], _os.environ, _sys.platform) if _is_entry_point else None
)
@@ -107,14 +100,11 @@ def _prepare_entry_point():
return
_reconfigure_entry_point_streams()
_expand_attached_np_short()
- # Set last, so a raise leaves the work retryable rather than silently
- # skipped. Neither call can currently raise -- the first swallows everything
- # and the second is pure argv manipulation -- but the ordering costs nothing.
+ # Set last, so a raise leaves the work retryable rather than silently skipped.
_entry_point_prepared = True
-# Canonicalise `-np<N>` only under the `unsloth` console-script;
-# third-party scripts that import unsloth_cli keep their argv intact.
+# Canonicalise `-np<N>` only under the console-script; imports keep their argv intact.
if _is_entry_point:
_prepare_entry_point()
del _entry_base, _is_entry_point
@@ -176,8 +166,7 @@ def _invocation_args(ctx):
return list(captured)
if not ctx.invoked_subcommand:
return _sys.argv[1:]
- # No capture and no tail to read: assume it holds a path, so an invocation
- # that cannot be read in full is refused rather than relocated.
+ # No capture and no tail: assume it holds a path, so refuse rather than relocate.
return [ctx.invoked_subcommand, *(list(getattr(ctx, "args", None) or []) or ["..."])]
@@ -193,13 +182,11 @@ def main(
help = "Show version and exit.",
),
):
- # Consume the import-time result once: a host calling the app repeatedly can
- # chdir between calls, so each later call is checked afresh.
+ # Consume the import-time result once: a host can chdir between repeated app() calls.
global _startup_guard
_guard, _startup_guard = _startup_guard, None
if _guard is None:
- # A host reaches this after commands.studio has resolved STUDIO_HOME at
- # import time, so moving now would leave that cached root behind.
+ # A host reaches this after commands.studio cached STUDIO_HOME, so moving now strands that root.
_guard = _check_working_directory(
_invocation_args(ctx),
_os.environ,
@@ -233,8 +220,7 @@ if not _windows_studio_mutation_entry:
help = "Deprecated alias for `unsloth start`.",
)
- # top-level `unsloth run` aliases `unsloth studio run`; same context
- # so unknown flags still pass through to llama-server.
+ # top-level `unsloth run` aliases `unsloth studio run`; same context so unknown flags pass through to llama-server.
app.command(
"run",
context_settings = {
diff --git a/unsloth_cli/__main__.py b/unsloth_cli/__main__.py
index 56b9b49ded6..c89bd6c6dc9 100644
--- a/unsloth_cli/__main__.py
+++ b/unsloth_cli/__main__.py
@@ -41,15 +41,11 @@ Output is identical to the console script, which takes three things:
import sys
-# Before the import, so a direct `python path/to/unsloth_cli/__main__.py` run
-# takes the console-script gate in __init__ rather than needing the call below.
+# Before the import, so a direct `python .../__main__.py` run takes the console-script gate in __init__.
sys.argv[0] = "unsloth"
import unsloth_cli # noqa: E402
unsloth_cli._prepare_entry_point()
-# sys.exit, like the generated console script's `sys.exit(app())`. Typer raises
-# SystemExit itself in standalone mode, so today both spellings exit the same way,
-# but a returned value has to become the exit status here too or the two routes
-# stop agreeing the moment one exists.
+# A returned value must become the exit status, like the console script's `sys.exit(app())`.
sys.exit(unsloth_cli.app(prog_name = "unsloth"))
diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.py
index 8a2a80d3ea6..ae04ccd77f9 100644
--- a/unsloth_cli/_inference.py
+++ b/unsloth_cli/_inference.py
@@ -14,10 +14,9 @@ from typing import List, Literal, Optional
import typer
-# Canonical speculative-decoding modes, mirroring the backend's
-# _CANONICAL_SPEC_MODES. Named once so the CLI's option annotations, the HTTP
-# payload builders and the in-process loader cannot drift apart when a mode is
-# added; typer reads it at runtime to validate --speculative-type.
+# Canonical speculative-decoding modes, mirroring the backend's _CANONICAL_SPEC_MODES. Named once
+# so the CLI's option annotations, the HTTP payload builders and the in-process loader cannot
+# drift apart; typer reads it at runtime to validate --speculative-type.
SpeculativeType = Literal[
"auto", "mtp", "dspark", "dflash", "ngram", "mtp+ngram", "off", "ngram-simple"
]
@@ -60,10 +59,10 @@ def urlopen_no_redirect(request, timeout):
return _no_redirect_opener.open(request, timeout = timeout)
-# /api/inference/load and /unload pad their body so a proxy cannot time a slow load
-# out, committing the 200 before the work finishes. A failure found after that travels
-# only in-band under this key (studio/backend/routes/inference.py), so a client that
-# treats any 200 as success reports a failed load as a successful one.
+# /api/inference/load and /unload pad their body so a proxy cannot time a slow load out,
+# committing the 200 before the work finishes. A failure found after that travels only in-band
+# under this key, so a client that treats any 200 as success reports a failed load as a
+# successful one.
_DEFERRED_ERROR_KEY = "_deferred_error"
@@ -141,14 +140,13 @@ def ensure_studio_backend_path() -> None:
def configure_quiet_logging() -> None:
import logging
- # The CLI never configures structlog, so without this every backend INFO
- # line prints. LOG_LEVEL is exported so the worker subprocess inherits it.
+ # The CLI never configures structlog, so without this every backend INFO line prints. LOG_LEVEL
+ # is exported so the worker subprocess inherits it.
level_name = os.environ.setdefault("LOG_LEVEL", "WARNING").upper()
level = getattr(logging, level_name, logging.WARNING)
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
- # Quieting logs must not fail a command before the import that really needs
- # structlog gets to report itself.
+ # Quieting logs must not fail a command before the import that really needs structlog gets to report itself.
try:
import structlog
except ModuleNotFoundError:
@@ -274,8 +272,8 @@ def visible_text(text: str, show_thinking: bool) -> str:
def stream_to_stdout(stream, show_thinking: bool) -> str:
- # Backends yield the full text-so-far on each step (llama.cpp ends with a
- # metadata dict, skipped); print the growing tail, return the raw text.
+ # Backends yield the full text-so-far on each step (llama.cpp ends with a metadata dict,
+ # skipped); print the growing tail, return the raw text.
raw = ""
shown = ""
for chunk in stream:
@@ -318,9 +316,8 @@ def collect_stream(stream, show_thinking: bool) -> str:
def raise_on_streamed_error(stream):
- # Match real backend errors by type (GenStreamError), not the "Error:" text
- # prefix, so a completion whose text opens with "Error:" is not misread as a
- # backend failure.
+ # Match real backend errors by type (GenStreamError), not the "Error:" text prefix, so a
+ # completion whose text opens with "Error:" is not misread as a backend failure.
try:
ensure_studio_backend_path()
from core.inference.orchestrator import GenStreamError
@@ -402,8 +399,8 @@ class ChatBackend:
return self._backend.generate_chat_response(**gen_kwargs)
def close(self) -> None:
- # Shut the worker down directly: the graceful unload_model waits for
- # an ack that compare mode can swallow, hanging exit for minutes.
+ # Shut the worker down directly: the graceful unload_model waits for an ack that compare mode can
+ # swallow, hanging exit for minutes.
try:
if self._kind == "gguf":
self._backend.unload_model()
@@ -620,8 +617,8 @@ def find_studio_server(timeout: float = 3.0) -> Optional[str]:
import urllib.request
base = os.environ.get("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888").rstrip("/")
- # Try the concrete loopback addresses in order and return the first that
- # answers, so the rest of the flow talks to that exact address.
+ # Try the concrete loopback addresses in order and return the first that answers, so the rest of
+ # the flow talks to that exact address.
for candidate in _loopback_candidate_bases(base):
request = urllib.request.Request(
f"{candidate}/api/health", headers = {"User-Agent": _USER_AGENT}
@@ -673,10 +670,9 @@ def verify_studio_identity(base: str, timeout: float = 3.0) -> bool:
parsed = urlparse(base)
host = parsed.hostname or ""
port = parsed.port or (443 if parsed.scheme == "https" else 80)
- # Resolve to one concrete address and talk to *that* address, then bind the
- # proof to (address, port). A name like localhost can resolve to a squatter on
- # ::1 while the real Unsloth is on 127.0.0.1; connecting to the resolved IP and
- # binding to it means a proof relayed from a different address/port won't match.
+ # Resolve to one concrete address and talk to *that* address, then bind the proof to (address,
+ # port). A name like localhost can resolve to a squatter on ::1 while the real Unsloth is on
+ # 127.0.0.1.
try:
ip = socket.getaddrinfo(host, port, type = socket.SOCK_STREAM)[0][4][0]
except Exception:
@@ -689,8 +685,8 @@ def verify_studio_identity(base: str, timeout: float = 3.0) -> bool:
headers = {"User-Agent": _USER_AGENT, "Host": parsed.netloc},
)
try:
- # No redirects: a 302 could relay a real Unsloth's proof (see urlopen_no_redirect).
- # Cap the read: the server is still unverified, so don't trust its length.
+ # No redirects: a 302 could relay a real Unsloth's proof (see urlopen_no_redirect). Cap the read:
+ # the server is still unverified.
with urlopen_no_redirect(request, timeout = timeout) as response:
proof = json.loads(response.read(65536).decode() or "{}").get("proof")
except Exception:
@@ -780,9 +776,8 @@ class HttpChatBackend:
if spec_draft_n_max is not None:
payload["spec_draft_n_max"] = spec_draft_n_max
try:
- # Read the body, don't close at the headers: a slow load commits its 200
- # early and pads until done, so closing here would generate mid-load and
- # discard the only report of a late failure.
+ # Read the body, don't close at the headers: a slow load commits its 200 early and pads until
+ # done, so closing here would generate mid-load and discard the only report of a late failure.
read_json_checking_deferred_error(
self._base + "/api/inference/load",
self._request("POST", "/api/inference/load", payload),
@@ -826,8 +821,7 @@ class HttpChatBackend:
)
def cumulative():
- # Accumulate SSE deltas into the full-text-so-far convention the
- # stream helpers expect.
+ # Accumulate SSE deltas into the full-text-so-far convention the stream helpers expect.
text = ""
with resp:
for raw_line in resp:
@@ -852,8 +846,8 @@ class HttpChatBackend:
if not delta:
continue
text += delta
- # An emoji can arrive split across two deltas as lone
- # surrogate halves: hold back a trailing half, merge pairs.
+ # An emoji can arrive split across two deltas as lone surrogate halves: hold back a trailing
+ # half, merge pairs.
visible = text
if "\ud800" <= visible[-1] <= "\udbff":
visible = visible[:-1]
@@ -881,8 +875,8 @@ def connect_studio_server(
if not base_url:
return None
- # Explicit server (UNSLOTH_STUDIO_URL) we can't safely attach to -> fail loudly;
- # opportunistic local discovery just falls back to a local load.
+ # Explicit server (UNSLOTH_STUDIO_URL) we can't safely attach to means fail loudly; opportunistic
+ # local discovery just falls back to a local load.
explicit = bool(os.environ.get("UNSLOTH_STUDIO_URL"))
def _refuse(reason: str):
@@ -895,8 +889,8 @@ def connect_studio_server(
)
raise typer.Exit(code = 1)
- # Only hand the self-issued JWT (signed with the local secret) to loopback: a
- # remote URL is unverified and a real remote Unsloth would reject it anyway.
+ # Only hand the self-issued JWT (signed with the local secret) to loopback: a remote URL is
+ # unverified and a real remote Unsloth would reject it anyway.
if not is_loopback_url(base_url):
return _refuse(
"it isn't a local Unsloth, so a self-issued token can't "
diff --git a/unsloth_cli/_model_catalog.py b/unsloth_cli/_model_catalog.py
index 29ca049af71..34555d3f2a9 100644
--- a/unsloth_cli/_model_catalog.py
+++ b/unsloth_cli/_model_catalog.py
@@ -137,19 +137,17 @@ def exported_entries() -> List[ModelEntry]:
if export_type == "gguf" and _gguf_export_task(path, name, base) in NON_CHAT_TASKS:
continue
if export_type == "gguf":
- # No complete quant means every candidate is zero-byte or short a shard, which
- # survives resolve_model_config() and fails only at load time. Drop it only when
- # positively unloadable: a falsy result also means "could not tell", and hiding a
- # real export on a transient read is worse than listing it.
+ # No complete quant means every candidate is zero-byte or short a shard, which survives
+ # resolve_model_config() and fails only at load time. Drop it only when positively unloadable: a
+ # falsy result also means "could not tell".
load_id = _preferred_complete_gguf(path)
if not load_id and not _local_dir_holds_a_payload(Path(path)):
continue
else:
load_id = None
- # Positively loadable, not merely "not torn". scan_exported_models types a
- # checkpoint as "lora" on adapter_config.json alone, so an interrupted export
- # leaves a config and no weights: nothing torn because there is no payload, and
- # the picker offered a directory load_peft_weights raises ValueError on.
+ # Positively loadable, not merely "not torn". scan_exported_models types a checkpoint as "lora" on
+ # adapter_config.json alone, so an interrupted export leaves a config and no weights and the
+ # picker offered a directory load_peft_weights raises ValueError on.
if not _local_dir_holds_a_payload(Path(path)):
continue
entries.append(ModelEntry("Fine-tunes", name, export_type, load_id or path))
@@ -190,14 +188,13 @@ def _reachable_snapshots(repo_path: Path, load_id: Optional[str] = None) -> List
return []
pinned = _pinned_snapshot(repo_path, load_id)
if pinned is not None:
- # Returned as validated, not re-checked against `available`. inventory_scan resolves
- # the snapshot it pins while cache_path keeps the configured spelling, so under a
- # symlinked cache root the two name one directory and fail lexical equality, and the
- # membership test would drop a good pin back onto the ref it was pinned away from.
+ # Returned as validated, not re-checked against `available`: inventory_scan resolves the snapshot
+ # it pins while cache_path keeps the configured spelling, so under a symlinked cache root the two
+ # name one directory and fail lexical equality, and the membership test would drop a good pin.
return [pinned]
try:
- # ValueError too: an undecodable ref raises UnicodeDecodeError, which is not an
- # OSError, and uncaught it leaves _safe hiding every Downloaded row over one repo.
+ # ValueError too: an undecodable ref raises UnicodeDecodeError, which is not an OSError, and
+ # uncaught it leaves _safe hiding every Downloaded row over one repo.
ref = (repo_path / "refs" / "main").read_text(encoding = "utf-8").strip()
except (OSError, ValueError):
ref = ""
@@ -219,8 +216,8 @@ def _complete_quants(snapshot: Path) -> Optional[set]:
complete = complete_snapshot_variants(str(snapshot))
except Exception:
return None
- # Empty here means the check could not tell, not that nothing loads: a genuinely
- # incomplete repo arrives partial and never reaches the picker. Describe what is on disk.
+ # Empty here means the check could not tell, not that nothing loads: a genuinely incomplete repo
+ # arrives partial and never reaches the picker.
return set(complete) or None
@@ -425,8 +422,8 @@ def cached_entries() -> List[ModelEntry]:
# A cached embedding/CLIP repo has task None like any chat repo; can_chat is the gate.
if row.get("capabilities", {}).get("can_chat") is False:
continue
- # A diffusion repo also carries no task, and its pipeline root has no config for
- # can_chat to read, so neither gate above catches it.
+ # A diffusion repo also carries no task, and its pipeline root has no config for can_chat to read,
+ # so neither gate above catches it.
if row.get("diffusers"):
continue
entries.append(ModelEntry("Downloaded", row["repo_id"], "", _cached_model_load_id(row)))
@@ -484,15 +481,15 @@ def _local_dir_holds_a_payload(path: Path) -> bool:
)
from utils.paths.path_utils import is_appledouble_metadata
- # A pipeline keeps its weights in component subdirs, so the torn test below reads an
- # empty root and would call every pipeline unserviceable.
+ # A pipeline keeps its weights in component subdirs, so the torn test below reads an empty root
+ # and would call every pipeline unserviceable.
if _is_diffusers_pipeline_dir(path):
return True
if _local_payload_is_torn(path):
return False
# iterdir, not glob("*.gguf"): the glob is case-sensitive on Linux and macOS while
- # _is_main_gguf_filename lowercases first, so a folder holding Model.GGUF was classified
- # as a GGUF model and then dropped by this gate, which decides whether it is listed.
+ # _is_main_gguf_filename lowercases first, so a folder holding Model.GGUF was classified as a
+ # GGUF model and then dropped by this gate.
try:
children = list(path.iterdir())
except OSError:
@@ -572,8 +569,8 @@ def local_folder_entries() -> List[ModelEntry]:
) or model.partial:
continue
is_gguf = model.model_format == "gguf" or model.path.lower().endswith(".gguf")
- # No format gate: _dir_model_format reports only "gguf" or None, so a safetensors
- # checkpoint arrives as None and a "safetensors" literal dropped every non-GGUF model.
+ # No format gate: _dir_model_format reports only "gguf" or None, so a safetensors checkpoint
+ # arrives as None and a "safetensors" literal dropped every non-GGUF model.
if _local_model_task(model) in NON_CHAT_TASKS:
continue
# No format gate, so embedding and CLIP exports get through; only this stops them.
@@ -584,10 +581,9 @@ def local_folder_entries() -> List[ModelEntry]:
if _local_is_a_diffusers_pipeline(model):
continue
target = model.load_id or model.id
- # A GGUF DIRECTORY goes through detect_gguf_model, which sorts by size and takes the
- # largest complete file, commonly the F16, while cached and exported rows resolve a
- # Q4-class quant. Same folder, dramatically bigger load by source alone, so an OOM
- # rather than a preference. Resolve it the same way here.
+ # A GGUF DIRECTORY goes through detect_gguf_model, which sorts by size and takes the largest
+ # complete file, commonly the F16, while cached and exported rows resolve a Q4-class quant. Same
+ # folder, dramatically bigger load by source alone, so an OOM rather than a preference.
if is_gguf:
target = _preferred_complete_gguf(target) or target
entries.append(
diff --git a/unsloth_cli/_studio_deps.py b/unsloth_cli/_studio_deps.py
index 4438f77fb49..b236bf69c0a 100644
--- a/unsloth_cli/_studio_deps.py
+++ b/unsloth_cli/_studio_deps.py
@@ -192,8 +192,8 @@ def _venv_site_packages(root: Path) -> List[Path]:
if versioned.is_dir():
return [versioned]
- # Test fixtures and incomplete venvs may not have a runnable interpreter or
- # version entry yet. One directory is unambiguous; several are not.
+ # Test fixtures and incomplete venvs may not have a runnable interpreter or version entry yet.
+ # One directory is unambiguous; several are not.
candidates = sorted(root.glob("lib/python*/site-packages"))
return candidates if len(candidates) == 1 else []
@@ -235,9 +235,8 @@ def _distributions_in(root: Path) -> Optional[tuple[Dict[str, str], set[str]]]:
if name:
canonical = _canonical(name)
version = dist.version or ""
- # pip renames the outgoing record to a `~` sibling while it lays
- # the replacement down, so a kill there can leave the backup as
- # the only record. It reads as one healthy version, but pip calls
+ # pip renames the outgoing record to a `~` sibling while laying the replacement down, so a kill
+ # there can leave the backup as the only record: it reads as one healthy version, but pip calls
# it an invalid distribution and the payload is renamed with it.
if not version or canonical in found or stem.startswith("~"):
conflicts.add(canonical)
@@ -258,9 +257,9 @@ def _requirements_root_in(root: Path) -> Optional[Path]:
reqs = path / "studio" / "backend" / "requirements"
if reqs.is_dir():
return reqs
- # An editable install can keep studio/ only in its source checkout. Ask the
- # foreign interpreter to follow its .pth/finder instead of treating the
- # absent site-packages copy as an incomplete environment.
+ # An editable install can keep studio/ only in its source checkout. Ask the foreign interpreter
+ # to follow its .pth/finder instead of treating the absent site-packages copy as an incomplete
+ # environment.
probe = (
"import json, pathlib, studio; "
"print(json.dumps(str(pathlib.Path(studio.__file__).resolve().parent / "
@@ -312,8 +311,7 @@ def install_state(extra_roots: Sequence[Path] = (), deep: bool = False) -> dict:
"missing": [],
"reason": "studio_install_manifest_missing",
}
- # The requested managed venv is the subject, even though the helper above
- # came from this CLI's own tree.
+ # The requested managed venv is the subject, even though the helper above came from this CLI's own tree.
root = _managed_root(extra_roots) or _venv_root_for_module(module)
foreign = root is not None and _resolved(root) != _resolved(Path(sys.prefix))
foreign_distributions = _distributions_in(root) if foreign else None
@@ -321,9 +319,9 @@ def install_state(extra_roots: Sequence[Path] = (), deep: bool = False) -> dict:
installed_conflicts = foreign_distributions[1] if foreign_distributions is not None else set()
req_root = _requirements_root_in(root) if foreign else None
if foreign and (foreign_distributions is None or req_root is None):
- # Never answer for the caller when the requested environment cannot be
- # inspected. That can turn a torn or ambiguous managed venv into a
- # healthy result merely because the external CLI has matching packages.
+ # Never answer for the caller when the requested environment cannot be inspected: that can turn
+ # a torn or ambiguous managed venv into a healthy result merely because the external CLI has
+ # matching packages.
return {
"ok": False,
"manifest_ok": False,
@@ -343,22 +341,19 @@ def install_state(extra_roots: Sequence[Path] = (), deep: bool = False) -> dict:
kwargs["installed_conflicts"] = installed_conflicts
if deep and _verify_install_supports(module, "deep"):
kwargs["deep"] = True
- # Without that venv's site-packages the scan answers for this
- # interpreter's tree. Guarded separately: a module new enough
- # for `deep` may predate `scan_paths`.
+ # Without that venv's site-packages the scan answers for this interpreter's tree. Guarded
+ # separately: a module new enough for `deep` may predate `scan_paths`.
if _verify_install_supports(module, "scan_paths"):
paths = [str(path) for path in _venv_site_packages(root)]
if paths:
kwargs["scan_paths"] = paths
return module.verify_install(**kwargs)
- # Off for `desktop-capabilities`: the Tauri preflight times it out at
- # 10s, and a probe that overruns repairs a healthy venv. `verify-install`
- # is untimed, so it opts in.
+ # Off for `desktop-capabilities`: the Tauri preflight times it out at 10s and a probe that
+ # overruns repairs a healthy venv. `verify-install` is untimed, so it opts in.
deep_kwargs = {"deep": True} if (deep and _verify_install_supports(module, "deep")) else {}
state = module.verify_install(root = root, **deep_kwargs)
if foreign and not state["deps_ok"]:
- # The manifest came from another venv but the dependency walk ran
- # here, so it says nothing about that venv.
+ # The manifest came from another venv but the dependency walk ran here, so it says nothing about that venv.
state = dict(state, deps_ok = True, missing = [])
state["ok"] = state["manifest_ok"]
state["reason"] = None if state["ok"] else state["reason"]
@@ -405,11 +400,10 @@ def _scan_paths() -> Dict[str, list]:
return {"path": paths} if paths else {}
-# Top-level dirs several wheels write into, so one uninstall deletes another's
-# files and the survivor's RECORD describes a file nothing recreates. einx and
-# torchao both ship test/conftest.py, and install_python_stack.py
-# force-reinstalls torchao every update; unsloth_zoo <= 2026.8.5 shipped
-# tests/ and scripts/ into the same squatted namespace.
+# Top-level dirs several wheels write into, so one uninstall deletes another's files and the
+# survivor's RECORD describes a file nothing recreates. einx and torchao both ship
+# test/conftest.py and install_python_stack.py force-reinstalls torchao every update;
+# unsloth_zoo <= 2026.8.5 shipped tests/ and scripts/ into the same squatted namespace.
_SHARED_NON_RUNTIME_ROOTS = frozenset(
(
"test",
@@ -426,9 +420,9 @@ _SHARED_NON_RUNTIME_ROOTS = frozenset(
)
)
-# Rewritten in place by our own setup: setup.ps1/setup.sh run `npm install`
-# inside the installed tree, and npm dedupes hoisted entries under
-# legacy-peer-deps, shrinking the lockfile below its recorded size.
+# Rewritten in place by our own setup: setup.ps1/setup.sh run `npm install` inside the installed
+# tree, and npm dedupes hoisted entries under legacy-peer-deps, shrinking the lockfile below its
+# recorded size.
_INSTALLER_REWRITTEN_NAMES = frozenset(("package-lock.json",))
@@ -486,8 +480,8 @@ def _installed_distribution_groups():
version = None
groups.setdefault(_canonical(name), []).append((dist, name, bool(version)))
continue
- # Wheel metadata directory names escape name separators as underscores,
- # so splitting off the final version is unambiguous.
+ # Wheel metadata directory names escape name separators as underscores, so splitting off the
+ # final version is unambiguous.
path = getattr(dist, "_path", None)
stem = os.path.basename(os.fspath(path)) if path is not None else ""
path_name, separator, _version = stem.removesuffix(".dist-info").rpartition("-")
@@ -582,15 +576,14 @@ def damaged_installed_files(limit: int = 8) -> List[str]:
entries: List[tuple] = []
owners: Dict[str, int] = {}
for records in _installed_distribution_groups().values():
- # An unreadable record cannot be trusted to describe the package tree
- # any more than a duplicated one can.
+ # An unreadable record cannot be trusted to describe the package tree any more than a duplicated one can.
ambiguous = len(records) > 1 or not all(readable for _dist, _name, readable in records)
for dist, name, _readable in records:
try:
record = dist.read_text("RECORD")
except Exception:
- # An unreadable or absent RECORD says nothing about damage: editable
- # installs and system packages legitimately have none.
+ # An unreadable or absent RECORD says nothing about damage: editable installs and system
+ # packages legitimately have none.
continue
if not record:
continue
@@ -599,8 +592,8 @@ def damaged_installed_files(limit: int = 8) -> List[str]:
# A trailing slash is a directory entry, which has nothing to check.
if not rel or rel.endswith("/"):
continue
- # Installer-owned metadata is rewritten in place and drifts from the
- # size recorded inside itself; .pyc is regenerated from source.
+ # Installer-owned metadata is rewritten in place and drifts from the size recorded inside
+ # itself; .pyc is regenerated from source.
if ".dist-info/" in rel or ".egg-info/" in rel or rel.endswith(".pyc"):
continue
try:
@@ -608,14 +601,13 @@ def damaged_installed_files(limit: int = 8) -> List[str]:
except Exception:
continue
key = os.path.normcase(str(target))
- # Before either filter: a row we cannot verify still owns the path
- # it claims, so another distribution's size is ambiguous too.
+ # Before either filter: a row we cannot verify still owns the path it claims, so another
+ # distribution's size is ambiguous too.
owners[key] = owners.get(key, 0) + 1
if ambiguous or _shared_non_runtime(rel):
continue
- # The size field is optional and real wheels do leave it blank. Keep
- # the row anyway with an unknown size: existence is still checkable,
- # and dropping the row meant a deletion went unreported.
+ # The size field is optional and real wheels do leave it blank. Keep the row with an unknown
+ # size: existence is still checkable, and dropping it meant a deletion went unreported.
recorded: Optional[int] = None
if len(row) >= 3 and row[2] and not _installer_rewritten(rel):
try:
@@ -629,15 +621,13 @@ def damaged_installed_files(limit: int = 8) -> List[str]:
try:
info = target.stat()
except OSError:
- # Multiple ownership makes the recorded SIZES ambiguous; it cannot
- # explain the file being gone, so this branch runs for shared paths
- # too.
+ # Multiple ownership makes the recorded SIZES ambiguous; it cannot explain the file being gone,
+ # so this branch runs for shared paths too.
found.append(f"{name}: {rel} is missing")
else:
if not stat.S_ISREG(info.st_mode):
- # A directory standing in for a recorded module still imports as
- # something else, and on POSIX its st_size (commonly 4096) can
- # sail past the shrinkage test.
+ # A directory standing in for a recorded module still imports as something else, and on POSIX
+ # its st_size (commonly 4096) can sail past the shrinkage test.
found.append(f"{name}: {rel} is not a regular file")
elif owners[key] == 1 and recorded is not None and info.st_size < recorded:
found.append(f"{name}: {rel} is {info.st_size} bytes, expected {recorded}")
@@ -653,10 +643,9 @@ def running_outside_managed_venv(extra_roots: Sequence[Path] = ()) -> bool:
anything answered from this interpreter would then describe the wrong tree.
"""
if not (Path(sys.prefix) / "pyvenv.cfg").is_file():
- # Not a venv at all. On Colab setup.sh deliberately installs the backend
- # into the system Python, whose distro-packaged RECORDs legitimately list
- # files the distro never installed (PEP 627), so a file check here
- # describes the distro rather than Unsloth.
+ # Not a venv at all. On Colab setup.sh deliberately installs the backend into the system Python,
+ # whose distro-packaged RECORDs legitimately list files the distro never installed (PEP 627), so
+ # a file check here describes the distro rather than Unsloth.
return True
return _managed_root(extra_roots) is not None
@@ -672,9 +661,9 @@ def _missing_studio_packages() -> List[str]:
return []
-# studio.txt names distributions, ModuleNotFoundError names the import. Only
-# pairs differing by more than PEP 503 normalisation need an entry, and each
-# import name below is itself a real but unrelated PyPI project.
+# studio.txt names distributions, ModuleNotFoundError names the import. Only pairs differing by
+# more than PEP 503 normalisation need an entry, and each import name below is itself a real but
+# unrelated PyPI project.
_IMPORT_TO_DISTRIBUTION = {
"jwt": "pyjwt",
"docx": "python-docx",
@@ -693,12 +682,11 @@ def studio_backend_imports(feature: str = "This command", *, studio_only: bool =
yield
except ModuleNotFoundError as exc:
studio_missing = _missing_studio_packages()
- # The failed import may not be a studio dependency at all: `train`
- # reaches torch through the same wrapper, so only offer the extra when
- # it helps.
+ # The failed import may not be a studio dependency at all: `train` reaches torch through the same
+ # wrapper, so only offer the extra when it helps.
trigger = exc.name or ""
- # Match on the owning distribution, never the import: `pip install jwt`
- # (or fastmcp.server) installs the wrong thing or nothing at all.
+ # Match on the owning distribution, never the import: `pip install jwt` (or fastmcp.server)
+ # installs the wrong thing or nothing at all.
top = trigger.split(".", 1)[0]
needed = _IMPORT_TO_DISTRIBUTION.get(top, top)
wanted = _canonical(needed)
diff --git a/unsloth_cli/_studio_runtime_gate.py b/unsloth_cli/_studio_runtime_gate.py
index 9e9ce3eba34..b0cd6ead44b 100644
--- a/unsloth_cli/_studio_runtime_gate.py
+++ b/unsloth_cli/_studio_runtime_gate.py
@@ -330,15 +330,10 @@ def ensure_managed_environment_is_idle(studio_home: Path) -> None:
if int(process.get("ProcessId") or -1) > 0
}
- # The updater may itself have been entered through the managed console shim,
- # so exempt verified launcher ancestors only: a managed backend that starts
- # an update as its child must still block replacement.
- #
- # venv\Scripts\python.exe is a redirector (bpo-34977): it starts base Python
- # as a child and waits, so a venv launch arrives as python.exe -> us. Our
- # image is then the base interpreter while sys.executable still names the
- # redirector, which identifies our direct parent as that launcher rather
- # than a live consumer. Exempt that one hop only; above it must be a shim.
+ # Exempt verified launcher ancestors only: a managed backend starting an update as its child must
+ # still block replacement.
+ # venv Scripts\python.exe is a redirector (bpo-34977), so exempt that one hop: our image is base
+ # Python while sys.executable names the shim.
excluded_pids = {os.getpid()}
descendant_pid = os.getpid()
self_executable = (process_by_pid.get(os.getpid()) or {}).get("ExecutablePath")
diff --git a/unsloth_cli/_studio_stage.py b/unsloth_cli/_studio_stage.py
index abd3cc4338c..dae50bde94d 100644
--- a/unsloth_cli/_studio_stage.py
+++ b/unsloth_cli/_studio_stage.py
@@ -188,8 +188,7 @@ def probe_console_script(venv: Path, env: dict[str, str]) -> None:
try:
result = _run([str(script), "-h"], cwd = venv.parent, env = env)
except OSError as exc:
- # A shebang naming an interpreter that is not there fails here, not with a
- # return code: the kernel refuses the exec.
+ # A shebang naming a missing interpreter fails here, not with a return code: the kernel refuses the exec.
raise StageError(f"staged launcher is not executable: {exc}") from exc
if result.returncode != 0:
raise StageError(f"staged launcher failed to start: {result.stderr.strip()[-2000:]}")
diff --git a/unsloth_cli/_system_dir_guard.py b/unsloth_cli/_system_dir_guard.py
index d7067547e62..39afbddd850 100644
--- a/unsloth_cli/_system_dir_guard.py
+++ b/unsloth_cli/_system_dir_guard.py
@@ -19,9 +19,8 @@ STUDIO_HOME against the working directory.
import os as _os
-# Set by Unsloth Desktop on every CLI child it owns (process.rs). Forging it
-# grants nothing: anyone who can set a child's environment can set its working
-# directory, and the move lands inside the caller's own account.
+# Set by Unsloth Desktop on every CLI child it owns (process.rs); forging it grants nothing,
+# the move lands in the caller's own account.
DESKTOP_MANAGED_ENV = "UNSLOTH_DESKTOP_MANAGED"
# The directory process.rs pins, so an older desktop lands in the same place.
@@ -59,8 +58,7 @@ def windows_roots(
roots.append(value)
if roots:
return roots
- # No Windows installation found: keep the guard alive on SystemRoot or the
- # default, never on a user-settable value.
+ # No Windows installation found: keep the guard on SystemRoot or the default, never on a user-settable value.
return [system_root or r"C:\Windows"]
@@ -144,9 +142,8 @@ def _outside_windows(candidate, windirs, pathmod, sep):
norm = _normalize(candidate, pathmod)
for windir in windirs:
windir_norm = _normalize(windir, pathmod)
- # A root-relative candidate carries no drive, so it equals no
- # drive-qualified root: "\Windows\System32\config\systemprofile" is
- # SYSTEM's profile on whichever drive, so compare that spelling too.
+ # A root-relative candidate carries no drive, so compare that spelling too:
+ # "\Windows\System32\config\systemprofile" is SYSTEM's profile on whichever drive.
for form in (windir_norm, pathmod.splitdrive(windir_norm)[1]):
if not form:
continue
@@ -181,8 +178,7 @@ def safe_user_dir(
for candidate in candidates:
if not _outside_windows(candidate, windirs, pathmod, sep):
continue
- # USERPROFILE and ~ can name the public profile themselves, so the check
- # is on the folder, not on which variable it came from.
+ # USERPROFILE and ~ can name the public profile, so check the folder, not which variable it came from.
if (
not allow_public
and public
@@ -193,11 +189,9 @@ def safe_user_dir(
return None
-# Commands the desktop runs that take no path from a user: Unsloth resolves its
-# venv, llama.cpp, auth, pid files and logs from the Unsloth home. `update` is here
-# so an older desktop can still upgrade from the tray. Everything else keeps the
-# hard error. Matched whole, not on the first word, since `studio update --local
-# <path>` resolves that path against the working directory.
+# Commands the desktop runs that take no path from a user; `update` is here so an older desktop
+# can still upgrade from the tray. Matched whole, since `studio update --local <path>`
+# resolves against the working directory.
_STUDIO_COMMANDS = (
("provision-desktop-auth",),
("desktop-capabilities",),
@@ -226,9 +220,8 @@ def _is_desktop_backend_launch(rest):
return True
-# Subcommands that take a path from the caller, by argument or by environment.
-# `run` takes --model and a raw llama-server tail; `update --local` installs
-# from a checkout the caller names.
+# Subcommands that take a path from the caller: `run` takes --model and a raw llama-server
+# tail, `update --local` a checkout.
_PATH_TAKING_STUDIO_COMMANDS = ("run", "update")
@@ -238,8 +231,7 @@ def _carries_a_value(arg):
return True
if arg.startswith("--"):
return "=" in arg
- # A short option carries its value in the same token from the second
- # character on: `-f.\dist` is Click's spelling of `--frontend .\dist`.
+ # A short option carries its value in the same token: `-f.\dist` is Click's spelling of `--frontend .\dist`.
return len(arg) > 2
@@ -251,10 +243,8 @@ def _takes_a_path(rest):
return False
if rest[0] in _PATH_TAKING_STUDIO_COMMANDS:
return tuple(rest) not in _STUDIO_COMMANDS
- # rest[0] is the subcommand name, skipped unless it is a flag (then there is
- # no subcommand). An attached value hides inside its own token, so a leading
- # dash does not clear it: Click reads both `--frontend=.\dist` and the short
- # `-f.\dist` as a value, and either carries a path a relocation would rebase.
+ # rest[0] is the subcommand name, skipped unless it is a flag. A leading dash does not clear
+ # an attached value: Click reads `--frontend=.\dist` and `-f.\dist` as values with a path.
tail = rest if rest[0].startswith("-") else rest[1:]
return any(_carries_a_value(arg) for arg in tail)
@@ -268,21 +258,18 @@ def is_relocatable_invocation(argv, environ):
args = [arg for arg in argv if arg]
if not args:
return False
- # Click handles top-level -h/--help/--version eagerly, before the callback
- # this runs from, so only `studio --help` actually arrives here.
+ # Click handles top-level -h/--help/--version eagerly, so only `studio --help` arrives here.
if all(arg in _HELP_FLAGS for arg in args):
return True
if args[0] != "studio":
- # The marker is inherited by everything the backend spawns, so it
- # authorises the desktop's studio commands and nothing else: rebasing
- # `train --dataset .\data.json` under a stray one would be worse than the
- # refusal it replaced.
+ # The marker is inherited by everything the backend spawns, so it authorises the desktop's
+ # studio commands only: rebasing `train --dataset .\data.json` under a stray one is worse
+ # than refusing.
return False
rest = args[1:]
if environ.get(DESKTOP_MANAGED_ENV) == "1" and not _takes_a_path(rest):
- # The marker covers a desktop build whose command shape this CLI does not
- # know yet, but must not widen the set to path-carrying commands: `studio
- # run --model .\local.gguf` from a marked shell would be rebased.
+ # The marker covers desktop builds whose command shape this CLI does not know yet, but must
+ # not widen to path-carrying commands like `studio run --model .\local.gguf`.
return True
if rest and all(arg in _HELP_FLAGS for arg in rest):
return True
@@ -291,11 +278,9 @@ def is_relocatable_invocation(argv, environ):
return tuple(rest) in _STUDIO_COMMANDS
-# Path overrides the caller may have written relative to the folder being left.
-# Unsloth resolves them with Path.resolve(), which anchors a relative value to the
-# working directory, so moving first would silently retarget them.
+# Path overrides written relative to the folder being left: Unsloth resolves them with
+# Path.resolve(), so moving first would silently retarget them.
_RELATIVE_PATH_ENV = (
- # Unsloth roots: storage_roots.py.
"UNSLOTH_STUDIO_HOME",
"STUDIO_HOME",
"UNSLOTH_STUDIO_DOCUMENTS_HOME",
@@ -303,7 +288,6 @@ _RELATIVE_PATH_ENV = (
"UNSLOTH_STUDIO_SANDBOX_HOME",
# `studio update` reads it, and that command relocates.
"STUDIO_LOCAL_REPO",
- # Engine and tool locations the user may point somewhere of their own.
"UNSLOTH_LLAMA_CPP_PATH",
"UNSLOTH_LLAMA_CPP_SCRIPTS_DIR",
"UNSLOTH_SD_CPP_PATH",
@@ -312,16 +296,14 @@ _RELATIVE_PATH_ENV = (
"WHISPER_SERVER_PATH",
"SD_CLI_PATH",
"SD_SERVER_PATH",
- # Model files llama-server reads straight from the environment, and Unsloth
- # reads back when it sizes a launch (llama_cpp.py). The URL and HF-repo
- # spellings are deliberately absent: they name no local file.
+ # Model files llama-server reads from the environment and Unsloth reads back when sizing a
+ # launch (llama_cpp.py). URL and HF-repo spellings are absent: they name no local file.
"LLAMA_ARG_MODEL",
"LLAMA_ARG_MMPROJ",
"LLAMA_ARG_MODEL_DRAFT",
"LLAMA_ARG_SPEC_DRAFT_MODEL",
- # Read straight from the environment as a file path: the ASIC table by
- # unsloth/import_fixes.py, the vLLM cache root by Unsloth when the caller set
- # it themselves (storage_roots.py only fills a blank one).
+ # Read straight from the environment as a file path: the ASIC table (import_fixes.py) and the
+ # vLLM cache root when the caller set it (storage_roots.py fills only a blank one).
"AMDGPU_ASIC_ID_TABLE_PATH",
"VLLM_CACHE_ROOT",
# A custom ggml backend, preserved into the llama.cpp child (llama_cpp.py).
@@ -337,7 +319,6 @@ _RELATIVE_PATH_ENV = (
"OLLAMA_MODELS",
"DG_VISUAL_BIN",
"UNSLOTH_DG_SHIM",
- # Caches.
"UNSLOTH_COMPILE_LOCATION",
"TORCHINDUCTOR_CACHE_DIR",
"UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR",
@@ -348,11 +329,10 @@ _RELATIVE_PATH_ENV = (
"HF_XET_CACHE",
"HF_DATASETS_CACHE",
"HF_ASSETS_CACHE",
- # The credential file: a relative value would follow the child and lose
- # access to gated repos.
+ # The credential file: a relative value would follow the child and lose access to gated repos.
"HF_TOKEN_PATH",
- # Read as written, and authoritative when non-blank (storage_roots.py), so
- # `unsloth studio update` would install from a different cache after a move.
+ # Authoritative when non-blank (storage_roots.py), so `unsloth studio update` would install
+ # from a different cache after a move.
"UV_CACHE_DIR",
"TRANSFORMERS_CACHE",
"SENTENCE_TRANSFORMERS_HOME",
@@ -365,26 +345,20 @@ _RELATIVE_PATH_ENV = (
"CUDA_ROOT",
)
-# Pinned when it can be, but never at the cost of the move. `studio update
-# --local` is the only command that reads STUDIO_LOCAL_REPO and it takes a path,
-# so it keeps the hard error; the bare update that does relocate drops the value
-# before anything reads it (commands/studio.py). Refusing a System32 update over
-# a stale setting nothing was going to look at would defeat the fallback this
-# guard exists for, so an unresolvable one is left behind instead.
+# Pinned when possible, never at the cost of the move: only `studio update --local` reads
+# STUDIO_LOCAL_REPO and it keeps the hard error, so an unresolvable one is left behind rather
+# than defeating the fallback.
_BEST_EFFORT_ENV = frozenset(("STUDIO_LOCAL_REPO",))
# The most a Windows environment variable holds, terminator included.
_WINDOWS_ENV_VALUE_LIMIT = 32767
-# The separator is Windows', not the host's: this guard only ever runs on
-# Windows, and os.pathsep would split "D:\\shared" apart anywhere else.
+# The separator is Windows', not the host's: os.pathsep would split "D:\shared" apart anywhere else.
_PATH_LIST_SEPARATOR = ";"
-# Values holding several separated directories, anchored entry by entry. A
-# relative PYTHONPATH entry is resolved at import time, so a move would let
-# whatever sits in the new directory shadow a managed import. PATH is left out on
-# purpose: refusing the whole move over one unresolvable entry in a list that long
-# would cost more than it protects.
+# Multi-directory values, anchored entry by entry: a relative PYTHONPATH entry is resolved at
+# import time and would let the new directory shadow a managed import. PATH is left out:
+# refusing the whole move over one unresolvable entry costs more than it protects.
_PATH_LIST_ENV = (
"UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH",
"CUDA_RUNTIME_DLL_DIR",
@@ -416,9 +390,8 @@ def pin_relative_overrides(
continue
if anchored is not None:
if len(anchored) >= _WINDOWS_ENV_VALUE_LIMIT:
- # Anchoring a value that was already near the limit can cross it,
- # and a variable Windows will not accept is a failure to report
- # here rather than one to discover in the next process.
+ # Anchoring a value already near the limit can cross it, and a variable Windows will not
+ # accept is a failure to report here.
raise ValueError(
f"{name} does not fit in an environment variable once it "
"names its folder in full"
@@ -429,8 +402,7 @@ def pin_relative_overrides(
raw = environ.get(name) or ""
if not raw.strip():
continue
- # Each entry is anchored on its own: one relative entry changes what the
- # whole list means.
+ # Each entry is anchored on its own: one relative entry changes what the whole list means.
entries = raw.split(_PATH_LIST_SEPARATOR)
anchored_entries = [
_anchor_list_entry(name, e, cwd, pathmod, abspath, expandvars, expanduser)
@@ -439,8 +411,6 @@ def pin_relative_overrides(
if anchored_entries != entries:
joined = _PATH_LIST_SEPARATOR.join(anchored_entries)
if len(joined) >= _WINDOWS_ENV_VALUE_LIMIT:
- # A value Windows will not accept is a failure to report here,
- # not one to discover when the next process is started.
raise ValueError(
f"{name} does not fit in an environment variable once each "
"entry names its folder in full"
@@ -450,22 +420,17 @@ def pin_relative_overrides(
return pinned
-# Values a consumer deliberately does not read as a plain path: anchoring one
-# changes what it means rather than moving a folder. Each exemption names the
-# variables whose reader proves it, since the syntax is only special there and a
-# directory really called "[llama]" or "%data%" is legal on Windows.
+# Values a consumer does not read as a plain path: anchoring one changes its meaning. Each
+# exemption names the reader that proves it, since a directory really called "[llama]" is
+# legal on Windows.
-# MLX_HOSTFILE holds either a filename or the host list itself, as JSON
-# (`unsloth_cli/_inference.py`, `_json_rank_count_from_env`).
+# MLX_HOSTFILE holds either a filename or the host list itself as JSON (_inference.py, _json_rank_count_from_env).
_INLINE_JSON_ENV = frozenset(("MLX_HOSTFILE", "MLX_IBV_DEVICES"))
-# Names whose readers disagree about %VAR% and $VAR: huggingface_hub calls
-# expandvars on HF_HOME (and on the XDG_CACHE_HOME it defaults from), HF_HUB_CACHE
-# and HF_ASSETS_CACHE, and Unsloth calls it on SENTENCE_TRANSFORMERS_HOME, but
-# Unsloth's own hf_cache_settings._canonical() does not, so it would read
-# %LOCALAPPDATA%\hf as a relative folder. Expanding here before deciding settles
-# it: both readers then see one absolute path. Scoped to these names because a
-# directory really called "%data%" is legal, and every other name is read as one.
+# Readers disagree about %VAR%/$VAR: huggingface_hub expandvars HF_HOME, XDG_CACHE_HOME,
+# HF_HUB_CACHE and HF_ASSETS_CACHE and Unsloth does SENTENCE_TRANSFORMERS_HOME, but
+# hf_cache_settings._canonical() does not. Expanding here makes both readers see one absolute
+# path.
_EXPANDED_ENV = frozenset(
(
"HF_HOME",
@@ -478,9 +443,8 @@ _EXPANDED_ENV = frozenset(
)
)
-# The pre-quant allowlist skips a bare on/off token precisely so there is no
-# "allow all" mode (`diffusion_prequant.py`); anchoring one would turn it into a
-# real allowlisted directory.
+# The pre-quant allowlist skips a bare on/off token so there is no "allow all" mode
+# (diffusion_prequant.py); anchoring one would make it a real allowlisted directory.
_TOGGLE_ENV = frozenset(("UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH",))
_TOGGLE_TOKENS = frozenset(("1", "true", "yes", "on", "0", "false", "no", "off"))
@@ -524,16 +488,15 @@ def pin_relative_sys_path(
if not isinstance(entry, str):
continue
try:
- # The empty entry is the working directory by definition; anything
- # else has to name something that is really there.
+ # The empty entry is the working directory by definition; anything else has to name something really there.
if entry.strip() and not exists(entry):
continue
anchored = _anchor_list_entry(
"PYTHONPATH", entry, cwd, pathmod, abspath, None, expanduser
)
except Exception:
- # Best effort, unlike the environment: an import root this process
- # already holds is not worth refusing the move over.
+ # Best effort, unlike the environment: an import root this process already holds is not worth
+ # refusing the move over.
continue
if anchored != entry:
syspath[index] = anchored
@@ -595,22 +558,17 @@ def _anchor(
"""
original = value = (value or "").strip()
if value.startswith("~"):
- # Written out rather than skipped: only some readers call expanduser
- # first, and llama_cpp.py hands UNSLOTH_LLAMA_CPP_PATH straight to Path(),
- # so a move would leave it naming a folder called "~" under the new
- # directory. It is what the caller meant either way.
+ # Written out rather than skipped: llama_cpp.py hands UNSLOTH_LLAMA_CPP_PATH straight to
+ # Path(), so a move would leave it naming a folder called "~".
value = (expanduser or pathmod.expanduser)(value)
if name in _EXPANDED_ENV and value:
- # Written out, so the reader that expands and the one that does not land
- # in the same folder. An unset variable is left exactly as written.
+ # Written out, so the reader that expands and the one that does not land in the same folder.
+ # An unset variable is left as written.
expandvars = expandvars or _os.path.expandvars
settled = _expand_settled(value, expandvars)
if settled is None:
- # One pass does not settle it, so writing the result back would have
- # the reader expand it a second time. What the reader does see is one
- # pass: if that names a folder on its own the value is safe to leave
- # alone, and if it does not, the folder it names depends on where the
- # process is standing and the move has to be refused instead.
+ # One pass does not settle it, so writing the result back would have the reader expand twice;
+ # if one pass does not name a folder on its own, the move has to be refused.
once = expandvars(value)
if _is_fully_qualified(once, pathmod):
return None
@@ -619,15 +577,14 @@ def _anchor(
if not value:
return None
if _is_fully_qualified(value, pathmod):
- # Already names one folder, but still worth writing back if expanding is
- # what made it name one: the reader that does not expand cannot see that.
+ # Already names one folder, but write back if expanding is what made it name one: the reader
+ # that does not expand cannot see that.
return value if value != original else None
if not _names_a_path(name, value):
return None
if pathmod.splitdrive(value)[0] or value.startswith(("\\", "/")):
- # "D:cache" is drive D's own current directory and "\cache" the root of
- # the current drive, neither of which join() knows, so ask the OS. A
- # failure reaches the caller, which then declines to move at all.
+ # Ask the OS: "D:cache" is drive D's current directory and "\cache" the current drive's root,
+ # neither of which join() knows.
return (abspath or pathmod.abspath)(value)
return pathmod.join(cwd, value)
@@ -647,16 +604,15 @@ def relocation_target(
return None
if home_isdir is None:
home_isdir = pathmod.isdir
- # A profile that has not mounted yet still has a writable parent, so makedirs
- # would build an empty second one that shadows the real one when it arrives.
+ # A profile that has not mounted yet still has a writable parent, so makedirs would build an
+ # empty second one that shadows the real one.
if not home_isdir(home):
return None
work_dir = pathmod.join(home, WORK_DIR_NAME)
try:
makedirs(work_dir, exist_ok = True)
except OSError:
- # An unwritable home is a broken profile and Unsloth must write there
- # anyway, so stop now rather than failing later.
+ # An unwritable home is a broken profile and Unsloth must write there anyway, so stop now.
return None
return work_dir
@@ -671,13 +627,11 @@ def blocked_message(
expanduser = None,
):
"""The error shown to someone who ran Unsloth from a system folder by hand."""
- # allow_public here only: a person can sensibly `cd C:\Users\Public`, but
- # relocating there would share one account's state with every other account.
+ # allow_public here only: relocating to C:\Users\Public would share one account's state with every other account.
home = safe_user_dir(environ, windir, pathmod, sep, expanduser, allow_public = True)
if home:
- # Quote it, or C:\Users\Jane Doe reaches Set-Location as two arguments.
- # PowerShell single quotes are verbatim ('' escapes an apostrophe); cmd
- # needs double quotes once extensions are off.
+ # Quote it, or C:\Users\Jane Doe reaches Set-Location as two arguments. PowerShell single
+ # quotes are verbatim; cmd needs double quotes once extensions are off.
home_ps = "'" + home.replace("'", "''") + "'"
home_cmd = '"' + home + '"'
cd_lines = (
@@ -732,8 +686,7 @@ def check_working_directory(
try:
cwd = getcwd()
except OSError:
- # The launch directory is gone: say so rather than name a folder they
- # were never in.
+ # The launch directory is gone: say so rather than name a folder they were never in.
return (
(
"Unsloth cannot determine its current folder. It may have been deleted,\n"
@@ -748,36 +701,31 @@ def check_working_directory(
return None, None, False
if not relocate or not is_relocatable_invocation(argv, environ):
- # `relocate = False` is the imported-as-a-library case: the command
- # modules are already loaded, so a move now is too late for the roots they
- # resolved at import time.
+ # `relocate = False` is the imported-as-a-library case: command modules already resolved their
+ # roots at import time.
return blocked_message(cwd, argv, environ, windirs, pathmod, sep, expanduser), "red", True
target = relocation_target(environ, windirs, pathmod, sep, expanduser, makedirs, home_isdir)
unpinnable = None
- # The caller's own process state when the guard runs inside a host, so nothing
- # stays rewritten unless the move actually happens.
+ # The caller's own process state when the guard runs inside a host, so nothing stays rewritten
+ # unless the move happens.
environ_before = dict(environ)
if syspath is None:
- # Resolved here rather than inside the pinning, or the console script,
- # which passes nothing, would rewrite the real sys.path with no snapshot
- # to put back when the move then fails.
+ # Resolved here rather than inside the pinning, or the console script (which passes nothing)
+ # would rewrite the real sys.path with no snapshot to restore.
import sys as _sys
syspath = _sys.path
syspath_before = list(syspath)
if target is not None:
try:
- # Before moving, or a relative override would end up naming a folder
- # under the new directory instead of theirs.
+ # Before moving, or a relative override would name a folder under the new directory instead of theirs.
pin_relative_overrides(environ, cwd, pathmod, abspath, expandvars, expanduser)
- # This interpreter read PYTHONPATH before the guard ran and keeps the
- # entries as written, resolving a relative one on every import, so it
- # needs anchoring here as well as in the environment.
+ # This interpreter read PYTHONPATH before the guard ran and resolves relative entries on every
+ # import, so it needs anchoring here too.
pin_relative_sys_path(cwd, pathmod, syspath, abspath, exists, expanduser)
except Exception as error:
- # An environment we cannot pin is one we must not move underneath.
- # Both reasons are real: a drive with no current directory, and a list
- # that no longer fits in a Windows variable once fully qualified.
+ # An environment we cannot pin is one we must not move underneath: a drive with no current
+ # directory, or a list that no longer fits in a Windows variable.
unpinnable = error
target = None
moved = False
@@ -795,9 +743,8 @@ def check_working_directory(
except OSError:
target = None
if target is None:
- # It landed somewhere the CLI still refuses, so go back: the
- # values written for the move only mean the same folder from the
- # directory they were written in.
+ # It landed somewhere the CLI still refuses, so go back: the values written for the move only
+ # mean the same folder from where they were written.
try:
chdir(cwd)
except OSError:
@@ -812,8 +759,8 @@ def check_working_directory(
if syspath_before != syspath:
syspath[:] = syspath_before
if unpinnable is not None:
- # Named separately from the profile case below: blaming the user folder
- # for an unpinnable override sends them looking in the wrong place.
+ # Named separately from the profile case below: blaming the user folder for an unpinnable
+ # override sends them looking in the wrong place.
return (
(
f"Unsloth cannot run from {cwd}, and could not move out of it\n"
@@ -825,8 +772,8 @@ def check_working_directory(
True,
)
if target is None:
- # Fail closed: nowhere usable outside the Windows tree. This text lands in
- # the desktop's logs, so it describes that case, not a shell.
+ # Fail closed: nowhere usable outside the Windows tree. This text lands in the desktop's logs,
+ # so it describes that case, not a shell.
return (
(
f"Unsloth cannot run from {cwd}, and no folder outside {windir} was\n"
diff --git a/unsloth_cli/claude_subagent_mcp.py b/unsloth_cli/claude_subagent_mcp.py
index 5b33429b0f9..6fc92bb8af1 100644
--- a/unsloth_cli/claude_subagent_mcp.py
+++ b/unsloth_cli/claude_subagent_mcp.py
@@ -34,9 +34,7 @@ from unsloth_cli.commands.start import (
_MAX_RESULT_CHARACTERS = 100_000
_CANCEL_POLL_SECONDS = 0.1
_CANCEL_GRACE_SECONDS = 2.0
-# A local server that accepts the connection and then never answers leaves the
-# child, and the parent waiting on it, blocked forever. Generous enough not to cut
-# a long legitimate run short; 0 restores the unbounded wait.
+# A server that accepts and never answers would block the child and the parent forever; 0 restores the unbounded wait.
_DEFAULT_TIMEOUT_SECONDS = 1800.0
@@ -87,7 +85,6 @@ def _stop_child(process: subprocess.Popen) -> None:
"""Stop the Claude child and any tool processes it started."""
if process.poll() is not None:
if os.name != "nt":
- # Leader exited, but its tool processes may still be running.
try:
os.killpg(process.pid, signal.SIGTERM)
except OSError:
@@ -129,7 +126,6 @@ def _stop_child(process: subprocess.Popen) -> None:
process.wait()
else:
if os.name != "nt":
- # Leader is gone; kill any surviving group members.
try:
os.killpg(process.pid, signal.SIGKILL)
except OSError:
@@ -176,11 +172,8 @@ def run_local_agent(
"--output-format",
"json",
"--no-session-persistence",
- # Strip human-blocking tools so the child runs unattended. Only the read-only
- # child's writers bite today, since a --print child is never offered the plan
- # or prompt tools; those are listed anyway so a version that starts offering
- # them cannot stall the subagent. Bash is denied read-only side because plan
- # mode gates it through the same local model, which is not a write barrier.
+ # Strip human-blocking tools so the child runs unattended; plan/prompt tools are listed
+ # pre-emptively, and Bash is denied read-only side because plan mode is not a write barrier.
"--disallowedTools",
(
"AskUserQuestion,EnterPlanMode,Edit,Write,NotebookEdit,Bash"
@@ -215,8 +208,7 @@ def run_local_agent(
popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
else:
popen_kwargs["start_new_session"] = True
- # Same CR/LF hazard as the Codex bridge: --append-system-prompt and the task
- # both span lines, so resolve npm shims rather than spawning the .cmd raw.
+ # Same CR/LF hazard as the Codex bridge: resolve npm shims rather than spawning the .cmd raw.
process = subprocess.Popen(
_resolved_launch_command(executable, command[1:], child_env),
**popen_kwargs,
@@ -366,8 +358,8 @@ def serve(
cancel_event.set()
def handle_shutdown(_signum: int, _frame: Any) -> None:
- # Claude Code sends SIGINT (possibly repeatedly) to cancel a tool call. Only
- # the first unwinds stdin; later ones must not interrupt process-tree cleanup.
+ # Claude Code may send SIGINT repeatedly; only the first unwinds stdin, later ones must not
+ # interrupt process-tree cleanup.
first_signal = not shutdown_started.is_set()
shutdown_started.set()
cancel_active()
diff --git a/unsloth_cli/codex_subagent_mcp.py b/unsloth_cli/codex_subagent_mcp.py
index 16d5d6d8860..6186c14fe5e 100644
--- a/unsloth_cli/codex_subagent_mcp.py
+++ b/unsloth_cli/codex_subagent_mcp.py
@@ -136,8 +136,8 @@ def run_local_agent(task: str, cancel_event: threading.Event | None = None) -> s
popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
else:
popen_kwargs["start_new_session"] = True
- # cmd.exe splits CR/LF inside `%*` and the prompt always spans lines, so
- # resolve npm shims here rather than spawning the .cmd raw.
+ # cmd.exe splits CR/LF inside `%*` and the prompt always spans lines, so resolve npm shims rather
+ # than spawning the .cmd raw.
launch_command = _resolved_launch_command(executable, command[1:], child_env)
process = subprocess.Popen(launch_command, **popen_kwargs)
try:
diff --git a/unsloth_cli/commands/_password_prompt.py b/unsloth_cli/commands/_password_prompt.py
index 55f50acbf1f..7ec0e042eef 100644
--- a/unsloth_cli/commands/_password_prompt.py
+++ b/unsloth_cli/commands/_password_prompt.py
@@ -17,12 +17,11 @@ import os
import sys
from typing import Callable, TextIO
-# Keep in sync with studio/backend/models/auth.py ChangePasswordRequest
-# (new_password min_length) and studio/backend/auth/storage.py.
+# Keep in sync with studio/backend/models/auth.py ChangePasswordRequest and auth/storage.py.
+# The bound is `new_password` min_length.
MIN_PASSWORD_LENGTH = 8
-# Env var that supplies the initial admin password non-interactively (mirror in
-# studio/backend/auth/terminal_prompt.py). Keep the name in sync.
+# Mirror of studio/backend/auth/terminal_prompt.py; keep the name in sync.
SUPPLIED_PASSWORD_ENV = "UNSLOTH_STUDIO_PASSWORD"
_BACKSPACE_CHARS = ("\x7f", "\x08")
@@ -57,7 +56,7 @@ class _RestoreTtyOnSignals:
continue
try:
self._previous.append((sig, signal.signal(sig, _restore_and_reraise)))
- except (ValueError, OSError): # non-main thread / unsupported
+ except (ValueError, OSError):
pass
return self
@@ -82,32 +81,28 @@ def _read_masked_posix(prompt: str, out: TextIO) -> str:
chars: list[str] = []
try:
with _RestoreTtyOnSignals(fd, old_attrs):
- # cbreak + ISIG off (mirrors terminal_prompt.py): with ISIG on,
- # Ctrl-Z would suspend mid-read and leave the shell no-echo before
- # the finally restores it. Ctrl-C/Ctrl-Z arrive as \x03/\x1a here.
+ # cbreak + ISIG off (mirrors terminal_prompt.py): with ISIG on, Ctrl-Z suspends mid-read and leaves
+ # the shell no-echo.
tty.setcbreak(fd)
new_attrs = termios.tcgetattr(fd)
new_attrs[3] &= ~termios.ISIG
termios.tcsetattr(fd, termios.TCSADRAIN, new_attrs)
- # Decode byte-at-a-time with errors="replace" (mirrors
- # terminal_prompt.py): text-mode read(1) can raise UnicodeDecodeError
- # on a pasted non-UTF-8 password or yield a lone surrogate that later
- # crashes pbkdf2. os.read + incremental decoder maps bad bytes to
- # U+FFFD and continues.
+ # os.read + incremental decoder with errors="replace": text-mode read(1) can raise or yield a lone
+ # surrogate that later crashes pbkdf2.
+ # It raises UnicodeDecodeError.
decoder = codecs.getincrementaldecoder(sys.stdin.encoding or "utf-8")("replace")
submitted = False
while not submitted:
raw = os.read(fd, 1)
if not raw: # stream ended mid-line: abort, don't submit
raise EOFError
- # One byte can complete >1 char, so iterate over the decoder's output.
for ch in decoder.decode(raw):
if ch in _SUBMIT_CHARS:
submitted = True
break
if ch == "\x03": # Ctrl-C (ISIG off: surfaces as a char)
raise KeyboardInterrupt
- if ch in ("\x04", "\x1a"): # Ctrl-D / Ctrl-Z
+ if ch in ("\x04", "\x1a"):
if not chars:
raise EOFError
continue
@@ -117,7 +112,7 @@ def _read_masked_posix(prompt: str, out: TextIO) -> str:
out.write("\b \b")
out.flush()
continue
- if ch < " ": # other control characters
+ if ch < " ":
continue
chars.append(ch)
out.write("*")
@@ -142,11 +137,11 @@ def _read_masked_windows(prompt: str, out: TextIO) -> str:
break
if ch == "\x03": # Ctrl-C: getwch swallows the signal, re-raise
raise KeyboardInterrupt
- if ch in ("\x04", "\x1a"): # Ctrl-D / Ctrl-Z
+ if ch in ("\x04", "\x1a"):
if not chars:
raise EOFError
continue
- if ch in ("\x00", "\xe0"): # function/arrow key: swallow the code
+ if ch in ("\x00", "\xe0"):
msvcrt.getwch()
continue
if ch in _BACKSPACE_CHARS:
diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.py
index f9da50c2d36..2f1bc0866dc 100644
--- a/unsloth_cli/commands/chat.py
+++ b/unsloth_cli/commands/chat.py
@@ -31,10 +31,8 @@ _HELP = (
def _you_prompt(colors: bool) -> str:
- # The prompt must go through input(), not a separate print — readline
- # redraws erase anything they didn't draw, eating the label. GNU readline
- # wants colors wrapped in \001/\002; libedit (macOS) prints those
- # literally, so it gets raw ANSI.
+ # Must go through input(): readline redraws erase text they did not draw. GNU readline wants
+ # \001/\002 around colors; libedit (macOS) prints those literally.
try:
import readline
except ImportError:
@@ -66,7 +64,6 @@ def _compare_blocked_reason(model_config) -> Optional[str]:
def _get_base_load_in_4bit(model_config) -> bool:
"""Determine load_in_4bit for base model based on tuned adapter precision."""
if not model_config.is_lora or not model_config.path:
- # Fallback to default if not a LoRA or no path
return True
try:
@@ -86,7 +83,6 @@ def _get_base_load_in_4bit(model_config) -> bool:
elif training_method == "qlora":
return True
elif not training_method:
- # Fallback: check base model name for -bnb-4bit suffix
if model_config.base_model and "-bnb-4bit" not in model_config.base_model.lower():
return False
return True
@@ -96,9 +92,8 @@ def _get_base_load_in_4bit(model_config) -> bool:
def _compare_needs_second_model() -> bool:
- # MLX can't toggle the adapter off, so compare loads the base separately.
- # detect_hardware() would print into the chat (and import torch), so
- # probe its MLX condition quietly: Apple Silicon with mlx installed.
+ # MLX cannot toggle the adapter off, so compare loads the base separately; probe MLX quietly since
+ # detect_hardware() prints into the chat and imports torch.
try:
from studio.backend.utils.hardware import hardware as hw
@@ -303,9 +298,7 @@ def chat(
compare_mode = compare
messages = []
- # Compare's base column: server mode keeps the tuned model remote and
- # loads the base locally; local MLX (no adapter toggle) does the same;
- # local CUDA just toggles the adapter on the one loaded model.
+ # Compare's base column: server mode and local MLX load the base separately; local CUDA just toggles the adapter.
dual_compare = compare_blocked is None and (server_mode or _compare_needs_second_model())
base_backend = None
@@ -328,8 +321,7 @@ def chat(
markup = False,
)
try:
- # Use the same precision as the tuned model for fair comparison
- base_load_opts = dict(load_opts) # Copy original options
+ base_load_opts = dict(load_opts)
base_load_opts["load_in_4bit"] = _get_base_load_in_4bit(model_config)
base_backend = load_chat_backend(base_id, fresh_backend = True, **base_load_opts)
except Exception as exc:
@@ -446,7 +438,6 @@ def chat(
render_columns(
"base", base_text, f"{name} (tuned)", tuned_text, console = console
)
- # History continues as the tuned model; base is just the reference.
answer = tuned_text
else:
if should_print:
diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.py
index 1d44a795b11..08797f08a88 100644
--- a/unsloth_cli/commands/inference.py
+++ b/unsloth_cli/commands/inference.py
@@ -98,9 +98,7 @@ def inference(
)
raise typer.Exit(code = 1)
- # A running Unsloth server keeps the model warm between runs. Under
- # mlx.launch, every rank must enter the local MLX path instead of rank 0
- # alone talking to a server.
+ # Under mlx.launch every rank must enter the local MLX path, not just rank 0 talking to a warm server.
load_opts = dict(
hf_token = hf_token,
max_seq_length = max_seq_length,
diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.py
index 4bff2f28764..4925b30e41a 100644
--- a/unsloth_cli/commands/train.py
+++ b/unsloth_cli/commands/train.py
@@ -78,8 +78,7 @@ def train(
config_overrides = config_overrides or {}
cfg.apply_overrides(**config_overrides)
- # CLI/env tokens take precedence; guard against unresolved typer.Option
- # (decorator interaction)
+ # CLI/env tokens take precedence; guard against unresolved typer.Option.
from typer.models import OptionInfo
if isinstance(hf_token, OptionInfo):
@@ -105,7 +104,6 @@ def train(
typer.echo("Error: provide --dataset or --local-dataset (or via --config)", err = True)
raise typer.Exit(code = 2)
- # A LoRA adapter dir has adapter_config.json
model_path = Path(cfg.model) if cfg.model else None
model_is_lora = (
model_path and model_path.is_dir() and (model_path / "adapter_config.json").exists()
@@ -122,7 +120,6 @@ def train(
trainer = _create_cli_trainer(cfg.model, hf_token)
- # Load model (trainer.is_vlm is set after this)
if not trainer.load_model(
model_name = cfg.model,
max_seq_length = cfg.training.max_seq_length,
@@ -150,7 +147,7 @@ def train(
ds, eval_ds = result
training_kwargs = cfg.training_kwargs()
- training_kwargs["wandb_token"] = wandb_token # CLI/env takes precedence
+ training_kwargs["wandb_token"] = wandb_token
started = trainer.start_training(dataset = ds, eval_dataset = eval_ds, **training_kwargs)
if not started:
diff --git a/unsloth_cli/options.py b/unsloth_cli/options.py
index 7a0433e87c1..8a6d4abe75d 100644
--- a/unsloth_cli/options.py
+++ b/unsloth_cli/options.py
@@ -65,7 +65,6 @@ def _collect_config_fields(config_class: type[BaseModel]) -> list[tuple[str, Any
for name, field_info in config_class.model_fields.items():
annotation = field_info.annotation
- # Recurse into nested models
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
for nested_name, nested_field in annotation.model_fields.items():
if nested_name in seen_names:
@@ -95,11 +94,9 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:
original_params = list(sig.parameters.values())
original_param_names = {p.name for p in original_params}
- # Build new parameters: config fields first, then original params
new_params = []
for field_name, field_info in fields:
- # Skip fields already defined in function signature (e.g., with envvar)
if field_name in original_param_names:
continue
annotation = field_info.annotation
@@ -107,7 +104,6 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:
help_text = field_info.description or ""
if _is_list_type(annotation):
- # Repeatable option: --flag a --flag b -> ["a", "b"]
default = typer.Option(None, flag_name, help = help_text)
param = inspect.Parameter(
field_name,
@@ -141,7 +137,6 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:
)
new_params.append(param)
- # Add original params, excluding config_overrides (will be injected)
for param in original_params:
if param.name != "config_overrides":
new_params.append(param)
@@ -155,7 +150,6 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:
if key in field_names:
if kwargs[key] is not None:
config_overrides[key] = kwargs[key]
- # Only delete if not an explicitly declared parameter
if key not in original_param_names:
del kwargs[key]
diff --git a/unsloth_cli/pi_subagent.ts b/unsloth_cli/pi_subagent.ts
index 9714bf86eb8..2406e496027 100644
--- a/unsloth_cli/pi_subagent.ts
+++ b/unsloth_cli/pi_subagent.ts
@@ -214,8 +214,8 @@ async function runLocalAgent(
}
if (event.type !== "message_end") return;
const message = event.message;
- // Pi reports model/API failures as message_end events while still
- // exiting 0, so the exit status alone cannot surface them.
+ // Pi reports model/API failures as message_end events while still exiting 0, so the exit status
+ // alone cannot surface them.
if (message?.stopReason === "error" || message?.stopReason === "aborted") {
childError =
(typeof message.errorMessage === "string" && message.errorMessage) ||
diff --git a/unsloth_cli/tests/test_claude_plan_gate.py b/unsloth_cli/tests/test_claude_plan_gate.py
index 85336c8dfe0..eed54c7bc7c 100644
--- a/unsloth_cli/tests/test_claude_plan_gate.py
+++ b/unsloth_cli/tests/test_claude_plan_gate.py
@@ -106,9 +106,8 @@ def test_plugin_still_writes_the_mcp_server_and_skill(tmp_path):
def test_wsl_run_clears_a_gate_left_by_an_earlier_windows_run(tmp_path, monkeypatch):
- # The plugin dir survives across runs when persisted, so a gate written by a
- # Windows run would otherwise be shipped into the distro with an interpreter
- # path it cannot execute.
+ # The plugin dir survives across runs when persisted, so a gate written by a Windows run would
+ # otherwise be shipped into the distro with an interpreter path it cannot execute.
plugin = _plugin(tmp_path)
gate = plugin / "hooks" / "plan_gate.py"
hooks = plugin / "hooks" / "hooks.json"
@@ -123,9 +122,9 @@ def test_wsl_run_clears_a_gate_left_by_an_earlier_windows_run(tmp_path, monkeypa
def test_hook_command_survives_a_missing_gate_and_a_path_with_spaces(tmp_path):
- # Handing the path straight to the interpreter makes a missing gate exit 2,
- # which Claude treats as a blocking error: the editing tool would then be
- # denied in every mode, not just plan. Going through runpy makes it exit 1.
+ # Handing the path straight to the interpreter makes a missing gate exit 2, which Claude treats as
+ # a blocking error: the editing tool would then be denied in every mode, not just plan. Going
+ # through runpy makes it exit 1.
plugin = _plugin(tmp_path / "dir with space")
hook = json.loads((plugin / "hooks" / "hooks.json").read_text())
command = hook["hooks"]["PreToolUse"][0]["hooks"][0]["command"]
@@ -157,9 +156,9 @@ def test_hook_command_survives_a_missing_gate_and_a_path_with_spaces(tmp_path):
@pytest.mark.parametrize("hostile", ["sub$(echo X)", "tick`echo X`", "var$HOME", "pct%TEMP%pct"])
def test_gate_survives_shell_metacharacters_in_its_path(tmp_path, hostile):
- # The hook command is run by a shell. A temp root holding these expands under
- # sh (or cmd, for %VAR%) before Python sees the path, so the gate is not found
- # and exits 1, which fails open and silently drops the routing message.
+ # The hook command is run by a shell. A temp root holding these expands under sh (or cmd, for
+ # %VAR%) before Python sees the path, so the gate is not found and exits 1, which fails open and
+ # silently drops the routing message.
plugin = _plugin(tmp_path / hostile)
command = json.loads((plugin / "hooks" / "hooks.json").read_text())["hooks"]["PreToolUse"][0][
"hooks"
diff --git a/unsloth_cli/tests/test_claude_subagent_mcp.py b/unsloth_cli/tests/test_claude_subagent_mcp.py
index 441fd359e47..3bbb204dc36 100644
--- a/unsloth_cli/tests/test_claude_subagent_mcp.py
+++ b/unsloth_cli/tests/test_claude_subagent_mcp.py
@@ -348,8 +348,8 @@ def test_read_only_local_child_uses_plan_mode(monkeypatch, tmp_path):
assert command[command.index("--permission-mode") + 1] == "plan"
disallowed = command[command.index("--disallowedTools") + 1]
assert disallowed == "AskUserQuestion,EnterPlanMode,Edit,Write,NotebookEdit,Bash"
- # Bash matters: plan mode routes it through a classifier served by this same
- # local model, so without the deny a "read-only" child can still write files.
+ # Bash matters: plan mode routes it through a classifier served by this same local model, so
+ # without the deny a "read-only" child can still write files.
prompt = command[command.index("--append-system-prompt") + 1]
assert "read-only local coding subagent" in prompt
@@ -481,8 +481,8 @@ def test_result_parser_accepts_diagnostics_before_json():
def test_child_is_stopped_when_it_produces_nothing_before_the_deadline(monkeypatch, tmp_path):
- # A local server that accepts and never answers used to block the child, and
- # the parent waiting on it, indefinitely. Measured past 400s before this.
+ # A local server that accepts and never answers used to block the child, and the parent waiting on
+ # it, indefinitely. Measured past 400s before this.
monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT", "0.3")
_stub_env(monkeypatch, tmp_path)
stopped = []
@@ -515,8 +515,8 @@ def test_timeout_can_be_disabled(monkeypatch):
def test_local_child_is_spawned_through_the_shim_resolver(monkeypatch, tmp_path):
- # Pins the wiring: a Windows .cmd must reach the npm parser, not Popen (#9167).
- # The parser behaviour itself is covered in test_start.py.
+ # Pins the wiring: a Windows .cmd must reach the npm parser, not Popen (#9167). The parser
+ # behaviour itself is covered in test_start.py.
_stub_env(monkeypatch, tmp_path)
monkeypatch.setattr(bridge.shutil, "which", lambda _: r"C:\\nodejs\\claude.cmd")
captured = {}
diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.py
index bd962d3842a..7c7b1824df9 100644
--- a/unsloth_cli/tests/test_inference_chat.py
+++ b/unsloth_cli/tests/test_inference_chat.py
@@ -117,8 +117,8 @@ def test_visible_text_holds_unclosed_think():
def test_visible_text_holds_partial_think_prefix():
- # Streams are cumulative, so the opening tag can arrive as "<", "<thi",
- # then "<think>". Hold possible tag prefixes until they are disambiguated.
+ # Streams are cumulative, so the opening tag can arrive as "<", "<thi", then "<think>". Hold
+ # possible tag prefixes until they are disambiguated.
assert visible_text("<", show_thinking = False) == ""
assert visible_text("<thi", show_thinking = False) == ""
assert visible_text("done.<thi", show_thinking = False) == "done."
@@ -290,8 +290,7 @@ def test_you_prompt_matches_readline_backend(monkeypatch):
assert chatmod._you_prompt(colors = True) == "\n\x1b[1;36mYou: \x1b[0m"
assert chatmod._you_prompt(colors = False) == "\nYou: "
- # Windows: no readline module at all; the console's own line editing
- # handles backspace, so plain ANSI color (no markers) is safe.
+ # Windows: no readline module at all; the console's own line editing handles backspace, so plain ANSI color is safe.
monkeypatch.setitem(sys.modules, "readline", None)
assert chatmod._you_prompt(colors = True) == "\n\x1b[1;36mYou: \x1b[0m"
assert chatmod._you_prompt(colors = False) == "\nYou: "
@@ -445,13 +444,13 @@ def test_catalog_cached_entries_filter_non_chat_rows(monkeypatch, tmp_path):
},
# An embedding/CLIP repo carries task None like any chat repo; can_chat separates them.
{"repo_id": "org/Embedder", "task": None, "capabilities": {"can_chat": False}},
- # An untrusted diffusion repo carries no task either, and its pipeline root has no
- # config for can_chat to read, so only its own flag keeps it out of chat.
+ # An untrusted diffusion repo carries no task either, and its pipeline root has no config for
+ # can_chat to read, so only its own flag keeps it out of chat.
{"repo_id": "org/Sdxl", "task": None, "diffusers": True},
]
- # The real variant lister, not a stub: it decides these labels and picks the load target,
- # so a stub tests the plumbing and none of the answer. Pulls neither torch nor fastapi, and
- # syspath_prepend is undone after the test, so the suite keeps its no-server-import property.
+ # The real variant lister, not a stub: it decides these labels and picks the load target, so a stub
+ # tests the plumbing and none of the answer. Pulls neither torch nor fastapi, and syspath_prepend
+ # is undone after the test.
monkeypatch.syspath_prepend(str(_REPO_ROOT / "studio" / "backend"))
monkeypatch.setattr(cat, "_cached_catalog_rows", lambda: (gguf_rows, model_rows))
@@ -860,8 +859,8 @@ def test_find_studio_server_none_when_not_running(monkeypatch):
def test_find_studio_server_prefers_ipv4_loopback_for_localhost(monkeypatch):
- # localhost resolving ::1-first must not hide an Unsloth bound to 127.0.0.1:
- # discovery tries each loopback address and returns the one that answers.
+ # localhost resolving ::1-first must not hide an Unsloth bound to 127.0.0.1: discovery tries each
+ # loopback address and returns the one that answers.
import socket
import urllib.request
@@ -2319,8 +2318,8 @@ def test_catalog_drops_an_export_with_no_loadable_payload(monkeypatch, tmp_path)
(whole / "adapter_model.safetensors").write_bytes(b"\0" * 2048)
monkeypatch.setattr(cat, "_path_can_chat", lambda *a, **k: None)
- # Warm the real hub.* chain BEFORE shadowing utils.models: _local_dir_holds_a_payload
- # imports through it, and a stub package would break that import rather than the test.
+ # Warm the real hub.* chain BEFORE shadowing utils.models: _local_dir_holds_a_payload imports
+ # through it, and a stub package would break that import rather than the test.
assert cat._local_dir_holds_a_payload(whole) is True
assert cat._local_dir_holds_a_payload(broken) is False
@@ -2446,8 +2445,8 @@ QUANT_LAYOUTS = [
([("Q4_K_M/Tiny-Q4_K_M.gguf", 16), ("Q8_0/Tiny-Q8_0.gguf", 16)], "Q4_K_M, Q8_0"),
# A split quant is ONE thing to pick. The glob listed every shard as its own label.
([(f"Tiny-Q4_K_M-0000{n}-of-00003.gguf", 16) for n in (1, 2, 3)], "Q4_K_M"),
- # The case the host decides: fnmatch normcases on Windows and not on Linux or macOS, so
- # "*.gguf" found this file on one platform only and the cache read differently per machine.
+ # The case the host decides: fnmatch normcases on Windows and not on Linux or macOS, so "*.gguf"
+ # found this file on one platform only and the cache read differently per machine.
([("Tiny-Q8_0.GGUF", 16)], "Q8_0"),
([("Tiny-Q4_K_M.gguf", 16), ("mmproj-F16.gguf", 16)], "Q4_K_M"),
]
diff --git a/unsloth_cli/tests/test_installer_download_markers.py b/unsloth_cli/tests/test_installer_download_markers.py
index 904bd98b1e8..b5159ed431f 100644
--- a/unsloth_cli/tests/test_installer_download_markers.py
+++ b/unsloth_cli/tests/test_installer_download_markers.py
@@ -29,8 +29,8 @@ _INSTALL_PS1 = _REPO_ROOT / "install.ps1"
_THRESHOLD = 52428800 # 50 MiB, the shipped default in both installers.
# Shaped like real `uv pip install` output, but the two nvidia sizes are picked to bracket
-# _THRESHOLD rather than to match those wheels (both are in fact far larger), so moving the
-# default in either direction changes what is marked. transformers lands unannounced.
+# _THRESHOLD rather than to match those wheels, so moving the default in either direction changes
+# what is marked. transformers lands unannounced.
_UV_OUTPUT = """Resolved 38 packages in 2.60s
Downloading torch (2.4GiB)
Downloading nvidia-cudnn-cu12 (674.0MiB)
@@ -47,8 +47,8 @@ Installed 38 packages in 9.20s
+ torch==2.10.0+cu126
"""
-# uv does not repeat a completion, but a repeat must not close a download twice: the app
-# would see an end for something it never saw start.
+# uv does not repeat a completion, but a repeat must not close a download twice: the app would see
+# an end for something it never saw start.
_UV_REPEATED_COMPLETION = """Downloading torch (2.4GiB)
Downloaded torch
Downloaded torch
@@ -171,9 +171,9 @@ def test_the_exit_code_survives_the_added_pipe(exit_code):
@pytest.mark.parametrize("verbose", [False, True])
def test_a_host_without_awk_still_installs(verbose):
- # install.sh supports minimal images that ship no awk, and this pipe now carries every
- # install command, so losing awk must cost the markers and nothing else. Without the
- # fallback the pipeline closes and the child dies of SIGPIPE, reporting exit 141.
+ # install.sh supports minimal images that ship no awk, and this pipe now carries every install
+ # command, so losing awk must cost the markers and nothing else. Without the fallback the pipeline
+ # closes and the child dies of SIGPIPE, reporting exit 141.
with tempfile.TemporaryDirectory() as tmp:
stub = Path(tmp) / "bin"
stub.mkdir()
@@ -187,14 +187,14 @@ def test_a_host_without_awk_still_installs(verbose):
assert (ok_rc, rc) == ("0", "42"), "a missing awk turned into SIGPIPE"
assert markers == [] and ok_markers == [], "markers cannot be produced without awk"
assert _UV_OUTPUT.strip() in shown, "the failure path lost the child's output"
- # The sink still has to differ by arm: quiet holds output in the log until something
- # fails, verbose passes it straight through.
+ # The sink still has to differ by arm: quiet holds output in the log until something fails, verbose
+ # passes it straight through.
assert (_UV_OUTPUT.strip() in ok_shown) is verbose
def test_a_marker_arrives_while_its_download_is_still_running():
- # Every other test reads output after the child exits, so a buffered marker still
- # shows up -- just too late. Verbose is the arm with the block-buffering redactor.
+ # Every other test reads output after the child exits, so a buffered marker still shows up, just
+ # too late. Verbose is the arm with the block-buffering redactor.
proc = subprocess.Popen(
["/bin/sh", "-c", _sh_harness('printf "Downloading torch (2.4GiB)\\n"; sleep 30')],
stdout = subprocess.DEVNULL,
@@ -282,8 +282,8 @@ def test_both_installers_emit_the_same_markers(options):
def test_both_installers_ship_the_same_threshold():
- # Execution only brackets the default between two fixture sizes; the literal pins it
- # exactly, and pins the two installers to each other so they cannot drift apart.
+ # Execution only brackets the default between two fixture sizes; the literal pins it exactly, and
+ # pins the two installers to each other so they cannot drift apart.
assert _extract_default() == f': "${{UNSLOTH_DL_MARKER_MIN_BYTES:={_THRESHOLD}}}"'
ps1 = _INSTALL_PS1.read_text(encoding = "utf-8")
assert f"$script:UvDownloadMarkerMinBytes = {_THRESHOLD}" in ps1
diff --git a/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py b/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py
index 931739142bc..b66bb5d7a07 100644
--- a/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py
+++ b/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py
@@ -270,8 +270,8 @@ def test_a_windows_tempfile_failure_skips_instead_of_aborting(monkeypatch, tmp_p
command whose real work is done.
"""
studio = _studio()
- # Captured BEFORE any patching: studio.tempfile is the tempfile module itself, so
- # patching through it replaces the global.
+ # Captured BEFORE any patching: studio.tempfile is the tempfile module itself, so patching through
+ # it replaces the global.
real_mkstemp = tempfile.mkstemp
calls = []
@@ -321,8 +321,8 @@ def test_a_windows_tempfile_failure_skips_instead_of_aborting(monkeypatch, tmp_p
monkeypatch.setattr(studio.os, "fdopen", lambda fd, *a, **k: _BadHandle(fd))
studio._run_fetched_installer_ps1(b"x", ["--shortcuts-only"], ["powershell.exe"], {})
assert not Path(made["path"]).exists(), "the temp script must not be left behind"
- # Asserted directly so the descriptor leak is caught on Linux too, rather than
- # only surfacing as an unlink failure on Windows.
+ # Asserted directly so the descriptor leak is caught on Linux too, rather than only surfacing as an
+ # unlink failure on Windows.
with pytest.raises(OSError):
os.fstat(made["fd"])
@@ -367,8 +367,8 @@ def test_a_local_installer_that_cannot_be_launched_falls_back_to_the_network(mon
checkout.mkdir()
(checkout / "install.sh").write_text("#!/bin/sh\n")
monkeypatch.setenv("STUDIO_LOCAL_REPO", str(checkout))
- # The tests run from a clone, so _PACKAGE_ROOT would otherwise supply a second
- # usable installer and the network would never be the fallback under test.
+ # The tests run from a clone, so _PACKAGE_ROOT would otherwise supply a second usable installer and
+ # the network would never be the fallback under test.
monkeypatch.setattr(studio, "_PACKAGE_ROOT", tmp_path / "no-checkout-here")
monkeypatch.setattr(studio.platform, "system", lambda: "Linux")
diff --git a/unsloth_cli/tests/test_studio_run_parallel_flag.py b/unsloth_cli/tests/test_studio_run_parallel_flag.py
index 558bb6203a2..d6e177b5333 100644
--- a/unsloth_cli/tests/test_studio_run_parallel_flag.py
+++ b/unsloth_cli/tests/test_studio_run_parallel_flag.py
@@ -125,10 +125,8 @@ def test_typer_parallel_aliases_are_subset_of_backend_denylist():
import inspect
import importlib.util
- # Load llama_server_args.py directly so the test doesn't need the
- # backend's full runtime chain (fastapi / structlog / loggers /
- # utils.hardware) installed -- the invariant is just about the
- # _DENYLIST_GROUPS tuple.
+ # Load llama_server_args.py directly so the test does not need the backend's full runtime chain
+ # installed; the invariant is just about the _DENYLIST_GROUPS tuple.
lsa_path = (
Path(__file__).resolve().parents[2]
/ "studio"
@@ -156,14 +154,13 @@ def test_typer_parallel_aliases_are_subset_of_backend_denylist():
)
-# test_in_venv_path_passes_parallel_to_run_server (below) is the runtime
-# equivalent of the retired source-text guard for hardcoded
-# `llama_parallel_slots = 4`.
+# test_in_venv_path_passes_parallel_to_run_server (below) is the runtime equivalent of the retired
+# source-text guard for hardcoded `llama_parallel_slots = 4`.
-# Re-exec arg-builder coverage. run() re-execs into the studio venv
-# (execvp on POSIX, Popen on Windows). Without explicit forwarding the
-# child reverts to typer defaults and silently drops the user's value.
+# Re-exec arg-builder coverage. run() re-execs into the studio venv (execvp on POSIX, Popen on
+# Windows), and without explicit forwarding the child reverts to typer defaults and silently drops
+# the user's value.
class _ExecCaptured(SystemExit):
@@ -685,8 +682,8 @@ def test_reexec_mixed_parallel_with_passthrough(monkeypatch):
"""--parallel + llama-server pass-through flags must all reach the child."""
result, captured = _invoke_run(
monkeypatch,
- # --top-k is now a first-class sampling flag (routed via UNSLOTH_SAMPLING_*), so use
- # --seed / --temp here, which remain genuine llama-server pass-through flags.
+ # --top-k is now a first-class sampling flag (routed via UNSLOTH_SAMPLING_*), so use --seed /
+ # --temp here, which remain genuine llama-server pass-through flags.
_BASE + ["--parallel", "8", "--seed", "42", "--temp", "0.7"],
)
assert len(captured) == 1
@@ -735,9 +732,8 @@ def test_reexec_forwards_load_in_4bit_in_both_directions(monkeypatch, user_flag,
assert other_polarity not in argv, f"unexpected {other_polarity} in child argv; got {argv}"
-# Runtime check: fake sys.prefix into the studio venv to bypass
-# re-exec, then assert run_server receives --parallel as
-# llama_parallel_slots.
+# Runtime check: fake sys.prefix into the studio venv to bypass re-exec, then assert run_server
+# receives --parallel as llama_parallel_slots.
class _RunServerCaptured(SystemExit):
@@ -840,9 +836,8 @@ def test_in_venv_path_passes_parallel_to_run_server(
run_server(llama_parallel_slots=N), not the old hardcoded 4."""
studio_mod = _load_run_command()
- # A real directory, not /fake: the launch gate creates STUDIO_HOME and
- # locks inside it, so an unwritable home now aborts the run before
- # run_server is ever reached and the flag under test goes unchecked.
+ # A real directory, not /fake: the launch gate creates STUDIO_HOME and locks inside it, so an
+ # unwritable home aborts the run before run_server is reached.
fake_venv = tmp_path / "studio" / "venv" / "unsloth_studio"
monkeypatch.setattr(sys, "prefix", str(fake_venv))
# Pin STUDIO_HOME so sys.prefix.startswith() picks the in-venv branch.
@@ -867,9 +862,9 @@ def test_in_venv_path_passes_parallel_to_run_server(
)
fake_backend_run.run_server = fake_run_server
fake_backend_run._resolve_external_ip = lambda: "127.0.0.1"
- # run() loads the backend via _load_run_module() (by file path), which
- # ignores a sys.modules mock with no matching __file__; inject it as the
- # cached run module so the stubbed run_server is used.
+ # run() loads the backend via _load_run_module() (by file path), which ignores a sys.modules mock
+ # with no matching __file__; inject it as the cached run module so the stubbed run_server is
+ # used.
monkeypatch.setattr(studio_mod, "_RUN_MODULE", fake_backend_run)
import typer as _typer
@@ -937,9 +932,8 @@ def test_in_venv_path_passes_api_only_to_run_server(
"""In-venv path must forward --api-only to run_server(api_only=...)."""
studio_mod = _load_run_command()
- # A real directory, not /fake: the launch gate creates STUDIO_HOME and
- # locks inside it, so an unwritable home now aborts the run before
- # run_server is ever reached and the flag under test goes unchecked.
+ # A real directory, not /fake: the launch gate creates STUDIO_HOME and locks inside it, so an
+ # unwritable home aborts the run before run_server is reached.
fake_venv = tmp_path / "studio" / "venv" / "unsloth_studio"
monkeypatch.setattr(sys, "prefix", str(fake_venv))
monkeypatch.setattr(studio_mod, "STUDIO_HOME", fake_venv.parent)
diff --git a/unsloth_cli/tests/test_studio_run_short_alias_clashes.py b/unsloth_cli/tests/test_studio_run_short_alias_clashes.py
index 4234eb75f40..3065040be24 100644
--- a/unsloth_cli/tests/test_studio_run_short_alias_clashes.py
+++ b/unsloth_cli/tests/test_studio_run_short_alias_clashes.py
@@ -126,8 +126,7 @@ def _invoke(monkeypatch, args):
return captured
-# (short_flag, value, llama-server long name). All were silently
-# mis-parsed pre-cleanup.
+# (short_flag, value, llama-server long name). All were silently mis-parsed pre-cleanup.
_PREVIOUSLY_BROKEN = [
("-fa", None, "--flash-attn"),
("-fit", None, "--fit"),
@@ -174,9 +173,9 @@ def test_dash_hf_documented_alias_still_works(monkeypatch):
assert argv[argv.index("--gguf-variant") + 1] == "UD-Q4_K_XL", argv
-# Legacy `-m` / `-hfr` / `-f` were typer aliases pre-PR. The
-# preprocessor promotes EXACT matches back to their typer params and
-# leaves clustered tokens (`-mg`, `-fa`, ...) in the pass-through tail.
+# Legacy `-m` / `-hfr` / `-f` were typer aliases pre-PR. The preprocessor promotes EXACT matches
+# back to their typer params and leaves clustered tokens (`-mg`, `-fa`, ...) in the pass-through
+# tail.
@pytest.mark.parametrize(
@@ -334,8 +333,8 @@ def test_consume_helper_preserves_value_when_no_match():
assert remaining == ["--top-k", "20"]
-# `-p` is typer short for --port, so Click clusters `-np8` as `-n -p 8`
-# (port=8, parallel dropped). The rewrite splits to `-np 8` pre-parse.
+# `-p` is typer short for --port, so Click clusters `-np8` as `-n -p 8` (port=8, parallel dropped).
+# The rewrite splits to `-np 8` pre-parse.
def test_expand_np_rewrites_attached_form(monkeypatch):
@@ -419,8 +418,8 @@ def test_consume_helper_rejects_empty_inline_value():
helper(["-m="], ("-m",), None, "--model")
-# Gate isolation: importing unsloth_cli from a third-party script must
-# leave its sys.argv intact. Pins the narrow basename set.
+# Gate isolation: importing unsloth_cli from a third-party script must leave its sys.argv intact.
+# Pins the narrow basename set.
@pytest.mark.parametrize(
diff --git a/unsloth_cli/tests/test_studio_runtime_gate_powershell.py b/unsloth_cli/tests/test_studio_runtime_gate_powershell.py
index 90c1b69b7ba..d5043efeb5b 100644
--- a/unsloth_cli/tests/test_studio_runtime_gate_powershell.py
+++ b/unsloth_cli/tests/test_studio_runtime_gate_powershell.py
@@ -50,12 +50,11 @@ def test_returns_the_bare_name_as_a_last_resort(monkeypatch, tmp_path):
assert resolve_windows_powershell() == "powershell.exe"
-# ── the callers ────────────────────────────────────────────────────────────────────
-#
-# Resolving in the gate alone does not fix #9440: setup() and update() both run the gate and
-# then hand off to PowerShell again, so every spawn on that path has to use the resolver or the
-# install dies at the next one with the same WinError 2.
+# The callers. Resolving in the gate alone does not fix #9440: setup() and update() both run the
+# gate and then hand off to PowerShell again, so every spawn on that path has to use the resolver
+# or the install dies at the next one with the same WinError 2.
+# ── the callers ────────────────────────────────────────────────────────────────────
_RESOLVED = ntpath.join(r"C:\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe")
diff --git a/unsloth_cli/tests/test_studio_stop.py b/unsloth_cli/tests/test_studio_stop.py
index 74e34d4fa19..72d936e284c 100644
--- a/unsloth_cli/tests/test_studio_stop.py
+++ b/unsloth_cli/tests/test_studio_stop.py
@@ -96,9 +96,8 @@ def test_stop_does_not_leave_the_older_instance_running(monkeypatch, tmp_path):
def test_stop_signals_each_server_once(monkeypatch, tmp_path):
- # A server writes its per-port file AND studio.pid. It stays alive while it
- # shuts down gracefully, so a second SIGTERM would hit the SIG_DFL the first
- # one installs and hard-kill it mid-cleanup.
+ # A server writes its per-port file AND studio.pid. It stays alive while it shuts down gracefully,
+ # so a second SIGTERM would hit the SIG_DFL the first installs and hard-kill it mid-cleanup.
studio_mod = _studio()
monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path)
monkeypatch.setattr(studio_mod, "_PID_FILE", tmp_path / "studio.pid")
@@ -131,8 +130,8 @@ def test_stop_removes_every_stale_file_for_one_pid(monkeypatch, tmp_path):
def test_stop_does_not_signal_a_reused_pid(monkeypatch, tmp_path):
- # Crash leaves a per-port file behind, the OS hands that PID to something
- # else: stop must drop the record, not SIGTERM an unrelated process.
+ # Crash leaves a per-port file behind, the OS hands that PID to something else: stop must drop the
+ # record, not SIGTERM an unrelated process.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): False)
_write_pid(tmp_path, "studio-8901-8550.pid", 8550)
@@ -145,8 +144,8 @@ def test_stop_does_not_signal_a_reused_pid(monkeypatch, tmp_path):
def test_stop_signals_a_live_server_whose_pid_has_a_stale_record(monkeypatch, tmp_path):
- # Crash leaves studio-8888-8550.pid, the OS reuses 8550 for a new server on
- # another port. The stale timestamp must not veto the live one.
+ # Crash leaves studio-8888-8550.pid, the OS reuses 8550 for a new server on another port. The
+ # stale timestamp must not veto the live one.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)
@@ -169,8 +168,8 @@ def test_stop_signals_a_live_server_whose_pid_has_a_stale_record(monkeypatch, tm
def test_a_bare_run_py_command_line_is_not_rejected(monkeypatch):
- # `cd studio/backend && python run.py --port 8901` has no "studio" or "unsloth"
- # in argv. Guessing from the command line deleted its record without stopping it.
+ # `cd studio/backend && python run.py --port 8901` has no "studio" or "unsloth" in argv; guessing
+ # from the command line deleted its record without stopping it.
studio_mod = _studio()
class _FakeProcess:
@@ -189,8 +188,8 @@ def test_a_bare_run_py_command_line_is_not_rejected(monkeypatch):
def test_an_untimed_record_is_trusted(monkeypatch):
- # A legacy `python run.py --port 8901` has no telltale argv, and the in-venv
- # path runs in-process. Guessing from the command line rejected real servers.
+ # A legacy `python run.py --port 8901` has no telltale argv and the in-venv path runs in-process;
+ # guessing from the command line rejected real servers.
studio_mod = _studio()
assert studio_mod._pid_is_studio_server(8550) is True
@@ -198,9 +197,9 @@ def test_an_untimed_record_is_trusted(monkeypatch):
def test_an_unverifiable_record_is_still_stopped(monkeypatch):
- # psutil is not a base CLI dependency, so the CLI meets timestamped records it
- # cannot check. The old `stop` signalled with no checks at all -- skipping one
- # would leave a live server running, the orphan bug this exists to fix.
+ # psutil is not a base CLI dependency, so the CLI meets timestamped records it cannot check. The
+ # old `stop` signalled with no checks at all, and skipping one would leave a live server
+ # running.
studio_mod = _studio()
monkeypatch.setitem(sys.modules, "psutil", None)
@@ -209,8 +208,8 @@ def test_an_unverifiable_record_is_still_stopped(monkeypatch):
def test_stop_signals_a_timestamped_record_without_psutil(monkeypatch, tmp_path):
- # Multiple servers on different ports: only the newest is also in studio.pid,
- # so the earlier ones are timestamp-only and must still be stopped.
+ # Multiple servers on different ports: only the newest is also in studio.pid, so the earlier ones
+ # are timestamp-only and must still be stopped.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)
monkeypatch.setitem(sys.modules, "psutil", None)
@@ -224,11 +223,10 @@ def test_stop_signals_a_timestamped_record_without_psutil(monkeypatch, tmp_path)
def test_the_untimed_legacy_record_does_not_cancel_a_timed_one(monkeypatch):
- # Every current server writes BOTH a timed per-port record and an untimed
- # studio.pid, so letting the untimed half win made this check inert exactly
- # where it matters: after a crash and a PID reuse, `stop` SIGTERMed whatever
- # unrelated process had inherited the PID. An untimed record carries no
- # information, so it must not overrule a start time that says "not ours".
+ # Every current server writes BOTH a timed per-port record and an untimed studio.pid, so letting
+ # the untimed half win made this check inert exactly where it matters: after a crash and a PID
+ # reuse, `stop` SIGTERMed whatever unrelated process inherited the PID. An untimed record carries
+ # no information, so it must not overrule a start time that says 'not ours'.
studio_mod = _studio()
class _FakeProcess:
@@ -248,8 +246,8 @@ def test_the_untimed_legacy_record_does_not_cancel_a_timed_one(monkeypatch):
def test_stop_does_not_signal_a_reused_pid_recorded_in_both_files(monkeypatch, tmp_path):
- # End to end for the case above: a crashed server left studio-8901-8550.pid
- # and studio.pid, and 8550 now belongs to something else entirely.
+ # End to end for the case above: a crashed server left studio-8901-8550.pid and studio.pid, and
+ # 8550 now belongs to something else.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)
@@ -430,8 +428,8 @@ def test_stop_discards_a_corrupt_pid_file(monkeypatch, tmp_path):
def test_stop_keeps_a_record_it_cannot_read(monkeypatch, tmp_path):
- # A root-owned record, or one caught mid-write, still belongs to a live
- # server. Deleting it is `stop` manufacturing the orphan it exists to fix.
+ # A root-owned record, or one caught mid-write, still belongs to a live server. Deleting it is
+ # `stop` manufacturing the orphan it exists to fix.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
path = tmp_path / "studio-8901-8550.pid"
path.write_text("8550", encoding = "utf-8")
@@ -451,8 +449,8 @@ def test_stop_keeps_a_record_it_cannot_read(monkeypatch, tmp_path):
def test_stop_does_not_claim_success_when_the_only_record_is_unreadable(monkeypatch, tmp_path):
- # A server started under sudo leaves a record we cannot read. Printing "no
- # running server" and exiting 0 tells the user the opposite of the truth.
+ # A server started under sudo leaves a record we cannot read. Printing "no running server" and
+ # exiting 0 tells the user the opposite of the truth.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
path = tmp_path / "studio-8901-8550.pid"
path.write_text("8550", encoding = "utf-8")
@@ -476,8 +474,7 @@ def test_stop_does_not_claim_success_when_the_only_record_is_unreadable(monkeypa
def test_stop_reports_failure_when_one_record_is_unreadable_but_another_stops(
monkeypatch, tmp_path
):
- # Stopping the servers we can see is still a partial result, and exiting 0
- # would hide the one we could not.
+ # Stopping the servers we can see is still a partial result, and exiting 0 would hide the one we could not.
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550, 8600})
_write_pid(tmp_path, "studio-8901-8550.pid", 8550)
hidden = tmp_path / "studio-8902-8600.pid"
@@ -519,8 +516,8 @@ def test_stop_reaches_every_server_when_one_record_cannot_be_removed(monkeypatch
def test_a_record_whose_pid_is_not_ascii_digits_is_discarded(monkeypatch, tmp_path):
- # A superscript two passes isdigit() but int() rejects it, so that gate alone
- # let a ValueError escape _read_pid_record and abort the whole command.
+ # A superscript two passes isdigit() but int() rejects it, so that gate alone let a ValueError
+ # escape _read_pid_record and abort the whole command.
studio_mod, _live, _killed = _install(monkeypatch, tmp_path, alive = set())
(tmp_path / "studio-8901-1.pid").write_text("²", encoding = "utf-8")
diff --git a/unsloth_cli/tests/test_studio_update_local_repo.py b/unsloth_cli/tests/test_studio_update_local_repo.py
index cb1dd5929cb..7fa810cccb4 100644
--- a/unsloth_cli/tests/test_studio_update_local_repo.py
+++ b/unsloth_cli/tests/test_studio_update_local_repo.py
@@ -96,8 +96,8 @@ def test_the_derived_root_is_used_when_nothing_is_set(monkeypatch):
def test_a_pypi_update_never_looks_for_a_checkout(monkeypatch, tmp_path):
- # Without --local there is no local repo to find, and a stale
- # STUDIO_LOCAL_REPO must not leak into the setup environment.
+ # Without --local there is no local repo to find, and a stale STUDIO_LOCAL_REPO must not leak into
+ # the setup environment.
site = tmp_path / "site-packages"
site.mkdir()
studio, seen = _neutered(monkeypatch)
@@ -109,9 +109,9 @@ def test_a_pypi_update_never_looks_for_a_checkout(monkeypatch, tmp_path):
def test_a_relative_override_is_absolutised(monkeypatch, tmp_path):
- # setup.sh does `cd "$SCRIPT_DIR"` before install_python_stack.py runs, so
- # a relative path handed straight through resolves against studio/ (which
- # has no pyproject.toml) and hits the exact uv error the guard replaces.
+ # setup.sh does `cd "$SCRIPT_DIR"` before install_python_stack.py runs, so a relative path handed
+ # straight through resolves against studio/ (which has no pyproject.toml) and hits the exact uv
+ # error the guard replaces.
checkout = tmp_path / "unsloth"
checkout.mkdir()
(checkout / "pyproject.toml").write_text("[project]\nname = 'unsloth'\n")
@@ -139,8 +139,8 @@ def test_a_tilde_override_is_expanded(monkeypatch, tmp_path):
def test_a_blank_override_falls_back_to_the_derived_root(monkeypatch):
- # `STUDIO_LOCAL_REPO= ` (install.sh resets it to empty) must not become
- # Path(" ") and fail the guard on a perfectly good checkout.
+ # `STUDIO_LOCAL_REPO= ` (install.sh resets it to empty) must not become Path(" ") and fail the
+ # guard on a perfectly good checkout.
studio, seen = _neutered(monkeypatch)
monkeypatch.setenv("STUDIO_LOCAL_REPO", " ")
result = CliRunner().invoke(studio.studio_app, ["update", "--local"])
@@ -201,9 +201,8 @@ def test_a_pypi_update_passes_no_checkout(monkeypatch):
def test_windows_is_shown_a_powershell_assignment(monkeypatch, tmp_path):
- # `VAR=value command` is POSIX shell syntax. PowerShell parses the
- # assignment as a command name, so the only recovery instruction the guard
- # prints was unusable on the platform the guard exists for.
+ # `VAR=value command` is POSIX shell syntax. PowerShell parses the assignment as a command name, so
+ # the only recovery instruction the guard prints was unusable on the platform it exists for.
import platform as _platform
site = tmp_path / "Lib" / "site-packages"
diff --git a/unsloth_cli/tests/test_studio_update_stage.py b/unsloth_cli/tests/test_studio_update_stage.py
index c1c40b67bab..eb9523589d2 100644
--- a/unsloth_cli/tests/test_studio_update_stage.py
+++ b/unsloth_cli/tests/test_studio_update_stage.py
@@ -221,8 +221,8 @@ def test_stage_refuses_when_the_previous_stage_could_not_be_cleared(monkeypatch,
_make_venv(tmp_path)
stale = _studio_stage.stage_root(tmp_path)
(stale / "leftover").mkdir(parents = True)
- # discard() swallows its errors, so a locked or undeletable tree would otherwise
- # be staged into and shipped as if it were a fresh clone.
+ # discard() swallows its errors, so a locked or undeletable tree would otherwise be staged into and
+ # shipped as if it were a fresh clone.
monkeypatch.setattr(_studio_stage, "discard", lambda root: None)
with pytest.raises(_studio_stage.StageError, match = "could not clear"):
@@ -357,8 +357,8 @@ def test_stage_relocates_the_launcher_the_update_rewrote(monkeypatch, tmp_path):
stage_venv = home / _studio_stage.STAGE_DIR_NAME / _studio_stage.VENV_NAME
def reinstalling_update(root: Path, args: list[str]) -> int:
- # What pip/uv do on every upgrade: rewrite the console script with the
- # interpreter named by absolute path, inside the stage.
+ # What pip/uv do on every upgrade: rewrite the console script with the interpreter named by
+ # absolute path, inside the stage.
(root / _studio_stage.VENV_NAME / "bin" / "unsloth").write_text(
f"#!{root / _studio_stage.VENV_NAME}/bin/python\nprint('cli')\n", encoding = "utf-8"
)
@@ -371,8 +371,8 @@ def test_stage_relocates_the_launcher_the_update_rewrote(monkeypatch, tmp_path):
_studio_stage.stage(home, update_args = [], echo = lambda _: None, run_update = reinstalling_update)
launcher = (stage_venv / "bin" / "unsloth").read_text(encoding = "utf-8")
- # Activation moves the venv out of .update-stage, so an absolute shebang into
- # it would leave the activated backend unable to start at all.
+ # Activation moves the venv out of .update-stage, so an absolute shebang into it would leave the
+ # activated backend unable to start at all.
assert launcher.startswith(_studio_stage.RELOCATABLE_SHEBANG)
assert str(_studio_stage.STAGE_DIR_NAME) not in launcher.splitlines()[1]
diff --git a/unsloth_cli/tests/test_studio_update_verify.py b/unsloth_cli/tests/test_studio_update_verify.py
index 0ff9f4ea0c1..b87e5e147ce 100644
--- a/unsloth_cli/tests/test_studio_update_verify.py
+++ b/unsloth_cli/tests/test_studio_update_verify.py
@@ -76,8 +76,8 @@ def site(tmp_path, monkeypatch):
d = tmp_path / "site-packages"
d.mkdir()
monkeypatch.syspath_prepend(str(d))
- # sys.path alone is not enough: the real environment's distributions would
- # still be discovered and could contribute findings of their own.
+ # sys.path alone is not enough: the real environment's distributions would still be discovered
+ # and could contribute findings of their own.
import importlib.metadata as md
real = md.distributions
@@ -167,28 +167,25 @@ def test_a_deleted_file_is_reported(site):
def test_deletion_is_seen_whatever_Distribution_files_does(site):
- # Distribution.files is not a usable basis for this check, and it is not
- # consistent either: newer CPython filters out entries whose file is gone
- # (so a deletion becomes invisible), older CPython lists them but with a
- # path that does not exist. Which one you get depends on the interpreter,
- # and this project supports >= 3.9, so pinning one behaviour would make the
- # test fail on the other. RECORD is parsed directly instead, which reports
- # the deletion on every version.
+ # Distribution.files is not a usable basis for this check and is not consistent either: newer
+ # CPython filters out entries whose file is gone, older CPython lists them with a path that
+ # does not exist. This project supports >= 3.9, so RECORD is parsed directly instead and the
+ # deletion is reported on every version.
import importlib.metadata as md
_make_dist(site, "gamma", {"gamma/a.py": b"a\n", "gamma/b.py": b"bb\n"})
(site / "gamma" / "b.py").unlink()
stale = [f for f in (md.distribution("gamma").files or []) if str(f) == "gamma/b.py"]
- # Either it was dropped, or it is listed and locate() does not resolve.
- # Both mean files() cannot tell you the file is gone.
+ # Either it was dropped, or it is listed and locate() does not resolve; both mean files()
+ # cannot tell you the file is gone.
assert not stale or not stale[0].locate().exists()
assert any("gamma/b.py is missing" in f for f in _deps().damaged_installed_files())
def test_a_file_larger_than_recorded_is_not_damage(site):
- # Two distributions claiming one path: descript-audio-codec ships a
- # top-level tests/__init__.py that another package overwrites. Flagging that
- # would block updates on a perfectly healthy install.
+ # Two distributions claiming one path: descript-audio-codec ships a top-level
+ # tests/__init__.py that another package overwrites, and flagging that would block updates on
+ # a healthy install.
_make_dist(
site,
"delta",
@@ -199,11 +196,10 @@ def test_a_file_larger_than_recorded_is_not_damage(site):
def test_a_shared_file_shorter_than_recorded_is_not_damage(site):
- # The mirror of the case above. Two distributions claiming one path is a
- # packaging collision, and whichever copy landed is the one on disk, so its
- # size says nothing about either RECORD -- in either direction. Only the
- # larger direction was excluded, so a collision that overwrote with a
- # shorter file was reported as corruption and blocked every update.
+ # The mirror of the case above: whichever copy landed is the one on disk, so its size says
+ # nothing about either RECORD in either direction. Only the larger direction was excluded, so
+ # a collision that overwrote with a shorter file was reported as corruption and blocked every
+ # update.
_make_dist(
site, "iota", {"shared/__init__.py": b"short\n"}, record_sizes = {"shared/__init__.py": 900}
)
@@ -221,10 +217,9 @@ def test_a_singly_owned_short_file_is_still_damage(site):
def test_the_scan_is_limited_to_this_interpreters_site_packages(monkeypatch, tmp_path):
- # distributions() searches every sys.path entry, so a damaged distribution
- # reachable only through an inherited PYTHONPATH failed every update while
- # sitting outside the installation, where neither printed repair command can
- # reach it. Only --no-verify broke the loop.
+ # distributions() searches every sys.path entry, so a damaged distribution reachable only
+ # through an inherited PYTHONPATH failed every update while sitting outside the installation,
+ # where neither printed repair command can reach it. Only --no-verify broke the loop.
external = tmp_path / "elsewhere"
(external / "ext-1.0.dist-info").mkdir(parents = True)
(external / "ext").mkdir()
@@ -248,8 +243,8 @@ def test_the_scan_is_limited_to_this_interpreters_site_packages(monkeypatch, tmp
def test_a_deleted_shared_file_is_still_reported(site):
- # Multiple ownership makes the recorded SIZES ambiguous; it cannot explain
- # the file being gone. Skipping shared paths outright hid real deletions.
+ # Multiple ownership makes the recorded SIZES ambiguous; it cannot explain the file being gone,
+ # and skipping shared paths outright hid real deletions.
_make_dist(site, "mu", {"shared/x.py": b"hello\n"}, record_sizes = {"shared/x.py": 10})
_make_dist(site, "nu", {}, record_sizes = {})
(site / "nu-1.0.dist-info" / "RECORD").write_text(
@@ -262,8 +257,8 @@ def test_a_deleted_shared_file_is_still_reported(site):
def test_a_row_without_a_recorded_size_is_still_checked(site):
- # The size field is optional and real wheels leave it blank. Dropping those
- # rows meant a deleted file was never reported.
+ # The size field is optional and real wheels leave it blank; dropping those rows meant a
+ # deleted file was never reported.
_make_dist(site, "xi", {"xi/__init__.py": b"y\n"})
(site / "xi-1.0.dist-info" / "RECORD").write_text("xi/__init__.py,,\n")
(site / "xi" / "__init__.py").unlink()
@@ -272,8 +267,8 @@ def test_a_row_without_a_recorded_size_is_still_checked(site):
def test_a_directory_standing_in_for_a_module_is_damage(site):
- # An empty directory is commonly 4096 bytes on POSIX, so it sails past the
- # shrinkage test while importing as something other than the recorded module.
+ # An empty directory is commonly 4096 bytes on POSIX, so it sails past the shrinkage test while
+ # importing as something other than the recorded module.
_make_dist(site, "omicron", {}, record_sizes = {})
(site / "omicron-1.0.dist-info" / "RECORD").write_text("omicron/mod.py,sha256=x,10\n")
(site / "omicron" / "mod.py").mkdir(parents = True)
@@ -282,8 +277,8 @@ def test_a_directory_standing_in_for_a_module_is_damage(site):
def test_installer_owned_metadata_is_ignored(site):
- # .dist-info files are rewritten in place and drift from the size recorded
- # inside themselves; two real distributions did exactly that.
+ # .dist-info files are rewritten in place and drift from the size recorded inside themselves;
+ # two real distributions did exactly that.
_make_dist(site, "epsilon", {"epsilon/__init__.py": b"e\n"})
info = site / "epsilon-1.0.dist-info"
(info / "RECORD").write_text(
@@ -308,11 +303,10 @@ def test_findings_are_capped(site):
def test_findings_are_capped_when_the_files_are_deleted(site):
- # Truncation and deletion take different branches, and only truncation was
- # covered. A wiped package -- `rm -rf` on a venv's torch, the shape a user
- # actually hits -- takes the deletion branch, so an uncapped one floods the
- # caller with a line per RECORD entry (~11.8k for torch), each of which the
- # desktop updater turns into its own IPC event.
+ # Truncation and deletion take different branches and only truncation was covered. A wiped
+ # package (`rm -rf` on a venv's torch, the shape users hit) takes the deletion branch, so an
+ # uncapped one floods the caller with a line per RECORD entry, about 11.8k for torch, each of
+ # which the desktop updater turns into its own IPC event.
files = {f"theta/m{i}.py": b"x" * 500 for i in range(40)}
_make_dist(site, "theta", files)
for rel in files:
@@ -416,8 +410,8 @@ def test_a_damaged_tree_exits_nonzero_and_names_the_files(monkeypatch, capsys):
def test_a_foreign_cli_stays_quiet(monkeypatch):
- # A pip-installed CLI can drive an update into a venv it does not live in;
- # its own file list would describe the wrong tree, so it must not accuse.
+ # A pip-installed CLI can drive an update into a venv it does not live in; its own file list
+ # would describe the wrong tree, so it must not accuse.
studio = _studio()
monkeypatch.setattr(studio._studio_deps, "running_outside_managed_venv", lambda *a: True)
@@ -429,11 +423,10 @@ def test_a_foreign_cli_stays_quiet(monkeypatch):
def test_a_system_python_is_not_treated_as_the_managed_venv(monkeypatch, tmp_path):
- # Colab has no Unsloth venv: studio/setup.sh installs the backend into the
- # system Python on purpose. Distro-packaged RECORDs there list files the
- # distro never installed (PEP 627), so running the file check would accuse
- # the distro of damaging Unsloth. Reproduced on Ubuntu system Python, which
- # reports an apt-owned `markdown-it-py: ../scripts/markdown-it is missing`.
+ # Colab has no Unsloth venv: studio/setup.sh installs the backend into the system Python on
+ # purpose, and distro-packaged RECORDs there list files the distro never installed (PEP 627),
+ # so the file check would accuse the distro of damaging Unsloth. Reproduced on Ubuntu system
+ # Python, which reports an apt-owned `markdown-it-py: ../scripts/markdown-it is missing`.
prefix = tmp_path / "usr"
prefix.mkdir()
monkeypatch.setattr(sys, "prefix", str(prefix))
@@ -474,10 +467,9 @@ def test_update_exposes_verify_defaulting_on():
def test_the_verify_help_does_not_promise_an_import_check():
- # The scan compares RECORD entries against the filesystem and imports
- # nothing, so it cannot see same-size corruption or an intact but
- # incompatible package. Saying it checks that the backend still imports
- # promises a stronger guarantee than it delivers.
+ # The scan compares RECORD entries against the filesystem and imports nothing, so it cannot see
+ # same-size corruption or an intact but incompatible package. Saying it checks that the
+ # backend still imports promises more than it delivers.
import inspect
opt = inspect.signature(_studio().update).parameters["verify"].default
@@ -524,9 +516,9 @@ def test_no_verify_skips_the_check(monkeypatch):
def test_a_tauri_update_is_verified_too(monkeypatch):
- # The Tauri path returns before the shortcut refresh, so a check placed
- # after that return would silently not run for desktop-initiated updates,
- # the one flow where the user never sees a terminal.
+ # The Tauri path returns before the shortcut refresh, so a check placed after that return would
+ # silently not run for desktop-initiated updates, the one flow where the user never sees a
+ # terminal.
verified = []
monkeypatch.setenv("UNSLOTH_TAURI_UPDATE", "1")
result = _run_update(monkeypatch, [], verified)
@@ -542,10 +534,9 @@ def test_a_tauri_update_is_verified_too(monkeypatch):
],
)
def test_a_custom_root_is_carried_into_the_reinstall_command(monkeypatch, capsys, system, expected):
- # The CLI shim is a bare symlink and _ensure_studio_env_exported only sets
- # os.environ for this process, so the shell that runs the printed command
- # has no UNSLOTH_STUDIO_HOME. Unqualified, it would build a fresh
- # ~/.unsloth/studio and leave the damaged custom root broken.
+ # The CLI shim is a bare symlink and _ensure_studio_env_exported only sets os.environ for this
+ # process, so the shell running the printed command has no UNSLOTH_STUDIO_HOME; unqualified,
+ # it would build a fresh ~/.unsloth/studio and leave the damaged custom root broken.
import platform as _platform
import typer
@@ -587,9 +578,9 @@ def test_a_root_with_spaces_is_quoted(monkeypatch, capsys):
],
)
def test_a_no_torch_install_keeps_that_mode_in_the_reinstall(monkeypatch, capsys, system, expected):
- # install.sh derives SKIP_TORCH from its flag or UNSLOTH_NO_TORCH only, so
- # following the plain command on a GGUF-only install pulls the whole PyTorch
- # stack -- multiple GB the user deliberately opted out of.
+ # install.sh derives SKIP_TORCH from its flag or UNSLOTH_NO_TORCH only, so following the plain
+ # command on a GGUF-only install pulls the whole PyTorch stack, multiple GB the user opted out
+ # of.
import platform as _platform
import typer
@@ -598,9 +589,9 @@ def test_a_no_torch_install_keeps_that_mode_in_the_reinstall(monkeypatch, capsys
monkeypatch.setattr(
studio._studio_deps, "damaged_installed_files", lambda *a, **k: ["x: y is missing"]
)
- # The stub records what root it was asked for: the manifest and marker live
- # in the venv, and an earlier attempt at this passed STUDIO_HOME, which is
- # one directory too high, so it read None and never fired in production.
+ # The stub records what root it was asked for: the manifest and marker live in the venv, and an
+ # earlier attempt passed STUDIO_HOME, one directory too high, so it read None and never fired
+ # in production.
seen = {}
def _module(*a, **k):
@@ -621,9 +612,9 @@ def test_a_no_torch_install_keeps_that_mode_in_the_reinstall(monkeypatch, capsys
@pytest.mark.parametrize("recorded", [False, None])
def test_an_unrecorded_or_torch_install_does_not_gain_the_flag(monkeypatch, capsys, recorded):
- # recorded_no_torch() returns None when nothing recorded the mode, and its
- # contract is that None is not False. Adding the flag on a guess would leave
- # a torch install without torch, so the flag is added only on an explicit True.
+ # recorded_no_torch() returns None when nothing recorded the mode, and its contract is that
+ # None is not False: adding the flag on a guess would leave a torch install without torch, so
+ # it is added only on an explicit True.
import platform as _platform
import typer
@@ -662,13 +653,11 @@ def test_the_default_root_keeps_the_plain_command(monkeypatch, capsys):
def test_the_message_covers_packages_the_installer_will_not_repair(monkeypatch, capsys):
- # install_python_stack installs the current requirement sets and prunes
- # nothing, and the installer never recreates the venv, so damage in an
- # orphan from an older release survives the reinstall it recommends and
- # would report the same failure forever. The scan is deliberately not
- # scoped to Unsloth's dependency closure: under-including there would let
- # real damage through, which is the failure this whole check exists to
- # catch. So the message has to carry the fallback instead.
+ # install_python_stack installs the current requirement sets and prunes nothing, and the
+ # installer never recreates the venv, so damage in an orphan from an older release survives the
+ # reinstall it recommends and would report the same failure forever. The scan is deliberately
+ # not scoped to Unsloth's dependency closure, since under-including would let real damage
+ # through, so the message carries the fallback instead.
import typer
studio = _studio()
@@ -683,11 +672,11 @@ def test_the_message_covers_packages_the_installer_will_not_repair(monkeypatch,
err = capsys.readouterr().err
assert "still listed after that" in err
assert "--force-reinstall" in err
- # Without --no-deps, pip resolves the damaged package's graph and
- # --force-reinstall can swap the pinned CUDA/ROCm torch build.
+ # Without --no-deps, pip resolves the damaged package's graph and --force-reinstall can swap
+ # the pinned CUDA/ROCm torch build.
assert "--no-deps" in err
- # A bare name would let --force-reinstall upgrade the orphan rather than
- # repair it, which --no-deps does not prevent.
+ # A bare name would let --force-reinstall upgrade the orphan rather than repair it, which
+ # --no-deps does not prevent.
assert "<package>==<installed version>" in err
assert "--no-verify" in err
@@ -700,8 +689,7 @@ def test_the_message_covers_packages_the_installer_will_not_repair(monkeypatch,
],
)
def test_the_repair_command_quotes_the_interpreter(monkeypatch, capsys, system, exe, expected):
- # Custom roots with spaces are supported, so an unquoted sys.executable
- # would split into several shell tokens and the command would not run.
+ # Custom roots with spaces are supported, so an unquoted sys.executable would split into several shell tokens.
import platform as _platform
import typer
@@ -723,10 +711,10 @@ def test_the_repair_command_quotes_the_interpreter(monkeypatch, capsys, system,
def test_a_shared_top_level_test_tree_is_not_damage(site):
- # Reported as `einx: test/conftest.py is missing`. einx and torchao both
- # ship it, and install_python_stack.py force-reinstalls torchao every
- # update, so pip removes the file and the pinned torchao does not ship it.
- # Nothing imports another project's fixtures, and no reinstall repairs it.
+ # Reported as `einx: test/conftest.py is missing`. einx and torchao both ship it and
+ # install_python_stack.py force-reinstalls torchao every update, so pip removes the file and
+ # the pinned torchao does not ship it. Nothing imports another project's fixtures, and no
+ # reinstall repairs it.
_make_dist(site, "einx", {"einx/__init__.py": b"e\n"})
(site / "einx-1.0.dist-info" / "RECORD").write_text(
"einx/__init__.py,sha256=x,2\ntest/conftest.py,sha256=x,20650\n"
@@ -735,9 +723,9 @@ def test_a_shared_top_level_test_tree_is_not_damage(site):
def test_an_installer_rewritten_lockfile_is_not_damage(site):
- # Reported as `package-lock.json is 27225 bytes, expected 28473`.
- # setup.ps1/setup.sh run `npm install` inside the installed tree, and npm
- # dedupes hoisted entries under legacy-peer-deps, shrinking the file.
+ # Reported as `package-lock.json is 27225 bytes, expected 28473`: setup.ps1/setup.sh run `npm
+ # install` inside the installed tree, and npm dedupes hoisted entries under legacy-peer-deps,
+ # shrinking the file.
lock = "studio/backend/core/data_recipe/oxc-validator/package-lock.json"
_make_dist(
site,
@@ -749,8 +737,8 @@ def test_an_installer_rewritten_lockfile_is_not_damage(site):
def test_a_deleted_installer_rewritten_file_is_still_damage(site):
- # Only the SIZE of these drifts, because npm rewrites the lockfile in place.
- # It never deletes it, so a missing one is real damage and must be reported.
+ # Only the SIZE of these drifts, because npm rewrites the lockfile in place. It never deletes
+ # it, so a missing one is real damage and must be reported.
lock = "studio/backend/core/data_recipe/oxc-validator/package-lock.json"
_make_dist(site, "unsloth", {"unsloth/__init__.py": b"u\n", lock: b"L" * 100})
(site / lock).unlink()
@@ -759,8 +747,8 @@ def test_a_deleted_installer_rewritten_file_is_still_damage(site):
def test_a_shared_top_level_scripts_tree_is_not_damage(site):
- # unsloth_zoo ships a top-level scripts/, the same squatted-namespace shape
- # as einx's test/. It has no __init__.py, so nothing imports it.
+ # unsloth_zoo ships a top-level scripts/, the same squatted-namespace shape as einx's test/. It
+ # has no __init__.py, so nothing imports it.
_make_dist(site, "upsilon", {"upsilon/__init__.py": b"u\n"})
(site / "upsilon-1.0.dist-info" / "RECORD").write_text(
"upsilon/__init__.py,sha256=x,2\nscripts/helper.py,sha256=x,99\n"
@@ -769,8 +757,8 @@ def test_a_shared_top_level_scripts_tree_is_not_damage(site):
def test_a_package_owned_tests_subdirectory_is_still_checked(site):
- # Only the shared top-level namespace is exempt; a tests/ tree inside a
- # package is that package's alone, so a deletion there is real.
+ # Only the shared top-level namespace is exempt; a tests/ tree inside a package is that
+ # package's alone, so a deletion there is real.
_make_dist(site, "rho", {"rho/tests/helper.py": b"h\n"})
(site / "rho" / "tests" / "helper.py").unlink()
found = _deps().damaged_installed_files()
@@ -800,8 +788,8 @@ def test_runtime_damage_still_fails_when_ignored_rows_are_present(site):
def test_ignored_rows_do_not_consume_the_finding_budget(site):
- # Filtering happens while RECORD is read, so harmless rows cannot crowd a
- # real one off a capped list. Unfiltered, these 40 fill limit = 3.
+ # Filtering happens while RECORD is read, so harmless rows cannot crowd a real one off a capped
+ # list; unfiltered, these 40 fill limit = 3.
files = {f"test/t{i}.py": b"x" for i in range(40)}
files["tau/__init__.py"] = b"t\n"
_make_dist(site, "tau", files)
@@ -812,11 +800,10 @@ def test_ignored_rows_do_not_consume_the_finding_budget(site):
def test_our_own_shared_top_level_trees_are_exempt_too(site):
- # Reported as `unsloth_zoo: tests/conftest.py is 8107 bytes, expected 11429`
- # on an install that was already current, so the update failed on every
- # retry. 11429 is the size in the 2026.8.5 wheel, which shipped a top-level
- # tests/ that any other wheel using that name overwrites. Squatting the
- # namespace does not make it ours, and nothing imports it at runtime.
+ # Reported as `unsloth_zoo: tests/conftest.py is 8107 bytes, expected 11429` on an install that
+ # was already current, so the update failed on every retry. 11429 is the size in the 2026.8.5
+ # wheel, which shipped a top-level tests/ that any other wheel using that name overwrites;
+ # squatting the namespace does not make it ours, and nothing imports it at runtime.
conftest = "tests/conftest.py"
_make_dist(
site,
@@ -828,9 +815,9 @@ def test_our_own_shared_top_level_trees_are_exempt_too(site):
def test_two_distributions_claiming_one_shared_path(site):
- # The real shape: a third party also ships tests/conftest.py and lands last,
- # so the bytes on disk match its RECORD and are shorter than unsloth_zoo's.
- # Neither the drift nor the deletion can affect startup.
+ # The real shape: a third party also ships tests/conftest.py and lands last, so the bytes on
+ # disk match its RECORD and are shorter than unsloth_zoo's. Neither the drift nor the deletion
+ # can affect startup.
rel = "tests/conftest.py"
_make_dist(site, "unsloth_zoo", {rel: b"u" * 11429})
_make_dist(site, "upsilon", {rel: b"c" * 8107})
@@ -841,8 +828,8 @@ def test_two_distributions_claiming_one_shared_path(site):
def test_our_own_shared_top_level_trees_may_also_vanish(site):
- # Same path, deleted rather than overwritten: the einx shape, but claimed by
- # a distribution of ours. No reinstall repairs it either.
+ # Same path, deleted rather than overwritten: the einx shape, but claimed by a distribution of
+ # ours. No reinstall repairs it either.
_make_dist(site, "unsloth_zoo", {"unsloth_zoo/__init__.py": b"z\n"})
(site / "unsloth_zoo-1.0.dist-info" / "RECORD").write_text(
"unsloth_zoo/__init__.py,sha256=x,2\nscripts/helper.py,sha256=x,99\n"
@@ -851,8 +838,8 @@ def test_our_own_shared_top_level_trees_may_also_vanish(site):
def test_our_own_runtime_trees_are_still_checked(site):
- # The exemption is scoped to the shared roots. Everything Unsloth actually
- # imports lives outside them and must still fail an update when damaged.
+ # The exemption is scoped to the shared roots; everything Unsloth actually imports lives
+ # outside them and must still fail an update when damaged.
_make_dist(
site,
"unsloth_zoo",