Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Trim comments in the studiobench harness and unsloth_cli (#10118)
Daniel Han25c10a30ddd5
diff --git a/tests/studio/studiobench/__main__.py b/tests/studio/studiobench/__main__.pyindex 1c6baa3cbeb..c0935538f70 100644--- a/tests/studio/studiobench/__main__.py+++ b/tests/studio/studiobench/__main__.py@@ -24,33 +24,22 @@ import timefrom pathlib import Pathfrom typing import Optional-# BUMPED BECAUSE THE INSTRUMENTS CHANGED, not because the tool gained features. This is one of the-# fields the comparability key is computed over, and it is the ONLY field that can separate a run-# taken before this change from one taken after it: the corpus is untouched, so the corpus hash,-# the tier and the rungs all match across the change.-#-# They must not match. `reasoning_toggle` now settles on a quiet DOM, so-# `highlight_spans_while_open` reads 74,250 where it used to read 44,075 on the same bundle, and-# `open_ms` now terminates on a settled mount rather than on the `data-state` flip, which makes it-# a different quantity rather than a better-measured one. A key that certified those two runs as-# comparable would be the exact failure this key exists to prevent, committed by the guard itself.-#-# Any future change to what an instrument MEASURES has to bump this for the same reason.+# BUMPED BECAUSE THE INSTRUMENTS CHANGED, not because the tool gained features: corpus, tier+# and rungs all match, so this is the only key field that can separate runs across it.+# Bump whenever what an instrument MEASURES changes.TOOL_VERSION = "0.2.0"TIERS = ("fast", "quick", "standard", "full")TIER_RUNGS = {- # One rung, and it is 100K on purpose. The 10K rung was measured across six PRs and could- # not separate any of them from the null control: at that size the UI work disappears- # underneath the scene's own scripted timings (copy_markdown read 204 ms on all fourteen- # arms). 100K is the smallest rung that carries real load -- jank index 29 against 0.6,- # worst frame 1,855 ms against 214 -- so it is the only rung worth an iteration loop.+ # One rung, 100K on purpose: 10K separated none of six PRs from the null control, while 100K+ # carries real load (jank index 29 against 0.6).+ # At 10K `copy_markdown` read 204 ms on all fourteen arms; the worst frame is 1,855 ms against 214."fast": ["100K"],"quick": ["1K", "10K"],"standard": ["1K", "10K", "100K"],"full": ["1K", "10K", "100K", "500K", "1M"],}-# Wall clock budgets, from the design. The deficit-scheduled pacer is what makes them honest: a-# slow machine gets the same stream duration, so it misses SLOTS rather than overrunning the tier.+# Wall clock budgets from the design. The deficit-scheduled pacer keeps them honest: a slow+# machine gets the same stream duration, so it misses SLOTS rather than overrunning the tier.TIER_BUDGET_S = {"fast": 5 * 60, "quick": 5 * 60, "standard": 20 * 60, "full": 60 * 60}@@ -106,10 +95,8 @@ def doctor(args) -> int:import subprocessimport playwright # noqa: F401- # In a SUBPROCESS. Starting and stopping Playwright's node driver just to read the engine- # paths leaves asyncio tearing down a cancelled task, and it prints a TargetClosedError- # traceback to stderr after the check has already passed. A doctor that prints a- # traceback next to `[ok]` is a doctor nobody believes.+ # In a SUBPROCESS: starting Playwright's node driver just to read engine paths leaves asyncio+ # printing a TargetClosedError traceback next to `[ok]`.probe = ("from playwright.sync_api import sync_playwright\n""with sync_playwright() as pw:\n"@@ -129,15 +116,8 @@ def doctor(args) -> int:if got.returncode != 0:raise RuntimeError((got.stderr or "the engine probe failed").strip().splitlines()[-1])text = got.stdout.strip()- # THE PACKAGE IS NOT THE ENGINE. `pip install playwright` and `playwright install` are two- # steps and the README says so, so the machine with the package and no downloaded binary is- # the ordinary case rather than an exotic one. Say so in the line the reader sees, with the- # command that fixes it, rather than reporting a bare `[ok]` they will act on.- #- # REPORTED, NOT FATAL, and deliberately so. This doctor has to keep running on a machine- # with no engine at all, because that is exactly the machine an external tester points it- # at first; a non-zero exit there would turn the one command that explains the problem- # into another thing that fails without saying why.+ # THE PACKAGE IS NOT THE ENGINE: `pip install playwright` and `playwright install` are two steps.+ # Reported, not fatal -- a machine with no engine is the first one a tester points this at.if not engines_installed(text):return (f"{text}; no engine is downloaded. Run `playwright install webkit` "@@ -300,8 +280,7 @@ def planned_rungs(args) -> list:"""if not args.rungs:return list(TIER_RUNGS[args.tier])- # Imported here, not at module scope, on the same rule the rest of this file follows: `--help`- # has to answer on a machine with nothing installed.+ # Imported here, not at module scope: `--help` has to answer on a machine with nothing installed.from .fixture.corpus import RUNGSrungs = [label.strip().upper() for label in args.rungs.split(",")]@@ -316,19 +295,17 @@ def planned_rungs(args) -> list:return rungs-#: What a cell costs BESIDES its film, measured from the steps `CellRunner._run_inner` runs around-#: it: seeding the thread and reading it back, the navigation and thread wait, the 1.5 s idle-#: calibration, the chars-per-token measurement, the drain and the two censuses. Comfortably over-#: what the small rungs cost and about what 1M costs.+#: What a cell costs BESIDES its film: seeding and read-back, navigation, the 1.5 s idle+#: calibration, chars-per-token, the drain and the two censuses.+# Measured from the steps `CellRunner._run_inner` runs around the film.CELL_OVERHEAD_S = 60#: One optional `--surfaces` sweep per arm, before the cells.SURFACE_SWEEP_S = 120#: The same generosity the tier budget already carries, applied to the planned work.WATCHDOG_MARGIN = 3-#: An absolute ceiling on the MEASUREMENT half of the deadline. The point of a watchdog is that a-#: wedged run cannot outlive it, so scaling with the plan must not scale without limit: `--reps-#: 1000` would otherwise arm a deadline measured in months and the watchdog would guarantee-#: nothing at all. 24 hours is past any plan this tool supports and short of forever.+#: An absolute ceiling on the MEASUREMENT half of the deadline: without it `--reps 1000` arms a+#: deadline measured in months and the watchdog guarantees nothing.+#: 24 hours is past any plan this tool supports and short of forever.WATCHDOG_MAX_MEASUREMENT_S = 24 * 60 * 60@@ -607,15 +584,10 @@ def run(args, ab_ref = None) -> int:"""from .runtime.types import OutDirLock, Paths- # THE ARM NAMES, BEFORE A SINGLE PROCESS EXISTS AND BEFORE A SINGLE DIRECTORY. Nothing below- # this line can be undone without the cleanup `finally`, and that block is a long way down; see- # `_windowed_arms`. The labels are READ OFF THE SPECS rather than spelled a second time, so the- # check and the sides cannot drift into disagreeing about what an arm is called -- and- # `side_specs` is a pure list build, so asking it this early starts nothing.- #- # AHEAD OF THE OUTPUT DIRECTORY as well as ahead of the processes. A mistyped `--windowed-arm`- # must cost nothing at all, and a directory tree with a lock file in it is not nothing. See- # test_studiobench_windowed_arm_names.+ # THE ARM NAMES, BEFORE ANY PROCESS OR DIRECTORY EXISTS -- nothing below here undoes without the+ # far-away cleanup `finally`. Labels are read off the specs so the check and the sides agree.+ # Ahead of the output directory too: a mistyped `--windowed-arm` must not cost a lock file.+ # Pinned by test_studiobench_windowed_arm_names.specs = side_specs(args, ab_ref)arm_labels = [label for label, _ref, _attach, _port, _password in specs]windowed = _windowed_arms(getattr(args, "windowed_arm", ""), arm_labels)@@ -656,19 +628,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lcorpus = Corpus.load()_log(f" corpus_hash {corpus.corpus_hash}")- # READ NOW, BEFORE ANYTHING IS STARTED OR MOVED. The probe's source is not needed until the- # browser is launched, but reading it there means a path typo raises after Unsloth and the pacer- # are up and before the cleanup `finally` that would stop them is entered, so a detached Unsloth- # keeps running and holds its port. The cheapest correct fix is to fail while there is nothing- # to clean up: a missing, unreadable or non-UTF-8 file is a mistake in the invocation, and the- # right moment to say so is the first second of the run.- #- # AHEAD OF `prepare_payload` FOR THE SAME REASON THE ARCHIVE IS AHEAD OF THE RECORDER. Reusing- # an `--out` without `--resume` archives the payload that is already there, so reading the- # probe afterwards meant a path typo exited 2 having run no benchmark and having already taken- # `payload.jsonl` off the path every reader looks at: `--report`, `--assert-liveness` and the- # next `--resume` all open that name. A refusal that the first millisecond of the run could- # have issued must not cost the previous run's payload its standard path.+ # READ NOW, BEFORE ANYTHING IS STARTED OR MOVED: reading the probe at launch time raises after+ # Unsloth is up but before the cleanup `finally`, leaving a detached server on its port.+ # Ahead of `prepare_payload`, or a late refusal takes payload.jsonl off its standard path.extra_init = os.environ.get("SBENCH_EXTRA_INIT_SCRIPT")extra_init_source = ""if extra_init:@@ -681,42 +643,26 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l)return 2- # EVERY A/B ARGUMENT CHECK, AHEAD OF `prepare_payload`, FOR THE REASON STATED DIRECTLY ABOVE.- # These three refuse the INVOCATION -- nothing they read comes from an Unsloth, a browser or a- # payload, only `args` and `specs`, both of which exist before this function starts anything.- # Left below the archive they cost the previous run its payload's standard path: a fresh- # `--ab X --home H --out DIR` over a finished DIR archived `payload.jsonl` and then exited 2- # having run no benchmark, so the next `--resume` found nothing to continue and silently- # re-ran the whole ladder, while `--report` and `--assert-liveness` opened the standard name- # and found no rows. `rollback_session_rows` states the rule: a refusal has to leave the- # payload it refused exactly as it found it. That rule covers `invalidate_stale_reports`- # below for the same reason it covers the archive: it REPLACES `summary.md` and `ab.md`, so- # a refusal reaching it would take the previous run's reports down with its payload.+ # EVERY A/B ARGUMENT CHECK, AHEAD OF `prepare_payload`: these refuse the INVOCATION and read only+ # `args` and `specs`. Below the archive they cost the previous run its payload's path, so the+ # next `--resume` found nothing. See `rollback_session_rows`.if ab_ref:if args.attach and not args.attach_b:_log(" --ab with --attach needs --attach-b URL: the second build has to be somewhere.")return 2- # Here rather than beside the init script it guards, because by then two Unsloth instances have been- # cloned, built and launched to run a validation that cannot say anything. See- # `stream_cost_injection_problem`.+ # Here rather than beside the init script it guards: by then two Unsloth instances have been+ # built and launched to run a validation that cannot say anything.injection_problem = stream_cost_injection_problem(specs, getattr(args, "inject_stream_cost_ms", None))if injection_problem:_log(f" {injection_problem}")return 2- # ONE HOME CANNOT HOLD TWO BUILDS. `install_studio` checks the ref out into a repo- # directory derived from the home and installs from it, so two arms sharing a home share- # one checkout: the second install overwrites the first and BOTH arms then serve whichever- # build was installed last. The A/B compares a build against itself and reports a clean,- # confident "no difference" -- which is the same failure mode as a copy timing that is- # really a sleep, and just as invisible in the payload.- #- # Measured, before this refused: two runs of the same pair, one at 716 ms and 718 ms for- # base and treatment, the other at 2,583 ms and 2,614 ms. Nearly equal WITHIN each run and- # 3.6x apart BETWEEN them, because each run was internally uniform and the two runs were- # serving different builds. Read as a within-run comparison it says the change does- # nothing; the difference was sitting between the runs the whole time.+ # ONE HOME CANNOT HOLD TWO BUILDS: `install_studio` derives the checkout from the home, so two+ # arms sharing a home share one checkout: the second install overwrites the first and both+ # arms then serve whichever build was installed last, so the A/B compares a build with itself.+ # Measured: two runs of the same pair, equal within each and 3.6x apart between them.+ # The pair read 716 ms and 718 ms within one run.if not args.attach and args.home:_log(" --home cannot be used with --ab: both arms would install into that one "@@ -726,23 +672,19 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l)return 2- # BEFORE the first install, the first launch and the first recorded row. See `prepare_payload`:- # a refusal that arrives after two clones and two builds has cost the caller an hour to say- # something it could have said in a millisecond, and an archive that arrives after the Recorder- # has opened the file has already appended this run's header to the payload it was moving.+ # BEFORE the first install, launch and recorded row: a refusal after two builds costs an hour to+ # say what a millisecond could, and a late archive has already appended this run's header.archived = prepare_payload(paths, requested_identity(args, ab_ref, corpus.corpus_hash), resume = bool(args.resume))- # AND THE REPORTS THE ARCHIVE LEFT BEHIND. See `invalidate_stale_reports`: this is the one- # point every reuse of an output directory passes through, whichever of the two ways it- # invalidates what is already sitting there.+ # AND THE REPORTS THE ARCHIVE LEFT BEHIND: the one point every reuse of an output directory+ # passes through. See `invalidate_stale_reports`.invalidate_stale_reports(paths.out, archived = archived, extra_init = extra_init)- # Armed AFTER the sides are known, because what it has to cover depends on them: see- # `watchdog_deadline_s`. Nothing between the `side_specs` call above and here can hang --- # `_windowed_arms` compares two sets, `Corpus.load` reads a generated fixture, and- # `prepare_payload` and `invalidate_stale_reports` each touch a handful of files in `--out`.+ # Armed AFTER the sides are known, because what it covers depends on them. Nothing between the+ # `side_specs` call and here can hang.+ # The budget is `watchdog_deadline_s`.watchdog = browser_mod.install_wall_clock_watchdog(watchdog_deadline_s(args.tier,@@ -759,20 +701,11 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_linstalls = []sides = []- # EVERY SIDE THIS RUN LAUNCHED IS STOPPED WHEN THE SETUP AROUND IT FAILS. The sides are- # acquired one after another, so the base is already SERVING while the treatment clones and- # builds, and the cleanup that stops them both is the `finally` under the cells -- which a- # failure up here never reaches. The Unsloth instances are the resources that outlive this process:- # `launch_studio` detaches the server with `setsid -f`, so an abandoned one keeps its port.- # It is not idle there. Unsloth's own launcher ABORTS rather than binding when it finds one of- # its own servers on the requested port (`studio/backend/run.py`, `_resolve_port` with- # `avoid_own_studio`), so the next attempt's server exits and `wait_for_healthz` takes its 200- # from the STALE process: that run measures the build this one installed while `run_meta`- # records the ref it was asked for.- #- # A RETURN IS A FAILURE HERE TOO. The health check below and the development-build gate leave- # by returning, with both Unsloth instances up, which is the same abandoned server by a quieter route --- # so the guard is a `finally` on the whole of setup rather than an `except` on the install.+ # EVERY SIDE THIS RUN LAUNCHED IS STOPPED WHEN THE SETUP AROUND IT FAILS: the base is already+ # SERVING while the treatment builds, and the cleanup `finally` under the cells is never reached+ # from up here. A stale detached server makes the next run's `wait_for_healthz` take its 200 from+ # the wrong build. A RETURN IS A FAILURE HERE TOO, so this is a `finally` over all of setup.+ # The next server exits on `avoid_own_studio` rather than binding.setup_complete = Falsetry:for label, ref, attach, port, password in specs:@@ -799,10 +732,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l})- # THE BUILD, NOW THAT IT IS KNOWN. `prepare_payload` has already agreed the refs match,- # and a ref is a pointer: see `commit_problems`. This is the first moment the commit- # behind it exists, and it is still before the browser, the pacer and every cell, so a- # resume onto a moved branch costs the install it has already paid and nothing more.+ # THE BUILD, NOW THAT IT IS KNOWN: `prepare_payload` has agreed the refs match, and a ref is a+ # pointer. Still before the browser and every cell, so a moved branch costs only the install.+ # `checkout_ref` is the only place a ref becomes a commit.if args.resume:resolved = resolved_commits(sides)commit_issues: list = []@@ -829,8 +761,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l_log(f" FATAL: {side['base_url']}/healthz did not answer 200")return 2+ # THE GATE, before anything else is measured. Both sides: an A/B where one side is a development+ # build puts the whole 3.2x inflation on one arm and reads as a colossal regression or win.# ── THE GATE. Before anything else is measured. Both sides, because an A/B where one side- # is a development build is worse than no A/B: the 3.2x inflation lands entirely on one arm# and reads as a colossal regression or win. ────────────────────verdict = Nonefor side in sides:@@ -859,15 +792,10 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lfrom .runtime.ab import origin_scopedinit_scripts = []- # ONE PROVIDER PER ORIGIN, because the provider lives in the BACKEND and localStorage is- # per-origin. An ATTACHED null control is `--attach U --attach-b U`, which `is_null_control`- # accepts on purpose: two sides, one Unsloth. Registering per SIDE there registers the pacer- # twice against the same backend, and `lifecycle.register_provider` is idempotent by DISPLAY- # NAME -- so the second registration DELETES the id the first side's seed script had already- # captured, and that script keeps selecting the dead id on every navigation it wins (the- # order init scripts run in is not defined, and `StudioAuth.rotate` re-adds them mid-run).- # A base cell that boots with a deleted provider selected renders "No longer offered" and- # throws `Connection not found` without ever asking for a completion.+ # ONE PROVIDER PER ORIGIN: the provider lives in the backend and localStorage is per-origin, so+ # an attached null control is two sides on one Unsloth. `register_provider` is idempotent by+ # DISPLAY NAME, so registering per side deletes the id the first side's seed script captured.+ # `is_null_control` is what recognises it.registered: dict = {}for index, side in enumerate(sides):side_install = installs[index][0]@@ -878,16 +806,15 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l)_log(f" {side['label']}: authenticated as {side_auth.username}")- # BOTH sides register the SAME pacer, so the bytes on the wire are identical by- # construction rather than by two configurations that are meant to agree.+ # BOTH sides register the SAME pacer, so the bytes on the wire are identical by construction+ # rather than by two configurations meant to agree.side_origin = side["base_url"].rstrip("/")shared = registered.get(side_origin)if shared is None:side_provider = pacer_provider(pacer.base_url, [model_id])- # Registered in the BACKEND, and the id it assigns is what the selection names. See- # lifecycle.register_provider: a provider that exists only in localStorage renders- # in the picker as "No longer offered" and send throws `Connection not found`- # without ever asking for a completion.+ # Registered in the BACKEND, and the id it assigns is what the selection names: a provider+ # existing only in localStorage renders as "No longer offered" and send throws+ # `Connection not found`.register_provider(side["base_url"], side_auth, side_provider)side_checkpoint = external_checkpoint_id(side_provider, model_id)registered[side_origin] = (side_provider, side_checkpoint)@@ -911,25 +838,22 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lauth_now,[provider],extra_local_storage = {- # The SELECTION, without which nothing is ever generated. See- # lifecycle.external_checkpoint_id.+ # The SELECTION, without which nothing is ever generated.+ # See lifecycle.external_checkpoint_id."unsloth_chat_last_external_checkpoint": cp,"unsloth_chat_connections_enabled": "true",},),)- # Origin-gated even in the single-target case, so the one-build and two-build paths are- # the same code and the gate cannot rot while unused.+ # Origin-gated even in the single-target case, so the one-build and two-build paths are the same+ # code and the gate cannot rot while unused.side["seed_script"] = _side_seedinit_scripts.append(_side_seed(side_auth))if getattr(args, "inject_stream_cost_ms", None) and side is not sides[0]:- # VALIDATION, not a measurement mode. Burns a known amount of main-thread time per- # SSE chunk on the TREATMENT side only, so an A/B whose two arms are otherwise the- # same build has a known answer. It is origin-gated like the seed above, because a- # context init script fires on every document and burning on both sides would- # inject the cost into the control as well and read back a recovery of zero.+ # VALIDATION, not a measurement mode: burns known main-thread time per SSE chunk on the TREATMENT+ # side only, so an otherwise-identical A/B has a known answer. Origin-gated, or recovery reads 0.from .instruments.selfcheck import stream_cost_injection_init_scriptinit_scripts.append(origin_scoped(@@ -944,38 +868,16 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lauth = sides[0]["auth"]- # ONE `add_init_script` CALL PER SCENE SCRIPT, which is what this always did.- #- # These were briefly concatenated into a single script, on the reasoning that Playwright- # says "the order of evaluation of multiple scripts installed via- # browserContext.addInitScript() and page.addInitScript() is not defined", and surfaces.js- # reads what dom.js and parity.js put on `window.__sb`. The reasoning is sound and the- # change was a REGRESSION, caught by CI: joining them means the browser parses and- # evaluates the three as one unit, so a throw or a parse error in any one of them stops- # the other two running as well. Separate scripts are separate failure domains, and on the- # CI fixture that difference cost `message_menu` its More button and turned a green job- # red. An ordering guarantee is not worth trading fault isolation for when the ordering- # has been correct in practice for the life of the file.- #- # surfaces.js is loaded unconditionally, even without --surfaces. It defines selectors and- # never runs on its own. Making the page's JS depend on a CLI flag would mean the flag- # changes what is on the page during the FILM as well, and the film's numbers must not- # depend on whether a later phase was asked for.+ # ONE `add_init_script` CALL PER SCENE SCRIPT. Concatenating them was a regression caught by CI:+ # one parse error stops all three. Separate scripts are separate failure domains. surfaces.js+ # loads even without --surfaces, so the film cannot depend on whether a later phase was asked for.init_scripts.append(resources.read_text("scene/dom.js"))init_scripts.append(resources.read_text("scene/parity.js"))init_scripts.append(resources.read_text("scene/surfaces.js"))- # AN EXTERNAL PROBE OR ABLATION ARM, its own script like the three above. One environment- # variable rather than a CLI flag per experiment, and WITH THE VARIABLE UNSET NOTHING IS- # APPENDED: the run is byte-identical to a run of a tree that does not have this hook. That- # property is the point. A potency probe perturbs the page it observes, so the probe run and- # the scored run have to be different runs of one harness, and the only safe way to arrange- # that is for the probe to be absent by default.- #- # BECAUSE THE ORDER IS UNDEFINED, A PROBE MUST BE SELF-CONTAINED. It cannot assume the- # scene scripts have run, so it cannot read `window.__sb` at install time. That is a real- # constraint and it is written down in CONTRIBUTING-perf.md rather than papered over with- # a guarantee this file is not in a position to give.+ # AN EXTERNAL PROBE OR ABLATION ARM, in its own script. One env var rather than a flag per+ # experiment, and WITH IT UNSET NOTHING IS APPENDED, so the probe run and the scored run are+ # different runs. Script order is undefined, so a probe cannot read `window.__sb` at install.if extra_init:init_scripts.extend(_probe_init_scripts(extra_init, extra_init_source))_log(@@ -993,11 +895,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lbundle = browser_mod.launch(args.engine, headless = not args.headed, init_scripts = init_scripts, log = _log)- # THE RETURN PATH for a probe installed by the hook above. Unsloth ships `connect-src- # 'self'`, so a beacon to a collector on another port is blocked by CSP before it leaves- # the page, and the payload schema has no row for a one-off probe. The console is what is- # left. Lines are filtered on a caller-supplied prefix so they can be recovered from the- # run log by exact match, and so a probe cannot drown the log in the app's own traffic.+ # THE RETURN PATH for a probe installed above: Unsloth ships `connect-src 'self'` so a beacon is+ # blocked by CSP, and the schema has no row for a one-off probe. Filtered on a caller prefix.console_prefix = os.environ.get("SBENCH_PAGE_CONSOLE")if console_prefix:bundle.page.on(@@ -1005,11 +904,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_llambda m: _log(f" [page] {m.text}") if m.text.startswith(console_prefix) else None,)if extra_init:- # A probe that throws on load is the same silence as a probe that was never installed,- # and the console filter above cannot show it because a failing probe never gets as far- # as printing its own prefix. Attached only when a probe was asked for, so an ordinary- # run is unchanged. `console.error` from the isolation wrapper arrives here as a- # console message rather than a page error, so both channels are listened to.+ # A probe that throws on load is the same silence as one never installed, and the console filter+ # cannot show it. Both channels are listened to: a wrapper's `console.error` is not a page error.bundle.page.on("pageerror", lambda err: _log(f" [page error] {err}"))bundle.page.on("console",@@ -1023,23 +919,16 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_ltime.sleep(1.0)procs = new_roots(os.getpid(), procs_before)- # AN INIT SCRIPT IS A SNAPSHOT AND THE RUN OUTLIVES IT. The scripts above carry the access- # token this run logged in with, they re-run on EVERY navigation, and there is one- # navigation per cell -- so after `StudioAuth` re-mints a token, the next `page.goto` would- # write the DEAD one back over whatever the SPA had rotated to for itself, which is the- # failure this fixes wearing a different hat. Playwright has no way to replace an init- # script and does not define the order they run in, so the seed script decides for itself- # whether to write: it compares its token's `exp` against the one in storage and defers to- # the later one. See `seed_init_script`. At most one extra script per hour of run.+ # AN INIT SCRIPT IS A SNAPSHOT AND THE RUN OUTLIVES IT: the scripts carry this run's access token+ # and re-run on every navigation, so after a re-mint the next `goto` would write the dead one+ # back. Playwright cannot reorder them, so the seed script defers to the later `exp`.for side in sides:side["auth"].on_rotate = lambda auth_now, side = side: bundle.context.add_init_script(side["seed_script"](auth_now))- # WHAT WAS IN THE FILE BEFORE THIS SESSION COULD WRITE A BYTE. `make_context` opens the- # `Recorder` on the next line, and the only check left after that point -- the ladder ratio,- # which is not known until the corpus has been measured -- has to be able to put the payload- # back the way it found it. See `rollback_session_rows`.+ # WHAT WAS IN THE FILE BEFORE THIS SESSION COULD WRITE A BYTE: `make_context` opens the Recorder+ # next, and the ladder-ratio check has to be able to put the payload back.mark = payload_mark(paths.payload_jsonl)ctx, session = make_context(bundle,@@ -1049,14 +938,13 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lpaths,_log,procs,- # ADOPTED, NOT TAKEN AGAIN. `run()` has held this directory since before the payload- # was archived; the `Recorder` writes its session id into the marker it already holds.+ # ADOPTED, NOT TAKEN AGAIN: `run()` has held this directory since before the payload was+ # archived, so the `Recorder` writes its session id into the marker it already holds.out_lock = out_lock,)rec = ctx.recorder- # The ladder this run PROMISED, resolved before it is announced. Recorded rather than- # left to be re-derived from the tier later: `--report` reads it back to decide which rungs- # a payload owes, and a `--rungs` override the payload did not carry made that answer wrong.+ # The ladder this run PROMISED, recorded rather than re-derived: `--report` reads it back to+ # decide which rungs a payload owes, and a `--rungs` override it did not carry made that wrong.rungs = planned_rungs(args)meta_row = {"row_type": "run_meta",@@ -1064,22 +952,16 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l"tool_version": TOOL_VERSION,"corpus_hash": corpus.corpus_hash,"studio_ref": args.branch if owns_studio else f"attached:{base_url}",- # WHICH COMMIT THAT REF NAMED, so a later `--resume` can tell a continuation from- # a branch that moved. Empty for an attached Unsloth, whose build is not visible- # from here. See `commit_problems`.+ # WHICH COMMIT THAT REF NAMED, so a later `--resume` can tell a continuation from a branch that+ # moved. Empty for an attached Unsloth. See `commit_problems`."studio_commit": sides[0].get("commit") or "","bundle": verdict.as_dict(),"platform": {"system": platform.system(),"machine": platform.machine(),- # WHICH PHYSICAL MACHINE, which `system` and `machine` do not answer between- # them. `platform.machine()` is the ARCHITECTURE -- "the machine type, e.g.- # 'AMD64'" -- so two ordinary Linux x86_64 boxes report `Linux` and `x86_64`- # alike, and every number this tool produces is machine-local: `report/render.py`- # says so in its own words, "machine-local; does not travel between machines".- # `platform.node()` is the computer's network name, which is the only thing- # recorded here that differs between two such hosts. Empty when it cannot be- # determined, exactly as the other two are.+ # WHICH PHYSICAL MACHINE, which `system` and `machine` do not answer: `platform.machine()` is the+ # ARCHITECTURE, so two Linux x86_64 boxes report identically. `platform.node()` is the only+ # recorded thing that differs; empty when undeterminable."node": platform.node(),"python": sys.version.split()[0],"engine": bundle.engine,@@ -1092,60 +974,35 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l"tier_rungs": TIER_RUNGS[args.tier],"reps": args.reps,"instrument_level": args.instrument_level,- # In the payload, not only in the log. Two runs with different fixtures are- # not comparable, and a fixture difference that is not recorded is one a later- # reader has no way to notice before quoting a ratio across it.+ # In the payload, not only in the log: two runs with different fixtures are not comparable, and+ # an unrecorded difference is one a later reader cannot notice before quoting a ratio across it."stream_tail_chars": args.stream_tail_chars,"corpus_dollars": bool(args.corpus_dollars),- # WHICH PROBE, IF ANY, WAS IN THE PAGE. Recorded next to the corpus hash and for- # the same reason: a payload nobody can audit against the page it measured has to- # be taken on trust. `null` is the normal case and the only scorable one.+ # WHICH PROBE, IF ANY, WAS IN THE PAGE: a payload nobody can audit against the page it measured+ # has to be taken on trust. `null` is the normal case and the only scorable one."probe_init_script": extra_init or None,- # WHETHER THE PROBE RAN BEFORE THE FILM, for the same reason and with the same- # consequence: it changes what the cell measures without moving the cell id, so- # `--resume` needs it on the record to be able to refuse a toggle. See- # `IDENTITY_AXES`.+ # WHETHER THE PROBE RAN BEFORE THE FILM: it changes what the cell measures without moving the+ # cell id, so `--resume` needs it on the record to refuse a toggle."click_probe": bool(getattr(args, "click_probe", False)),- # AN ARM RUNNING THIS IS NOT A MEASUREMENT OF THE BUILD, and the payload has to- # say so itself. A reader who finds a treatment arm 40% slower has no other way- # to discover that the harness put the 40% there on purpose, and `--resume` reads- # it back as an identity axis so a calibration cannot be continued as an ordinary- # run or the other way round. See `IDENTITY_AXES`.+ # AN ARM RUNNING THIS IS NOT A MEASUREMENT OF THE BUILD, and the payload has to say so: nothing+ # else tells a reader the harness put the 40% there. `--resume` reads it as an identity axis."inject_stream_cost_ms": getattr(args, "inject_stream_cost_ms", None),- # WHICH BROWSER ACTUALLY RENDERED THIS, which the engine name does not settle.- #- # `--headed` flips `headless` at launch, and since Playwright 1.57 the two modes- # default to DIFFERENT BINARIES -- headed resolves to `chrome`, headless to- # `chrome-headless-shell`. This workflow pins `playwright>=1.45,<2`, so that split is- # in range. It is not a mode flag with a mode-shaped effect: one report measures- # headless-shell at two to three times the processing time of chrome-stable, headless- # falls back to software rendering for GPU-accelerated work, and the compositor still- # ticks at 60 Hz in headless unless BeginFrameControl is driven by hand.- #- # This tool measures frames, jank and time-to-settle. A payload recorded headed and one- # recorded headless are two different renderers, and the payload has to say which it- # was before anything compares them.+ # WHICH BROWSER ACTUALLY RENDERED THIS, which the engine name does not settle: since Playwright+ # 1.57 headed resolves to `chrome` and headless to `chrome-headless-shell`, which is measured at+ # two to three times chrome-stable and falls back to software rendering. This tool measures+ # frames and jank, so the payload must name the renderer.+ # This workflow pins `playwright>=1.45,<2`, so the split is inside the pinned range."headed": bool(getattr(args, "headed", False)),}rec.emit(meta_row)from .scoring import payload_rules- # THE COMPARABILITY KEY, emitted as its own row so a prose comparison can be checked later.- #- # `floor_table.load` already refuses to POOL payloads across tiers and corpora, and that- # guard works. What it cannot reach is a comparison made in PROSE between two separately- # published runs, which is how a wrong number nearly propagated here twice.- #- # Computed over the run_meta row that was actually emitted, not over a second dict built- # to look like it. A hand-copied duplicate is one more value whose provenance is not- # stable, which is the whole failure this module exists to stop: the key has to describe- # the payload, not a parallel description of it that can drift.- #- # Keyed on the COMPUTED corpus hash rather than on the harness commit, because a commit is- # a claim about provenance and the hash is the thing itself: one tree in this campaign ran- # a corpus that matched neither the branch's pre-fix nor its post-fix hash, silently and- # self-consistently, and only printing the hash revealed it.+ # THE COMPARABILITY KEY, its own row so a prose comparison can be checked later. `floor_table`+ # refuses to pool across tiers and corpora but cannot reach a comparison made in PROSE, which+ # nearly propagated a wrong number twice. Keyed on the COMPUTED corpus hash: one tree ran a+ # corpus matching neither the pre-fix nor the post-fix hash, and only printing it revealed that.+ # `floor_table.load` is the refusal.rec.emit({"row_type": "comparability",@@ -1163,12 +1020,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lrec.gate("production_build", verdict.production, verdict.as_dict())if args.tier == "fast":- # Said in the log AND recorded in the payload. A fast-tier reading is a DIRECTION, not- # a number: it runs one rung, a 47 s film and however few repetitions the caller asked- # for, so its detection floor is wider than the standard tier's and it has no null- # control of its own unless one is run alongside. The gate exists so the analysis layer- # can refuse to pool a fast payload with a standard one -- a fast reading quoted against- # a standard floor is the single most likely way this tier gets somebody a wrong answer.+ # Said in the log AND recorded. A fast-tier reading is a DIRECTION, not a number: one rung, a 47 s+ # film, a wider detection floor and no null control, so the analysis layer must refuse to pool it._log("")_log(" FAST TIER: for iteration while you are changing something, not for reporting.")_log(@@ -1190,12 +1043,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lelse "standard measurement protocol",},)- # THE SAME SHAPE OF GATE, for the same reason. A probe forces layout on a schedule of its- # own, so its payload is perturbed, and "the docs say probe payloads are never scored" is a- # convention rather than a gate. A convention is what this subsystem keeps being wrong- # about: the run still records ordinary cells and still renders an A/B table, so a probe- # invocation on the standard tier produces something that reads exactly like a result. The- # gate is what lets `floor_table` refuse it instead of trusting the caller to remember.+ # THE SAME SHAPE OF GATE: a probe forces layout on its own schedule, so its payload is perturbed,+ # and "probe payloads are never scored" is a convention rather than a gate. The run still renders+ # an A/B table, so a probe invocation reads like a result unless `floor_table` can refuse it.rec.gate("probe_free",not extra_init,@@ -1215,16 +1065,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l_log("")image_path = ensure_probe_image(paths)- # WHICH SIDE, IF ANY, IS ALLOWED THE WINDOWED READINESS GATE.- #- # Per side and never global, so an A/B of the shipped build against a virtualised one runs- # the base arm on the strict full-mount gate it has always had. A flag that relaxed both- # sides would let a base arm that failed to finish mounting sail through and be compared- # against a treatment that did the same, which is the exact "flattering garbage" the gate- # exists to refuse -- on both arms at once, so the ratio would still look fine.- #- # `windowed` was parsed and validated at the top of this function, before the installs, the- # pacer and the browser existed. Only the application of it is left here.+ # WHICH SIDE, IF ANY, IS ALLOWED THE WINDOWED READINESS GATE. Per side and never global: relaxing+ # both would compare two half-mounted arms with the ratio still looking fine. Parsed earlier;+ # only its application is left here.for side in sides:side["readiness_mode"] = MODE_WINDOWED if side["label"] in windowed else MODE_FULLif side["readiness_mode"] == MODE_WINDOWED:@@ -1274,9 +1117,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lif args.surfaces:_sweep_surfaces(sides, ctx, paths)- # The ladder is sized by the ratio MEASURED on this corpus, not by the provisional 4.0.- # Measured here, before any cell is built, because a rung named in tokens is planned in- # characters and the ratio is what makes those the same claim.+ # The ladder is sized by the ratio MEASURED on this corpus, not the provisional 4.0, and+ # measured before any cell is built because a rung named in tokens is planned in characters.cells = build_cells(rungs,corpus,@@ -1292,9 +1134,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lcorpus_dollars = args.corpus_dollars,)if args.stream_tail_chars or args.corpus_dollars:- # Loud, because both change the fixture. A payload produced under either of them is- # not comparable with one produced without, and the pair that says so is printed here- # and written into the run manifest above.+ # Loud, because both change the fixture: a payload produced under either is not comparable with+ # one produced without._log(f" FIXTURE CHANGED: stream tail {args.stream_tail_chars or 'default'}, "f"dollars {'on' if args.corpus_dollars else 'off'}. Compare only against a run "@@ -1304,10 +1145,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lladder_ratio = cells[0][0].meta["ladder_chars_per_token"]rec.gate("ladder_ratio_measured", not ladder_ratio["provisional"], ladder_ratio)- # THE RATIO IS PART OF WHAT THE PAYLOAD MEASURED, and this is the first moment it is- # known -- the same shape as the commit check above, and for the same reason: the- # rungs already in the file were built at the ratio they record, the rungs still owed- # would be built at the one measured now, and one report prints both as one ladder.+ # THE RATIO IS PART OF WHAT THE PAYLOAD MEASURED, and this is the first moment it is known: rungs+ # already in the file were built at the ratio they record, rungs still owed at the one measured+ # now, and one report prints both as one ladder.if args.resume:ratio_issues: list = []for recorded in recorded_identities(paths.payload_jsonl):@@ -1315,12 +1155,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lif problem not in ratio_issues:ratio_issues.append(problem)if ratio_issues:- # AND NOTHING THIS SESSION WROTE SURVIVES THE REFUSAL. Unlike the commit check- # above, this one runs after the `Recorder` has opened the payload, so refusing- # is not enough on its own: `run_meta` is already in the file and- # `recorded_ladder` folds every one of them, so a refused- # `--resume --rungs 1K,10K` would leave 10K promised and unrecorded in a- # payload that was complete. See `rollback_session_rows`.+ # AND NOTHING THIS SESSION WROTE SURVIVES THE REFUSAL: this runs after the Recorder opened the+ # payload, so a refused `--resume --rungs 1K,10K` would leave 10K promised and unrecorded.rec.close()rollback_session_rows(paths.payload_jsonl, mark)raise SystemExit(@@ -1352,9 +1188,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l]work = interleave(cells, targets)if not order_is_balanced(work):- # Said out loud rather than silently absorbed: with an odd number of reps one side- # always runs first, so anything drifting monotonically through the session lands on- # the other one instead of cancelling.+ # Said out loud rather than absorbed: with an odd number of reps one side always runs first, so+ # anything drifting monotonically through the session lands on the other rather than cancelling._log(" WARNING: the run order is not balanced (use an even --reps). Linear drift ""within the session is charged to whichever side runs second."@@ -1364,11 +1199,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l"row_type": "ab_plan","base_ref": sides[0]["ref"],"treatment_ref": sides[1]["ref"],- # WHICH SERVER THE TREATMENT WAS, when the caller attached one. `run_meta`- # already records the base that way in `studio_ref`; without the same for the- # treatment a resume can only compare the `--ab` label, which says nothing about- # the build that answered on the port. See `requested_identity`. Empty when this- # run installed the treatment itself: then the ref above is the identity.+ # WHICH SERVER THE TREATMENT WAS, when the caller attached one: without it a resume compares only+ # the `--ab` label, which says nothing about the build that answered. Empty when we installed it."treatment_url": "" if sides[1]["owns"] else sides[1]["base_url"],# The treatment's half of `studio_commit`. Same reason, same emptiness rule."treatment_commit": sides[1].get("commit") or "",@@ -1380,10 +1212,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lwork = [(None, cell, plan) for cell, plan in cells]if done:- # AT PAIR GRANULARITY. An A/B pair whose two arms are not both recorded is re-run- # whole, so the resumed session never measures one arm on its own -- see- # `ab.skippable_cells`. For a run without --ab every pair holds one cell and this is the- # set it already was.+ # AT PAIR GRANULARITY: an A/B pair whose two arms are not both recorded is re-run whole, so a+ # resumed session never measures one arm on its own.from .runtime.ab import skippable_cellsdone = skippable_cells(work, done)_log(f" resuming: {len(done)} cells already in {paths.payload_jsonl.name}")@@ -1417,9 +1247,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lrec.close()if ab_ref:- # The cells THIS session was asked to measure. A resumed A/B skips whole pairs or none- # (`ab.skippable_cells`), so this is either every planned cell or none of them, and it is- # what `_render_ab` checks the payload against before it is allowed to print a verdict.+ # The cells THIS session was asked to measure: a resumed A/B skips whole pairs or none, so this+ # is every planned cell or none, and it is what `_render_ab` checks the payload against.+ # The skip set comes from `ab.skippable_cells`._render_ab(paths,sides,@@ -1462,8 +1292,8 @@ def _sweep_surfaces(sides: list, ctx, paths) -> None:recorder = ctx.recorder,)except Exception as exc: # noqa: BLE001- # Recorded as a failed gate rather than swallowed. A sweep that raised and a sweep- # that found nothing look identical in a payload that only carries the rows.+ # Recorded as a failed gate rather than swallowed: a sweep that raised and a sweep that found+ # nothing look identical in a payload that only carries rows._log(f" the surface sweep failed: {type(exc).__name__}: {exc}")ctx.recorder.gate(f"surface_sweep:{label}", False, {"error": f"{type(exc).__name__}: {exc}"}@@ -1476,9 +1306,8 @@ def _sweep_surfaces(sides: list, ctx, paths) -> None:out = paths.out / f"surfaces_{label}.md"out.write_text(text, encoding = "utf-8")_log(f"surface coverage manifest written to {out}")- # PASSES only when every non-conditional surface was reached AND the digests were scoped.- # An unscoped sweep reports one page-wide digest per surface, which agrees everywhere for- # reasons that have nothing to do with the surfaces.+ # PASSES only when every non-conditional surface was reached AND the digests were scoped: an+ # unscoped sweep reports one page-wide digest per surface, which agrees everywhere.passed = manifest["not_reached_hard"] == 0 and manifest["digests_scoped"]ctx.recorder.gate(f"surface_sweep:{label}", passed, manifest)@@ -1595,10 +1424,8 @@ def _render_ab(out = paths.out / "ab.md"- # NO TABLE AT ALL for a probe run, rather than a table with a warning printed above it. The- # warning scrolls off; `ab.md` sits in the output directory and gets pasted into a pull- # request. This is the same refusal `--report` and `floor_table` make, on the same evidence,- # and it is the entry point that actually runs at the end of every session.+ # NO TABLE AT ALL for a probe run, rather than a table with a warning above it: the warning+ # scrolls off while `ab.md` gets pasted into a pull request.from .scoring.from_payload import probe_scriptsprobes = probe_scripts(records)@@ -1610,11 +1437,8 @@ def _render_ab()_log("")_log(reason)- # OVERWRITTEN, not left alone and not deleted. `--resume` reuses the output directory, so- # an `ab.md` from an earlier clean run of the same directory would survive this refusal- # and sit at the standard artifact path, where it reads as this run's result. Deleting it- # would leave whoever opens the path with nothing to explain the absence, so the file is- # replaced by the refusal itself.+ # OVERWRITTEN, not left alone and not deleted: `--resume` reuses the directory, so an earlier+ # clean `ab.md` would read as this run's result, while deleting it explains nothing.stale = paths.out / "ab.md"if stale.exists():stale.write_text(f"# No A/B table\n\n{reason}\n", encoding = "utf-8")@@ -1622,27 +1446,16 @@ def _render_ab(_log("")return- # A FULLY RESUMED A/B MEASURED NOTHING OF ITS OWN. `--resume` against a finished output skips- # every cell and is a success, but the ratio is scoped to THIS session, so re-rendering here- # replaced a real table with NO READING and exited 0 while doing it. The run that measured- # keeps its report; a run with no prior table still gets one, since there is nothing to lose.- #- # AFTER THE PROBE REFUSAL, NOT BEFORE IT. The payload the resume keeps is the payload this- # check is deciding on behalf of, and a probed one is not scorable no matter which session- # recorded it. Run first, this returned early for the one sequence that needs the refusal- # most: a fresh PROBE run into a directory that already holds a clean `ab.md`, killed after- # its last cell was fsynced and before it rendered -- `archive_payload` moves `payload.jsonl`- # and nothing else, so the clean table stays at the standard artifact path -- and then- # resumed, which finds every cell complete, records none of its own and left that table- # standing over an unscorable probe payload.+ # A FULLY RESUMED A/B MEASURED NOTHING OF ITS OWN: the ratio is scoped to THIS session, so+ # re-rendering replaced a real table with NO READING and exited 0. AFTER the probe refusal, or+ # this returns early for exactly the sequence that needs the refusal most.if not any(r.get("row_type") == "cell" and r.get("session_id") == session_id for r in records):if out.exists():_log(f"\nno cell ran in this session; keeping the A/B table already at {out}")return- # Detected, not declared. `--ab main` IS a null control whether or not the caller says so, and- # a null control that renders as an ordinary A/B invites somebody to quote "7.7% faster" from a- # build compared with itself. See `is_null_control`.+ # Detected, not declared: `--ab main` IS a null control whether the caller says so or not, and+ # one rendered as an ordinary A/B invites somebody to quote a build compared with itself.is_null = is_null_control(sides)label = _ab_label(sides, is_null)@@ -1674,8 +1487,8 @@ def _render_ab(out.write_text(text, encoding = "utf-8")_log(f"A/B table written to {out}")if missing:- # A noise floor derived from a partial null control is a number about the cells that did- # not die, quoted afterwards at every real A/B on this machine.+ # A noise floor derived from a partial null control is a number about the cells that did not+ # die, quoted afterwards at every real A/B on this machine._log("the plan did not complete, so no noise floor is derived from it")returnif is_null:@@ -1695,103 +1508,51 @@ def _render_ab()-#: THE PAYLOAD IDENTITY. The axes that decide whether a cell already in a payload measures the-#: same thing a new invocation is asking for.-#:-#: `cell_id` is `r{rung}.{arm}.rep{rep}` and nothing else, so every one of these can change under a-#: cell id that stays identical: the tier picks the FILM (the standard film runs 243 s, the quick-#: one 77.5 s and the fast one 47 s, with different budgets), the cadence picks the rate the reply-#: streams at, the instrument level decides how much of the number is the instrument, the corpus-#: hash is the fixture, the engine is what rendered every frame, and the two refs are the builds-#: under test.-#:-#: `rungs` and `reps` are deliberately NOT here. Resuming with more repetitions or another rung is-#: a legitimate continuation -- it ADDS cells rather than reinterpreting the ones already recorded.+#: THE PAYLOAD IDENTITY: the axes deciding whether a recorded cell measures what a new invocation+#: asks for. `cell_id` is only `r{rung}.{arm}.rep{rep}`, so tier, cadence, instrument level,+#: corpus, engine and both refs can change under one id. `rungs`/`reps` are NOT here: they ADD+#: cells rather than reinterpreting recorded ones.IDENTITY_AXES = ("tier","cadence","engine",- # THE INSTRUMENTS THEMSELVES. `TOOL_VERSION` is bumped when what an instrument MEASURES- # changes and for no other reason -- this commit bumped it to 0.2.0 because- # `reasoning_toggle.open_ms` now terminates on a settled mount rather than on the `data-state`- # flip, which makes it a different quantity under the same name. None of that moves a cell id,- # so `--resume` into a half-finished 0.1.0 payload from an upgraded tree skipped the completed- # cells and appended the rest measured by 0.2.0: one ladder built from two instruments, which- # is the same failure the tier and the corpus are on this list for. `merged_run_meta` names the- # mixture afterwards, but only `--compare` and a floor-gated `floor_table.render` call it --- # plain `--report` takes the FIRST header and pools the two quantities without a word. Refused- # here instead, before anything is installed and before anything is measured.+ # THE INSTRUMENTS THEMSELVES: `TOOL_VERSION` moves only when what an instrument MEASURES changes,+ # and none of that moves a cell id, so `--resume` appended 0.2.0 measurements under 0.1.0 ids.+ # Plain `--report` takes the FIRST header and pools the two quantities without a word.+ # 0.2.0 was the bump: `reasoning_toggle.open_ms` now ends on a settled mount, not the+ # `data-state` flip."tool_version","instrument_level","corpus_hash","studio_ref","treatment_ref","treatment_url",- # Added by this branch with `--stream-tail-chars` / `--corpus-dollars`: both change the reply- # the cell streams without moving its id, so they belong to the identity for the same reason- # the tier does. Recorded already, so no schema change and a payload from before them reads- # back as the defaults (see `HISTORICAL_DEFAULTS`).+ # `--stream-tail-chars` / `--corpus-dollars`: both change the reply the cell streams without+ # moving its id. A payload from before them reads back as the defaults."stream_tail_chars","corpus_dollars",- # `--click-probe`, whose own help text says it "makes the cell's timings incomparable with a- # cell that did not run it". That is the definition of an identity axis: the probe runs a full- # `page.click`, a real mouse click, a dispatch, a focus and a hover over the thread before the- # film starts, and the cell then records a `composer_click_ms` measured on a composer those- # paths have already been through plus a `click_attribution` block a cell without the flag- # does not have at all. None of it moves the cell id, so a resume that toggles the flag skips- # the completed cells and appends the rest under the same ids -- the one ladder built from two- # films this check exists to refuse.+ # `--click-probe`, whose help text says it makes timings incomparable: it drives a real click,+ # focus and hover before the film and adds a `click_attribution` block, none of it in the id."click_probe",- # `--inject-stream-cost-ms`, on the same rule and for a larger perturbation than either of- # those two. It burns a known amount of main-thread time per SSE chunk on the TREATMENT arm,- # so a treatment cell recorded with it is a different reading from one recorded without it,- # under a cell id that cannot tell -- the id carries the rung, the arm and the repetition and- # stops there. Off it, `--resume` had two ways to lie about a calibration: against a FINISHED- # uninjected payload every pair is skippable, the run exits 0 having measured nothing, and the- # recovery gate is answered from cells that were never injected; against a HALF-FINISHED- # injected one the resume drops the flag and the ladder ends up part injected and part not.- # Both land as a recovery fraction near zero, which reads as "the metric is blind" and is the- # single verdict this flag exists to make trustworthy.+ # `--inject-stream-cost-ms`, a larger perturbation on the TREATMENT arm only. Off this list, a+ # resume could answer the recovery gate from cells never injected, which reads as a blind metric."inject_stream_cost_ms",- # `SBENCH_EXTRA_INIT_SCRIPT`, the external probe. Unlike the axes above it cannot end in a- # wrong NUMBER -- `refuse_if_probed` reads every `run_meta` in the file and every scoring entry- # point calls it, so one probed session makes the whole payload unscorable and there is no flag- # to override that. It is here for what the refusal costs instead. The variable is an- # ENVIRONMENT variable rather than a flag, so it survives in a shell after the experiment that- # set it is over, and `--resume` into a half-finished clean payload then installs both sides,- # runs the rungs that are still owed with the probe in the page, and appends a probed- # `run_meta` to the file. The payload is append-only and the refusal is whole-file, so the- # cells that were recorded cleanly before it are unscorable from then on and nothing this tool- # offers takes it back. Refusing here costs a millisecond and happens before the first install.+ # `SBENCH_EXTRA_INIT_SCRIPT`, the external probe. It cannot end in a wrong number, so it is here+ # for what the refusal COSTS: an env var survives in a shell, and a resume would append a probed+ # `run_meta` that makes every previously clean cell unscorable, irreversibly."probe_init_script",- # `--headed`. On the same rule as `engine` directly above the others, and for the same reason- # the engine is there at all: it decides which browser binary renders the film. Since- # Playwright 1.57 headed and headless resolve to different executables (`chrome` against- # `chrome-headless-shell`), headless falls back to software rendering for GPU-accelerated- # work, and the compositor's own pacing differs. A resume that toggles it would skip the- # completed cells and append the rest under the same ids, building one ladder from two- # renderers -- exactly what this check exists to refuse.+ # `--headed`, for the same reason `engine` is here: headed and headless resolve to different+ # executables with different rendering, so toggling it builds one ladder from two renderers."headed",)-#: The axes that describe the SECOND side, which only exist when a run has one. See-#: `identity_problems`: an A/B judged against a run that is not one may not differ on them.+#: The axes that describe the SECOND side, which only exist when a run has one. An A/B judged+#: against a run that is not one may not differ on them.TREATMENT_AXES = ("treatment_ref", "treatment_url")-#: The axes whose ABSENCE from a payload is itself a reading, and what it reads as.-#:-#: `identity_problems` otherwise skips an axis the payload never declared, because an axis a row-#: never declared cannot be a difference. That is right for the axes `run_meta` has always carried:-#: a payload missing one of those is a payload this check has nothing to say about.-#:-#: It is WRONG for the ones below. They arrived with the flag or the variable that sets them, so a-#: payload written before them did not decline to record a value -- there was no way to ask for-#: anything but the default, and it ran under `stream_tail_chars = None`, `corpus_dollars = False`,-#: `click_probe = False`, `inject_stream_cost_ms = None` and `probe_init_script = None` by-#: construction. Skipping them therefore accepted `--resume --stream-tail-chars 24000` against such-#: a payload, skipped its completed cells, and recorded the rest under a different streamed fixture-#: beneath the same cell ids: one ladder built from two films, which is what this check exists to-#: refuse. Absence proves the value here, so it is read as the value.+#: The axes whose ABSENCE from a payload is itself a reading. `identity_problems` otherwise skips+#: an undeclared axis, which is wrong for these: they arrived with the flag that sets them, so an+#: older payload ran under the default by construction and one ladder was built from two films.HISTORICAL_DEFAULTS = {"stream_tail_chars": None,"corpus_dollars": False,@@ -1801,33 +1562,25 @@ HISTORICAL_DEFAULTS = {"headed": False,}-#: THE RATIO THE LADDER WAS SIZED BY, carried on every cell's `meta` by `session.build_cells`.-#: Not in `IDENTITY_AXES`: those are checked by `prepare_payload` before anything is installed,-#: and this one is not known until `build_cells` has measured the corpus. Checked instead by-#: `ladder_ratio_problems`, on the same footing as `COMMIT_AXES` -- late enough to have the answer,-#: early enough that nothing has been measured under it.+#: THE RATIO THE LADDER WAS SIZED BY, carried on every cell's `meta`. Not in `IDENTITY_AXES`,+#: which are checked before anything is installed, because this is not known until `build_cells`+#: has measured the corpus; `ladder_ratio_problems` checks it instead.+# Carried by `session.build_cells`.LADDER_RATIO_AXIS = "ladder_chars_per_token"-#: How close two ratios must be to be the same ratio. `measure_chars_per_token` rounds its answer-#: to three decimals and `PROVISIONAL_CHARS_PER_TOKEN` is exact, so every value that reaches a-#: payload sits on a 0.001 grid: half a grid step separates "the same measurement, re-read through-#: JSON" from "a different one". Wide enough that 5.0 never refuses 5.0 for a float's sake, narrow-#: enough that the case this exists for -- a provisional 4.0 against tiktoken's 3.336 on this-#: corpus, a fifth of the character load of every rung -- can never be inside it.+#: How close two ratios must be to be the same ratio. Payload values sit on a 0.001 grid, so half+#: a step separates a re-read from a different measurement: 5.0 never refuses 5.0, and a+#: provisional 4.0 can never be inside tiktoken's 3.336.+# `measure_chars_per_token` rounds its answer onto that grid.LADDER_RATIO_TOLERANCE = 5e-4-#: The arm a run without `--ab` records every cell under. `Cell.arm` in `runtime.types`; an A/B-#: overwrites it with the target's label, `base` or `treatment`. Named here because it is what-#: tells `recorded_identities` which of the two a session already in the payload was.+#: The arm a run without `--ab` records every cell under; an A/B overwrites it with base or+#: treatment. Named here because it tells `recorded_identities` which a recorded session was.+# `Cell.arm` in `runtime.types`.SINGLE_ARM = "A0"-#: THE BUILD, as opposed to the name it was asked for by. A ref is a POINTER: `main`, a topic-#: branch and a movable tag all resolve afresh on every install, and `checkout_ref` is the only-#: thing that ever knows which commit one named. So these are not in `IDENTITY_AXES` and are not-#: checked by `prepare_payload`: that refusal is deliberately made BEFORE anything is installed,-#: and a commit cannot be known before the fetch that resolves it. They are checked instead by-#: `commit_problems`, once the sides are up and before the first cell -- late enough to have the-#: answer, early enough that nothing has been measured under it. See `run`.+#: THE BUILD, as opposed to the name it was asked for by. A ref is a POINTER that resolves afresh+#: on every install, so `commit_problems` checks these once the sides are up, not earlier.COMMIT_AXES = ("studio_commit", "treatment_commit")@@ -1844,41 +1597,33 @@ def requested_identity(args, ab_ref, corpus_hash: str) -> dict:return {"tier": args.tier,"cadence": args.cadence,- # THE INSTRUMENTS THIS TREE CARRIES, as a module constant rather than anything the caller- # can ask for. A payload recorded by another version was measured by other instruments.+ # THE INSTRUMENTS THIS TREE CARRIES, a module constant rather than anything the caller can ask+ # for: a payload recorded by another version was measured by other instruments."tool_version": TOOL_VERSION,- # THE ENGINE THAT WILL RENDER, not the flag that asked for it. `--engine` defaults to- # nothing and `browser.launch` resolves the platform's desktop webview family, so the- # requested value has to be resolved the same way `launch` resolves it -- otherwise the- # commonest engine change of all, one run naming `--engine chromium` and the resume- # naming none, reads as no difference at all. Resolvable here, before anything is- # installed, because `default_engine` reads `platform.system()` and nothing else.+ # THE ENGINE THAT WILL RENDER, not the flag that asked for it: `--engine` defaults to nothing and+ # `browser.launch` resolves the platform's webview family, so the request must resolve the same+ # way or the commonest engine change of all reads as no difference.+ # Resolvable before anything is installed: `default_engine` reads `platform.system()` and+ # nothing else."engine": getattr(args, "engine", "") or default_engine()[0],"instrument_level": args.instrument_level,"corpus_hash": corpus_hash,"studio_ref": base_ref,"treatment_ref": ab_ref or "",- # THE ATTACHED TREATMENT IS THE SERVER, NOT THE LABEL. `studio_ref` folds an attached base- # down to its URL because that is the only thing about it this harness can see; the- # treatment carried nothing but the name typed after `--ab`, and that name is free. So- # `--attach A --attach-b B --ab fix --resume`, re-run later against `--attach-b C`, passed- # the identity check, skipped every completed treatment cell and reported B's measurements- # as C's result. Empty for a treatment this run installs itself, whose ref IS its identity.+ # THE ATTACHED TREATMENT IS THE SERVER, NOT THE LABEL: carrying only the free-form name let+ # `--attach-b B` pass the identity check against a later `--attach-b C` and report B's numbers+ # as C's. Empty for a treatment this run installs itself.+ # The field is `studio_ref`; the resolved build is `requested_identity`."treatment_url": attach_b if (ab_ref and attach_b) else "","stream_tail_chars": args.stream_tail_chars,"corpus_dollars": bool(args.corpus_dollars),"click_probe": bool(getattr(args, "click_probe", False)),"inject_stream_cost_ms": getattr(args, "inject_stream_cost_ms", None),- # Spelled the same way `run_meta` records it, through `bool`, so the recorded False and- # the requested value are the same type. Adding the axis without this line made every- # resume refuse itself -- the payload declared `False` and the request declared `None` --- # which is the failure mode an identity axis has when it is only half wired up.+ # Spelled the same way `run_meta` records it, through `bool`, so the recorded False and the+ # requested value are the same type; without this every resume refused itself."headed": bool(getattr(args, "headed", False)),- # FROM THE ENVIRONMENT, because that is where this one is asked for: the probe hook is a- # variable rather than a flag on purpose, so `args` never sees it and there is nothing to- # read it from. Spelled exactly as `run_meta` records it -- the path as it was given -- for- # the same reason `studio_ref` is, so the requested value and the recorded one compare- # without a second convention to keep in step.+ # FROM THE ENVIRONMENT, because the probe hook is a variable rather than a flag, so `args` never+ # sees it. Spelled exactly as `run_meta` records it, so requested and recorded compare directly."probe_init_script": os.environ.get("SBENCH_EXTRA_INIT_SCRIPT") or None,}@@ -1914,17 +1659,13 @@ def recorded_identities(payload_path) -> list:by_session[session] = {}order.append(session)if row_type == "cell":- # THE MODE A SESSION MEASURED IN, declared by the cells it actually wrote rather- # than by a header it wrote before it measured anything. A run that died between- # `run_meta` and `ab_plan` recorded no cells and so declares no mode, and resuming- # it is not a transition; a session holding cells declares the mode those cells- # were recorded under, whichever row types the version that wrote them emitted.+ # THE MODE A SESSION MEASURED IN, declared by the cells it wrote rather than a header written+ # before it measured anything: a run that died before `ab_plan` declares no mode.arm = str((row.get("cell") or {}).get("arm") or row.get("arm") or "")if arm:by_session[session].setdefault("mode", "single" if arm == SINGLE_ARM else "ab")- # THE RATIO THE SESSION'S RUNGS WERE SIZED BY, on the same rule as the mode: read- # off a cell the session actually wrote, so a session that recorded none declares- # none. See `ladder_ratio_problems`.+ # THE RATIO THE SESSION'S RUNGS WERE SIZED BY, on the same rule as the mode: read off a cell the+ # session actually wrote. See `ladder_ratio_problems`.sized = ((row.get("cell") or {}).get("meta") or {}).get("ladder_chars_per_token")if isinstance(sized, dict) and sized.get("chars_per_token") is not None:by_session[session].setdefault(@@ -1934,13 +1675,9 @@ def recorded_identities(payload_path) -> list:for axis in IDENTITY_AXES + COMMIT_AXES:if axis in row:by_session[session][axis] = row[axis]- # THE ENGINE IS NESTED, under `run_meta.platform`, so the loop above cannot see it.- # Read from there rather than from a new top-level field for the same reason as- # everything else here: a payload written before this check existed still declares- # what it rendered with, and is judged on it. `run_meta` is emitted AFTER- # `browser.launch` returns, so what is recorded is the RESOLVED engine and a session- # that never got a browser up declares none -- and an axis a row never declared- # cannot be a difference, so that session still resumes.+ # THE ENGINE IS NESTED under `run_meta.platform`, so the loop above cannot see it. `run_meta` is+ # emitted after `browser.launch`, so the RESOLVED engine is recorded and a session that never got+ # a browser up declares none -- and an undeclared axis cannot be a difference.if row_type == "run_meta":engine = str((row.get("platform") or {}).get("engine") or "")if engine:@@ -1954,16 +1691,10 @@ def recorded_identities(payload_path) -> list:def identity_problems(recorded: dict, requested: dict) -> list:"""Every axis on which a recorded session and this invocation disagree."""problems = []- # AN A/B AND A RUN THAT IS NOT ONE MAY NOT SHARE A PAYLOAD. The arm in the cell id ("A0"- # against "base"/"treatment") keeps the new cells from being SKIPPED, which is why this used- # to be waved through -- but it does not keep them out of each other's REPORT. `score_payload`- # hands every cell to `measures_from_records`, which keys by rung and keeps the first reading,- # and `_completion_by_rung` keeps a failure over a success at the same rung. So a single-build- # run that died at 10K, resumed as `--ab fix` and re-measured successfully on both arms, still- # printed `INCOMPLETE: the base died at 10K`, scored the rung 0 and reported ONSET RUNG: none,- # while the identical resume WITHOUT `--ab` -- same crash, same repair, same cell id, so the- # dead attempt is superseded -- scored it 82.2. The mode is part of what a payload measures,- # so a run that changes it resumes nothing: it starts a payload of its own.+ # AN A/B AND A RUN THAT IS NOT ONE MAY NOT SHARE A PAYLOAD. The arm in the cell id keeps the new+ # cells from being SKIPPED but not out of each other's REPORT: `measures_from_records` keeps the+ # first reading per rung and `_completion_by_rung` keeps a failure over a success, so a resume as+ # `--ab fix` scored a rung 0 where the same resume without `--ab` scored 82.2.requested_mode = "ab" if requested.get("treatment_ref") else "single"recorded_mode = recorded.get("mode")if recorded_mode is not None and recorded_mode != requested_mode:@@ -1972,22 +1703,19 @@ def identity_problems(recorded: dict, requested: dict) -> list:f"mode: the payload was recorded by a run measuring {names[recorded_mode]}, "f"this run measures {names[requested_mode]}")- # Is there a second side on BOTH sides of this comparison? When there is not, the treatment- # axes describe a side one of them does not have, and the mode difference above is the whole- # of what they disagree on. Decided from the ref, so that an A/B whose treatment this run- # installed -- which records no treatment URL -- is still judged against an attached one on- # the axis where the two of them do differ.+ # Is there a second side on BOTH sides of this comparison? When there is not, the treatment axes+ # describe a side one of them lacks. Decided from the ref, so an installed treatment still+ # compares against an attached one on the axis where they do differ.both_ab = bool(requested.get("treatment_ref")) and bool(recorded.get("treatment_ref"))for axis in IDENTITY_AXES:declared = axis in recordedif declared:got = recorded[axis]elif axis in HISTORICAL_DEFAULTS:- # Not declared, but not silent either: this axis postdates the payload, so the payload- # ran under the default. See `HISTORICAL_DEFAULTS`.+ # Not declared, but not silent either: this axis postdates the payload, so the payload ran under+ # the default. See `HISTORICAL_DEFAULTS`.got = HISTORICAL_DEFAULTS[axis]else:- # An axis this payload never declared. See `recorded_identities`.continueif axis in TREATMENT_AXES and not both_ab:continue@@ -2319,8 +2047,8 @@ def _resume_set(paths) -> set:except ValueError:continuefor row in latest_attempt_rows(records):- # Only a COMPLETED cell is skipped. A cell that died is re-run, because its failure- # may have been the machine and not the build.+ # Only a COMPLETED cell is skipped: a cell that died is re-run, because its failure may have+ # been the machine and not the build.if row.get("row_type") == "cell" and row.get("completed"):done.add(row.get("cell_id"))return done@@ -2338,8 +2066,8 @@ def _summarise(rows: list, paths) -> None:for r in rows:actions = r.get("actions") or []ran = sum(1 for a in actions if a.get("ran"))- # The PEAK, not the end state: the film's last two actions reopen the thread and delete- # a message, so an end-of-film census describes a thread that is no longer there.+ # The PEAK, not the end state: the film's last two actions reopen the thread and delete a+ # message, so an end-of-film census describes a thread that is no longer there.census = r.get("census_peak") or r.get("census_after") or {}_log(f"{r['cell_id']:<16} {r.get('assistant_chars_in_dom') or 0:>10,} "@@ -2352,11 +2080,9 @@ def _summarise(rows: list, paths) -> None:if not r.get("completed"):f = r.get("failure") or {}_log(f" FAILED: {f.get('kind')}: {str(f.get('message'))[:100]}")- # SAID UNDER THE TABLE THAT PRINTS THEM. `elems` and `spans` come from `census_peak`, whose- # action is chosen by a max() over per-action censuses that race their own teardown, so it is- # not the same moment on two arms. Differencing these two columns between arms is what- # produced a withdrawn "43% more standing DOM" claim. The columns stay because the high-water- # mark is a useful diagnostic; the warning stays with them so it cannot be read without it.+ # SAID UNDER THE TABLE THAT PRINTS THEM: `elems` and `spans` come from `census_peak`, whose action+ # is chosen by a max() over censuses racing their own teardown, so it is not the same moment on+ # two arms. Differencing them produced a withdrawn "43% more standing DOM" claim._log("\n elems/spans are census_peak: a diagnostic high-water mark, taken at whichever ""action mounted most.\n Do NOT difference them between arms. For a cross-arm census use "@@ -2427,9 +2153,8 @@ def report_only(args) -> int:_log(f"no payload at {path}")return 2- # `--rungs` first, then the payload's own ladder, then the tier. An explicit `--tier` still- # wins over the recording, so a reader can deliberately score a payload against another tier's- # ladder; what is gone is the DEFAULT tier silently shortening somebody else's run.+ # `--rungs` first, then the payload's own ladder, then the tier. An explicit `--tier` still wins;+ # what is gone is the DEFAULT tier silently shortening somebody else's run.recorded = [] if getattr(args, "tier_explicit", True) else recorded_ladder(path)if args.rungs:declared = _rung_tokens(args.rungs.split(","))@@ -2442,20 +2167,17 @@ def report_only(args) -> int:try:text, ladder, _payload = build_report(path, declared)except SystemExit as exc:- # A REFUSAL, not a crash, and it has to reach the artefact rather than only the terminal.- # `SystemExit` is not an `Exception`, so without this clause it would leave the process- # before the write below. `--resume` reuses the output directory, so a `summary.md` from- # an earlier clean report of the same directory would survive the refusal and sit next to- # a now-probed payload, reading as its result. Same reasoning as the stale `ab.md` in- # `_render_ab`: overwritten rather than deleted, so opening the path gives the reason.+ # A REFUSAL, not a crash, and it has to reach the artefact rather than only the terminal:+ # `SystemExit` is not an `Exception`, so without this clause a clean `summary.md` would sit next+ # to a now-probed payload. Overwritten rather than deleted, as with the stale `ab.md`._log(str(exc))if out.exists():out.write_text(f"# No summary\n\n{exc}\n", encoding = "utf-8")_log(f" a previous {out} was replaced by this refusal")return 2except Exception as exc: # noqa: BLE001- # A payload that cannot be scored is reported as such rather than half-rendered: a- # partial report is exactly the artefact that gets quoted without its caveats.+ # A payload that cannot be scored is reported as such rather than half-rendered: a partial+ # report is exactly the artefact that gets quoted without its caveats._log(f"could not build a report from {path}: {type(exc).__name__}: {exc}")return 1@@ -2482,21 +2204,10 @@ def compare_payloads(args) -> int:if not path.exists():print(f"no such payload: {path}")return 2- # EVERY header, not the first. `--resume` appends a second `run_meta` describing the run- # that continued the file, and it may legitimately have extended the ladder: `rungs` is- # deliberately outside `IDENTITY_AXES` because adding a rung ADDS cells rather than- # reinterpreting the recorded ones. Keying on the first header therefore hashes a ladder- # the file has since outgrown, and declares a resumed payload comparable with the one-rung- # run it started as. `floor_table.tiers_of` and `corpora_of` were both fixed for exactly- # this; `--compare` was the reader left behind.- #- # READ THE WAY EVERY OTHER READER IN THIS TOOL READS. The payload is append-only and every- # row is flushed as it is written, so a run killed during its last append leaves valid rows- # followed by a torn one -- the failure this format is chosen to survive, and the reason- # `report.read_records` counts a malformed line instead of raising. `recorded_identities`- # and `_resume_set` skip one too. Parsing it here with a bare `json.loads` made `--compare`- # the only reader that answered an interrupted payload with a JSONDecodeError traceback- # rather than with the check it was asked for.+ # EVERY header, not the first: `--resume` appends a second `run_meta` and may legitimately have+ # extended the ladder, so keying on the first hashes a ladder the file has outgrown. Read the way+ # every other reader does -- a run killed mid-append leaves a torn last line that+ # `report.read_records` counts rather than raising on.records, discarded = read_records(path)if discarded:print(@@ -2508,8 +2219,8 @@ def compare_payloads(args) -> int:print(f"{path} carries no run_meta row, so it cannot be checked at all")return 2if conflicts:- # A file whose own headers disagree is not one measurement, so no single key can stand- # for it and the honest answer is a refusal rather than a token.+ # A file whose own headers disagree is not one measurement, so no single key can stand for it+ # and the honest answer is a refusal.print(f"{path} disagrees with ITSELF across its own run_meta rows:")for line in conflicts:print(f" {line}")@@ -2588,24 +2299,16 @@ def assert_liveness(args) -> int:except ValueError:problems.append("a payload line is not valid JSON")- # A CELL THAT WAS RE-RUN IS JUDGED ON THE RUN THAT FINISHED IT, which is the same rule the- # report and the A/B already apply and the same helper that applies it. `--resume` appends to- # the payload it continues and re-runs the cells that died under the SAME deterministic- # `cell_id`, so the dead attempt's `completed: false` and its NOT RUN actions are still in the- # file after a resumed run has succeeded. Read raw, this loop found them forever: the resumed- # run exited 0 and `--report` scored the retry, while `--assert-liveness` on the same payload- # failed permanently on a cell that had already been re-run. A gate that cannot be satisfied- # by fixing the run is a gate people learn to pass with `--allow-not-run`.+ # A CELL THAT WAS RE-RUN IS JUDGED ON THE RUN THAT FINISHED IT, as the report and the A/B do:+ # the dead attempt's `completed: false` stays in the file, so read raw, `--assert-liveness`+ # failed permanently on a cell that had already been re-run.from .scoring.from_payload import ATTEMPT_ROW_TYPES, latest_attempt_rows- # AN ATTEMPT THAT NEVER REACHED ITS CELL ROW IS A FAILURE, NOT AN ABSENCE. `latest_attempt_rows`- # names the latest attempt from ANY attempt-keyed row, because the Recorder flushes and fsyncs- # every action and window row while `CellRunner.run` writes the terminal `cell` row in a- # `finally` a SIGKILL, an OOM kill or a lost machine never reaches. So the killed attempt- # supersedes the completed one and then contributes no cell row of its own -- and this loop,- # reading cell rows only, dropped the cell out of the payload entirely. A resume killed inside a- # cell whose first attempt had already recorded `completed: false` with NOT RUN actions turned- # `1` into `0`: the gate stopped reporting a failure that is still written in the file.+ # AN ATTEMPT THAT NEVER REACHED ITS CELL ROW IS A FAILURE, NOT AN ABSENCE: the Recorder fsyncs+ # every action row while the terminal `cell` row is written in a `finally` a SIGKILL never+ # reaches, so reading cell rows only turned a recorded failure into a 0.+ # `CellRunner.run` writes it.+ # `latest_attempt_rows` is what has to notice.attempted, recorded = set(), set()kept = latest_attempt_rows(rows)for row in kept:@@ -2628,32 +2331,23 @@ def assert_liveness(args) -> int:for action in row.get("actions") or []:name = action.get("action") or action.get("name") or "?"if action.get("slot_missed"):- # A MISSED SLOT IS NOT AN ACTION THE PLATFORM CANNOT PERFORM, so it is classified- # before the allowance rather than inside it. `scene/schedule.py` records an- # overrun as `ran = False, slot_missed = True`, so under an allowance-first order a- # listed name never reached this branch at all and its slot miss left the run- # silently: an excuse written for "the fixture cannot mount this" swallowed a fact- # about the machine. Ordering it first is what keeps the two buckets below honest.- # It is a machine-speed fact whether or not the action also reports ran=False, so- # it lands in `missed` and is judged against `--allow-slot-misses`, never here.+ # A MISSED SLOT IS NOT AN ACTION THE PLATFORM CANNOT PERFORM, so it is classified before the+ # allowance: an excuse written for "the fixture cannot mount this" was swallowing a fact about+ # the machine. It lands in `missed` and is judged against `--allow-slot-misses`.+ # `scene/schedule.py` records an overrun as `ran=False, slot_missed=True`.missed.append(f"{where}: {name} missed its slot ({action.get('reason') or '?'})")elif not action.get("ran"):- # THE ALLOWANCE IS EXACTLY WHAT ITS NAME SAYS. `--allow-not-run` excuses an action- # the fixture cannot mount at all; it is not a blanket exemption from the gate.- # `image_upload` is listed in studiobench-ci.yml only because the fixture cannot- # mount it, so if it ever does mount, an upload that produces no attachment must- # be a failure rather than an excuse inherited from a different reason.+ # THE ALLOWANCE IS EXACTLY WHAT ITS NAME SAYS: `--allow-not-run` excuses an action the fixture+ # cannot mount at all. If `image_upload` ever does mount, an upload with no attachment fails.if name in allowed:continueproblems.append(f"{where}: {name} NOT RUN ({action.get('reason') or 'no reason'})")elif action.get("expect_ok") is False:- # RAN IS NOT DID WHAT IT CLAIMED. `scoring/from_payload.py` and- # `report/payload.py` both already refuse a timing whose own assertion failed;- # this loop did not, so `ran = True` with `expect_ok = False` fell past both- # branches above and the gate exited 0. A selector regression that fails EVERY- # cell's assertion left the workflow green with the surface non-functional --- # the same "absent reported as no effect" this gate exists for, one branch over.- # It is a scene problem, not machine speed, so no allowance and no slack reach it.+ # RAN IS NOT DID WHAT IT CLAIMED: the scoring and report layers both refuse a timing whose+ # assertion failed, but this loop did not, so a selector regression failing every cell's+ # assertion left the workflow green. A scene problem, so no allowance reaches it.+ # The two layers are `scoring/from_payload.py` and `report/payload.py`.+ # The shape that fell through both branches was `ran=True` with `expect_ok=False`.problems.append(f"{where}: {name} ran but its own assertion failed "f"({action.get('reason') or 'no reason'})"@@ -2674,8 +2368,8 @@ def assert_liveness(args) -> int:+ (f", {len(allowed)} action(s) allowed not to run" if allowed else ""))if missed and not over:- # Said out loud rather than passed over: a run with missed slots has holes in its table,- # and the only thing this exit code claims is that the harness was not the cause.+ # Said out loud rather than passed over: a run with missed slots has holes in its table, and the+ # only thing this exit code claims is that the harness was not the cause._log(" the missed slots above are machine speed, not a harness fault, but every one of ""them is a hole in this run's table. Do not quote a number from this payload."@@ -2691,9 +2385,8 @@ def parse_args(argv: list):ap.add_argument("--tier",choices = TIERS,- # No default here, and `quick` is applied below instead. `--report` needs to tell "the- # caller asked for this ladder" apart from "the caller said nothing", because a payload- # records the ladder its run promised and that answer beats a CLI default.+ # No default here, and `quick` is applied below: `--report` must tell "the caller asked for this+ # ladder" from "the caller said nothing", because a payload's recorded ladder beats a CLI default.default = None,help = ("fast ~5min (100K only, the iteration loop), quick ~5min (1K,10K, a wiring check), "@@ -2763,10 +2456,8 @@ def parse_args(argv: list):"--windowed-arm",metavar = "ARMS",dest = "windowed_arm",- # An ENV FALLBACK as well as the flag. `scripts/pr_perf_sweep.py` builds studiobench's- # command line itself and is shared with other people's in-flight sweeps, so adding an- # argument there mid-run is a hazard; this lets the sweep driver select the gate through- # the environment without anybody editing the driver.+ # An ENV FALLBACK as well as the flag: `scripts/pr_perf_sweep.py` builds this command line+ # itself and is shared with other in-flight sweeps, so adding an argument mid-run is a hazard.default = os.environ.get("SBENCH_WINDOWED_ARM", ""),help = "comma-separated arm labels (base, treatment) that mount a WINDOW of the thread ""rather than all of it, and are therefore gated on the windowed readiness signal "@@ -2855,9 +2546,8 @@ def parse_args(argv: list):ap.add_argument("--branch", default = "main", help = "Unsloth ref to install when not attaching")ap.add_argument("--home", help = "UNSLOTH_STUDIO_HOME for an install")ap.add_argument("--port", type = int, default = 5399)- # `unsloth`, not `admin`. Unsloth's first run prints "DEFAULT ADMIN ACCOUNT CREATED / username:- # unsloth", and the wrong one answers 401 with a message about resetting the PASSWORD, which- # sends you looking in the wrong place.+ # `unsloth`, not `admin`: Unsloth's first run prints "DEFAULT ADMIN ACCOUNT CREATED / username:+ # unsloth", and the wrong one answers 401 with a message about resetting the PASSWORD.ap.add_argument("--username", default = "unsloth")ap.add_argument("--password", default = "")ap.add_argument(@@ -2905,8 +2595,7 @@ def parse_args(argv: list):"investigation by about 3.2x",)args = ap.parse_args(argv)- # Whether the ladder was ASKED FOR or merely defaulted. Only `--report` cares; everything else- # sees the tier it always saw.+ # Whether the ladder was ASKED FOR or merely defaulted. Only `--report` cares.args.tier_explicit = args.tier is not Noneif args.tier is None:args.tier = "quick"diff --git a/tests/studio/studiobench/analysis/__init__.py b/tests/studio/studiobench/analysis/__init__.pyindex e3780ab8f08..caa8fdd7992 100644--- a/tests/studio/studiobench/analysis/__init__.py+++ b/tests/studio/studiobench/analysis/__init__.py@@ -46,16 +46,13 @@ class CellFailure(RuntimeError):self.detail = detail-# ── the no-bare-zero convention (INTERFACES.md, "Rules that apply to every dict") ──-#-# `0` means "measured, and it was zero". It never means "did not run". Getting-# this wrong is the single cheapest way to publish a false negative: a frame-# budget of 0.00 ms reads as a fast app and is indistinguishable from an-# instrument that never attached. Every numeric key these layers emit therefore-# goes through one of the two helpers below, so the distinction is structural-# rather than remembered.+# The no-bare-zero convention (INTERFACES.md, "Rules that apply to every dict"). `0` means+# "measured, and it was zero" and never "did not run": a frame budget of 0.00 ms reads as a fast+# app and is indistinguishable from an instrument that never attached. Every numeric key these+# layers emit goes through one of the two helpers below, so the distinction is structural.+# ── the no-bare-zero convention (INTERFACES.md, "Rules that apply to every dict") ──def measured(key: str, value: Any) -> dict:"""A value that WAS measured, even if it came out zero."""return {key: value, f"{key}_attempted": True}diff --git a/tests/studio/studiobench/analysis/behaviour.py b/tests/studio/studiobench/analysis/behaviour.pyindex da0bd223e76..5f3596dc7a6 100644--- a/tests/studio/studiobench/analysis/behaviour.py+++ b/tests/studio/studiobench/analysis/behaviour.py@@ -54,42 +54,29 @@ from typing import Any, Optionalfrom .parity import MATCH, NOT_APPLICABLE, NOT_COMPARABLE, NOT_EXERCISED#: Verdict for a behavioural invariant that moved. Distinct from the digest's DIFFER so a report-#: can never present the two as the same kind of evidence.+#: cannot present the two as the same kind of evidence.BROKEN = "broken"-#: How far a quantity that should be IDENTICAL may drift. 2%, the same figure the seeded-versus--#: streamed equivalence check uses, and for the same reason: two runs of one build never produce-#: bit-identical character counts once a stream is involved.+#: How far a quantity that should be IDENTICAL may drift. 2%, as in the seeded-versus-streamed+#: equivalence check: two runs of one build never produce bit-identical character counts once a+#: stream is involved.EXACT_TOLERANCE = 0.02-#: How far the scroll extent may drift. 10%, much looser, because a windowed list computes its-#: total height from estimated row heights and corrects them as real rows are measured. Loose-#: enough to admit a correct virtualizer, nowhere near loose enough to admit one that dropped-#: rows: the failure this catches is an extent that is a FRACTION of the real one, not one that is-#: 6% out.+#: How far the scroll extent may drift. 10%, looser, because a windowed list computes its height+#: from estimated row heights and corrects them; the failure this catches is an extent that is a+#: FRACTION of the real one, not one 6% out.EXTENT_TOLERANCE = 0.10-#: How much of the thread the clipboard must carry, and how much more than the thread it may carry.-#:-#: TWO-SIDED, AND MEASURED AGAINST THE THREAD, because there are two ways to get a copy wrong and-#: the check used to see only one of them.-#:-#: The lower bound catches TRUNCATION, which is the defect this invariant was written for: a-#: windowed mount cannot select what it has not mounted, so a naive copy carries only the visible-#: fraction. Measured at 0.61 of the thread on a real 100K arm.-#:-#: The upper bound catches SUBSTITUTION, which is what a fix for the first defect turns into if-#: nobody is watching. Serialising from the message store is the right repair, but the obvious-#: serialiser is the "save this reply" one, which emits reasoning, tool-call arguments and tool-#: results -- none of which a user can select, because the panes holding them are collapsed and a-#: collapsed Radix Collapsible is not in the DOM at all. Measured at 2.16 of the thread on the same-#: arm: the truncation was fixed and the content was then wrong in the other direction.-#:-#: The gap between them is the honest allowance for two different serialisations of the same-#: content. The base arm's clipboard is the DOM's RENDERED TEXT; a store-based copy is markdown-#: SOURCE, so fences, emphasis and LaTeX delimiters exist in one and not the other. Measured at-#: ~0.9% on a scale fixture. Ten percent is generous against that and still refuses 2.16 by a-#: factor of twenty.+#:How much of the thread the clipboard must carry, and how much more than the thread it may carry.+#:TWO-SIDED, AND MEASURED AGAINST THE THREAD, because there are two ways to get a copy wrong.+#: The lower bound catches TRUNCATION: a windowed mount cannot select what it has not mounted,+#: so a naive copy carries only the visible fraction (0.61 of the thread on a real 100K arm).+#: The upper bound catches SUBSTITUTION, what a fix for the first turns into unwatched:+#: serialising from the message store is right, but the obvious serialiser emits reasoning,+#: tool-call arguments and tool results, none of which a user can select (2.16 on the same arm).+#: The gap between them allows for two serialisations of the same content: the base arm's+#: clipboard is the DOM's RENDERED TEXT, a store-based copy is markdown SOURCE (~0.9% on a scale+#: fixture). Ten percent is generous against that and still refuses 2.16 by a factor of twenty.MIN_CLIPBOARD_COVERAGE = 0.95MAX_CLIPBOARD_COVERAGE = 1.10@@ -175,8 +162,8 @@ def clipboard_coverage(base_row: dict, treat_row: dict) -> list[dict]:):mounted, total = _expect(row, "messages_mounted"), _expect(row, "messages_total")if not _expect(row, "clipboard_readable"):- # NOT A PASS. An unreadable clipboard is a surface that went unmeasured, and this is- # the one invariant where "we could not tell" must never look like "it was fine".+ # NOT A PASS. An unreadable clipboard is a surface that went unmeasured, and this is the one+ # invariant where 'we could not tell' must never look like 'it was fine'.out.append(_check(f"clipboard_readable:{label}",@@ -196,19 +183,14 @@ def clipboard_coverage(base_row: dict, treat_row: dict) -> list[dict]:required = True,))- # THE CHECK THAT DETECTS THE DATA LOSS, scored against THE THREAD rather than against the- # other arm -- which is what the docstring above has always said and what the code did not do.- #- # Comparing the two clipboards directly, at a 2% tolerance, asks whether two different- # serialisations of the same conversation are the same LENGTH. They are not and cannot be: the- # base arm's clipboard is the DOM's rendered text and a store-based copy is markdown source.- # A correct fix therefore fails that comparison, and the only way to make it pass is to widen- # the tolerance until it stops testing anything.- #- # The reference is the thread's own visible text, measured by the arm that has all of it in- # the DOM: on a fully mounted arm `Selection.toString()` over the whole thread IS the thread.- # If neither arm mounts everything there is no reference and the pair is not comparable, which- # is reported rather than assumed either way.+ # THE CHECK THAT DETECTS THE DATA LOSS, scored against THE THREAD rather than the other arm.+ # Comparing the two clipboards directly at 2% asks whether two different serialisations of the+ # same conversation are the same LENGTH. They cannot be, so a correct fix fails and the only+ # way to pass is to widen the tolerance until it tests nothing.+ # The reference is the thread's own visible text, measured by the arm that has all of it: on a+ # fully mounted arm `Selection.toString()` over the thread IS the thread. If neither arm mounts+ # everything there is no reference and the pair is not comparable, which is reported rather+ # than assumed.reference = _expect(base_row, "selected_chars")base_full = _expect(base_row, "mounted_fraction")if not isinstance(reference, (int, float)) or reference <= 0 or base_full != 1:@@ -242,8 +224,7 @@ def clipboard_coverage(base_row: dict, treat_row: dict) -> list[dict]:required = True,))- # Reported, never gated: on a windowed arm the selection is SUPPOSED to be short, and gating- # on it would fail the fix.+ # Reported, never gated: on a windowed arm the selection is SUPPOSED to be short.out.append(_check("selection_shrank_as_expected",@@ -295,33 +276,22 @@ def thread_survives_reopen(base_row: dict, treat_row: dict) -> list[dict]:completed = _reopen_completed(row)detail = f"the thread had {before} messages and came back with {after}"# MATCHING COUNTS ARE ONLY AN INVARIANT IF THE THREAD ACTUALLY CAME BACK.- #- # `messages_after` is `threadTotal()`, which is `aria-setsize` -- the store's DECLARATION of- # how long the conversation is, published by the very first reopened row. When the rebuild- # times out, scene/actions.py records `ran = True`, `expect_ok = False`, a null `reopen_ms`- # and the outstanding conditions, and leaves the two counts equal because both are that same- # declaration. Scored as equality it read as a held invariant, the route check passed- # because the sidebar click had worked, and the pair came out MATCH with the rebuild having- # timed out at three of eighteen messages mounted. "A declaration is not a rebuild" is the- # defect the action itself was fixed for; this is the same defect one layer up.- #+ # `messages_after` is `aria-setsize`, the store's DECLARATION of the conversation length,+ # published by the first reopened row. On a timed-out rebuild scene/actions.py records+ # `ran=True, expect_ok=False` and leaves the two counts equal because both are that+ # declaration, so equality read as a held invariant with three of eighteen messages mounted.+ # A timed-out rebuild also leaves `reopen_ms` null.+ # Read through `threadTotal()`.# NOT COMPARABLE RATHER THAN BROKEN, AND ONLY FOR THE EQUAL CASE:- #- # counts DISAGREE BROKEN, whatever the gate said. The thread came back shorter than it- # left, which is the data loss this invariant exists for, and a- # readiness failure corroborates it rather than excusing it. Routing- # this to NOT COMPARABLE would have downgraded the one finding the- # whole action is written to catch.- # counts AGREE, no- # finished rebuild NOT COMPARABLE. The comparison was not made: both numbers are the- # same declaration read off a thread that never finished building. And- # the timeout that produced it is bounded by the harness's OWN- # remaining budget (a 10 s floor, a 60 s ceiling, against the 180 s the- # cell's opening gate had), so calling it BROKEN would file a budget- # exhaustion on a shared machine as a user-visible defect of the arm.- #- # The arm's failure is not lost by this: `ran = True, expect_ok = False` already excludes- # the cell from scoring through `report/payload.py`, and the reason travels with the row.+ # counts DISAGREE: BROKEN whatever the gate said. The thread came back shorter than it left,+ # which is the data loss this exists for; routing it to NOT COMPARABLE would downgrade the one+ # finding the action is written to catch.+ # counts AGREE with no finished rebuild: NOT COMPARABLE. Both numbers are the same declaration+ # off a thread that never finished building, and the timeout is bounded by the harness's own+ # remaining budget, so BROKEN would file a budget exhaustion as a defect of the arm.+ # The arm's failure is not lost: `ran=True, expect_ok=False` already excludes the cell from+ # scoring, and the reason travels with the row.+ # The exclusion happens in `report/payload.py`.if before is None or after is None:ok: Optional[bool] = Nonerequired = False@@ -343,8 +313,8 @@ def thread_survives_reopen(base_row: dict, treat_row: dict) -> list[dict]:f"(outstanding {failed or 'unrecorded'})")out.append(_check(f"reopen_keeps_every_message:{label}", ok, detail, required = required))- # The route matters as much as the count. A row measured after a full page navigation is a- # row about a page load; see `_click_or_navigate` in scene/actions.py.+ # The route matters as much as the count: a row measured after a full page navigation is a row+ # about a page load. See `_click_or_navigate`.via = _expect(row, "reopened_via")out.append(_check(@@ -429,21 +399,15 @@ def _comparable_extents(base_row: dict, treat_row: dict) -> tuple[Any, Any, str]def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:"""scroll_after: the gesture covers the ground it commanded and no more, on both arms."""out = []- # THE PAIR'S REFERENCE EXTENT, NOT THE ARM'S OWN, and for the same reason `_drift` divides by- # the larger of the two: an estimate correction is the arm closing the gap to the REAL extent,- # so the gap is what bounds it, and the arm that is wrong is the one whose own extent is the- # worse yardstick. Taken per arm the two checks read one tolerance off two denominators, and- # disagreed inside it: extents of 10,000 and 9,050 pass `scroll_extent` at 9.5% drift while- # the 950 px correction that closes that very gap came out BROKEN against a ceiling granting- # 10% of 9,050. A false red is not free here -- it removes the cell from `readings_by_arm`,- # takes its healthy partner with it through the arm intersection, and- # `unmeasured_planned_cells` can then VOID the plan -- so the two now enforce the same- # allowance on the same quantity.- #- # It only ever loosens: `max` is never below the arm's own extent. An arm carrying NO extent- # still gets no ceiling rather than borrowing its partner's, because that would newly bound an- # arm this check has always left unbounded above, which is the one direction that could invent- # a red rather than retire one.+ # THE PAIR'S REFERENCE EXTENT, NOT THE ARM'S OWN, for the same reason `_drift` divides by the+ # larger of the two: an estimate correction closes the gap to the REAL extent, so the gap+ # bounds it. Per arm, extents of 10,000 and 9,050 pass `scroll_extent` at 9.5% drift while the+ # 950 px correction closing that gap came out BROKEN against 10% of 9,050.+ # A false red is not free: it removes the cell from `readings_by_arm`, takes its healthy partner+ # with it through the arm intersection, and `unmeasured_planned_cells` can then VOID the plan.+ # It only ever loosens: `max` is never below the arm's own extent. An arm with NO extent still+ # gets no ceiling rather than borrowing its partner's, which would newly bound an arm always+ # left unbounded above.reference = max((abs(extent)@@ -457,30 +421,22 @@ def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:commanded = _expect(row, "commanded_px")travelled = _expect(row, "travelled_px")# THE ALLOWANCE IS A FRACTION OF THE EXTENT, so it is taken on the extent. `bottom` is- # `scrollHeight - clientHeight`, and reading `EXTENT_TOLERANCE` off it is the same defect- # `_extent_of` exists to fix one check below: on a 10,000 px extent behind an 800 px- # viewport it grants 920 px where the tolerance says 1,000, so a 941 px correction -- 9.4%- # of the extent, inside the declared 10% -- came out BROKEN.+ # `scrollHeight - clientHeight`: on a 10,000 px extent behind an 800 px viewport it grants 920+ # px where the tolerance says 1,000, so a 941 px correction inside the declared 10% came out+ # BROKEN.extent, _reconstructed = _extent_of(row)# BOUNDED ABOVE AS WELL AS BELOW, and the ceiling is DERIVED rather than chosen.- #- # The lower bound is what this invariant was written for: Unsloth's intent-aware autoscroll- # snapping a programmatic move back to the bottom leaves the gesture having covered- # nothing. But the predicate was `fraction >= 0.9` and nothing above it, so an arm whose- # viewport moved TWICE as far as commanded passed exactly as 1.0 did and the pair returned- # MATCH. `travelled` sums `|scrollTop_after - target_before|`, so every pixel above- # `commanded` is the viewport being moved by something other than the gesture -- the same- # anchor instability this action exists to detect, in the other direction.- #- # WHY THIS CEILING AND NOT A NUMBER PICKED TO LOOK SYMMETRIC WITH 0.9. Overshoot has one- # legitimate source: a windowed list correcting estimated row heights, which moves the- # offset by the error in the estimate. Those errors total the error in the arm's extent,- # and this file already declares how far the extent may be wrong (`EXTENT_TOLERANCE`). So- # the gesture may exceed its command by that fraction of the arm's own extent, both terms- # read off the row. No second constant, and it scales with the rung.- #- # It degrades to no ceiling rather than to a guess: an arm carrying no extent gets the- # lower bound alone, said in the detail, because a ceiling of zero fails every correct arm.+ # The lower bound is what this was written for: intent-aware autoscroll snapping a programmatic+ # move back to the bottom leaves the gesture having covered nothing. But `fraction >= 0.9` with+ # no upper bound passed an arm whose viewport moved TWICE as far as commanded; `travelled` sums+ # |scrollTop_after - target_before|, so every pixel above `commanded` is the same anchor+ # instability in the other direction.+ # WHY THIS CEILING. Overshoot has one legitimate source, a windowed list correcting estimated+ # row heights, and those errors total the error in the arm's extent, already declared as+ # `EXTENT_TOLERANCE`. So the gesture may exceed its command by that fraction of the arm's own+ # extent, both terms read off the row: no second constant, and it scales with the rung.+ # It degrades to no ceiling rather than a guess: an arm carrying no extent gets the lower bound+ # alone, said in the detail, because a ceiling of zero fails every correct arm.ceiling: Optional[float] = Noneif (isinstance(commanded, (int, float))@@ -506,14 +462,12 @@ def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:)out.append(_check(f"scroll_travelled:{label}", ok, detail))# THE EXTENT, NOT `bottom`, AND AT THE EXTENT'S OWN ALLOWANCE. This compared `bottom` through- # `_same_number` and so through `EXACT_TOLERANCE`, 2%, while `scroll_extent` grants the same- # physical quantity 10% a few checks earlier and says why a correct virtualizer needs it. An- # arm inside the declared allowance was reported behaviourally BROKEN, and a false red is not- # free: it removes the cell from `readings_by_arm`, takes its healthy partner with it through- # the arm intersection, and `unmeasured_planned_cells` can then VOID the plan.- #- # `_same_number` is deliberately NOT widened. Its other three keys -- `selected_chars`,- # `visible_chars`, `clipboard_chars` -- are not extents and are correctly strict at 2%.+ # `_same_number` at 2% while `scroll_extent` grants the same physical quantity 10%, so an arm+ # inside the declared allowance was reported BROKEN, and a false red removes the cell, its+ # partner, and can VOID the plan.+ # The 2% is `EXACT_TOLERANCE`.+ # `_same_number` is deliberately NOT widened: its other three keys are not extents and are correctly strict at 2%.+ # They are `selected_chars`, `visible_chars` and `clipboard_chars`.b_ext, t_ext, what = _comparable_extents(base_row, treat_row)drift = _drift(b_ext, t_ext)out.append(@@ -527,9 +481,9 @@ def scroll_travelled(base_row: dict, treat_row: dict) -> list[dict]:return out-#: action -> the invariants that apply to it. An action absent from this table has no behavioural-#: invariant declared and is reported as UNCHECKED rather than as passing, on the same principle-#: that keeps NOT_COMPARABLE out of the pass column in the digest report.+#: action -> the invariants that apply to it. An action absent from this table has no declared+#: invariant and is reported as UNCHECKED rather than as passing, on the principle that keeps+#: NOT_COMPARABLE out of the pass column.INVARIANTS = {"select_all_copy": clipboard_coverage,"select_text": lambda b, t: [@@ -581,9 +535,8 @@ def compare_behaviour(base_row: Optional[dict], treat_row: Optional[dict]) -> digot = rule(base_row, treat_row)checks.extend(got if isinstance(got, list) else [got])- # A REQUIRED CHECK THAT COULD NOT BE READ VOIDS THE PAIR. See `_check`: without this, an- # action whose entire subject went unmeasured scores MATCH on the strength of the checks that- # happened to survive.+ # A REQUIRED CHECK THAT COULD NOT BE READ VOIDS THE PAIR: without this, an action whose entire+ # subject went unmeasured scores MATCH on the checks that survived.unread = [c for c in checks if c.get("required") and c["ok"] is None]if unread:return {@@ -600,13 +553,10 @@ def compare_behaviour(base_row: Optional[dict], treat_row: Optional[dict]) -> di"checks": checks,}# A PASS REQUIRES AN ACTION-SPECIFIC INVARIANT TO HAVE HELD.- #- # The scroll extent is checked on every action and it is a property of the THREAD, not of the- # action: it holds or fails identically across all eighteen. Counting an action as passing- # because the thread's scrollbar was the right length would report `model_change`,- # `image_upload` and `settings` as verified on a windowed arm when nothing whatsoever about- # them was examined. That is the same mistake `compare_rows` exists to prevent on the digest- # side, where an action that never ran used to contribute a match.+ # The scroll extent is checked on every action but is a property of the THREAD, so it holds or+ # fails identically across all eighteen. Counting an action as passing on it would report+ # `model_change`, `image_upload` and `settings` as verified when nothing about them was+ # examined.specific = [c for c in checks if c["ok"] is not None and c["invariant"] != "scroll_extent"]if not specific:return {diff --git a/tests/studio/studiobench/analysis/bridge_build.py b/tests/studio/studiobench/analysis/bridge_build.pyindex f14fed7d030..f82697b4aa8 100644--- a/tests/studio/studiobench/analysis/bridge_build.py+++ b/tests/studio/studiobench/analysis/bridge_build.py@@ -29,9 +29,8 @@ from typing import Any, Callable, Sequencefrom . import CellFailurefrom .symbols import FAILED, Bridge, build_bridge-# A `RungRunner` puts the page into the state for one rung and returns when the-# work for that rung is finished. It must be deterministic: the same rung must-# do the same amount of work every time, or the count vectors are noise.+# A `RungRunner` puts the page into the state for one rung and returns when that rung's work is+# finished. It must be deterministic, or the count vectors are noise.RungRunner = Callable[[Any, str], None]DEFAULT_BRIDGE_RUNGS: tuple[str, ...] = ("bridge_s", "bridge_m", "bridge_l")@@ -177,38 +176,23 @@ PROFILER_RECORDER_JS = """"""-# ═══════════════════════════════════════════════════════════════════════════# Build provenance: is the thing under measurement the thing we think it is?-# ═══════════════════════════════════════════════════════════════════════════-#-# The worst output this layer can produce is a real number from the wrong-# bundle under the right label. A dev server inflates React's own work several-# fold and would MANUFACTURE the symptom we are hunting; a stale shipping dist-# left in the output directory would silently measure code without the-# profiling renderer. Both look like a clean run.-#-# Three independent checks, because each catches something the others cannot:-#-# 1. `/@vite/client` must not answer 200. That file exists only when a Vite-# dev server is serving, and it is a fact about the SERVER.-# 2. React must report `bundleType: 0` in the same renderer entry as-# `rendererPackageName: "react-dom"`. That is a fact about the RENDERER-# THAT ACTUALLY LOADED, which is the only thing that matters and the only-# one of the three that a build-config mistake cannot fake.-# 3. `__STUDIOBENCH_ATTRIBUTION_BUILD__` must be true. That is a fact about-# WHICH dist is mounted, and it is what catches a stale shipping build-# sitting in the directory passed to `--frontend`.-#-# DO NOT ADD A `jsxDEV` GREP. It false-positives: `hast-util-to-jsx-runtime`,-# which Streamdown pulls in, references the dev JSX runtime in shipping code, so-# a production bundle contains the string and the check condemns a correct-# build.-#-# `bundleType` is read through the DevTools global hook. React only injects into-# that hook if it already exists when the renderer initialises, so the stub must-# be installed with `add_init_script` BEFORE the first `goto`, exactly like the-# JWT seeding in `studio_test_kit.auth`. Installed after, the hook is empty and-# the check reads as "React is missing" on a perfectly good page.+# The worst output this layer can produce is a real number from the wrong bundle under the right+# label: a dev server inflates React's own work several fold and would MANUFACTURE the symptom,+# and a stale shipping dist would silently measure code without the profiling renderer. Both look+# like a clean run.+# Three independent checks: `/@vite/client` must not answer 200 (a fact about the SERVER); React+# must report `bundleType: 0` in the same renderer entry as `rendererPackageName: "react-dom"` (a+# fact about the RENDERER THAT ACTUALLY LOADED, the only one a build-config mistake cannot fake);+# and `__STUDIOBENCH_ATTRIBUTION_BUILD__` must be true (which dist is mounted, catching a stale+# shipping build in the `--frontend` directory).+# DO NOT ADD A `jsxDEV` GREP: `hast-util-to-jsx-runtime`, which Streamdown pulls in, references+# the dev JSX runtime in shipping code, so a production bundle contains the string and the check+# condemns a correct build.+# `bundleType` is read through the DevTools global hook, and React only injects into it if it+# already exists when the renderer initialises, so the stub must be installed with+# `add_init_script` BEFORE the first `goto`. Installed after, the hook is empty and the check+# reads as 'React is missing' on a good page.DEVTOOLS_HOOK_STUB_JS = """(() => {@@ -241,14 +225,10 @@ DEVTOOLS_HOOK_STUB_JS = """})();"""-# React's `bundleType`: 0 is a production build, 1 is development. Confirmed on-# a real pair: the dev server reported 1 and the profiling build reported 0.-#-# NOTE THAT 0 DOES NOT MEAN "profiling renderer is loaded". React's profiling-# build is a production build, so it also reports 0, and `bundleType` alone-# cannot tell the two apart. That is why `assert_profiling_build_loaded` is a-# separate check on `onRender` actually firing, and why neither one substitutes-# for the other.+# React's `bundleType`: 0 is production, 1 is development. Confirmed on a real pair. NOTE THAT 0+# DOES NOT MEAN 'profiling renderer is loaded' (React's profiling build is a production build and+# also reports 0) which is why `assert_profiling_build_loaded` separately checks `onRender`+# actually firing.BUNDLE_TYPE_PRODUCTION = 0BUNDLE_TYPE_DEVELOPMENT = 1@@ -270,10 +250,9 @@ def assert_production_bundle(page: Any, *, base_url: str | None = None) -> dict["or DEVTOOLS_HOOK_STUB_JS was installed after the first navigation instead of ""through add_init_script before it, in which case React had nothing to inject into.",)- # Match on the SAME entry, not across entries. A page can host more than one- # renderer (react-dom plus react-reconciler in a canvas library, say), and- # checking `bundleType` from one against `rendererPackageName` from another- # is how a development react-dom passes behind a production sibling.+ # Match on the SAME entry, not across entries: a page can host more than one renderer, and+ # checking `bundleType` from one against `rendererPackageName` from another is how a development+ # react-dom passes behind a production sibling.dom = [r for r in renderers if str(r.get("rendererPackageName") or "") == "react-dom"]if not dom:raise CellFailure(diff --git a/tests/studio/studiobench/analysis/classify.py b/tests/studio/studiobench/analysis/classify.pyindex 06bd975540f..940c99d4ed4 100644--- a/tests/studio/studiobench/analysis/classify.py+++ b/tests/studio/studiobench/analysis/classify.py@@ -57,11 +57,9 @@ RAF = "raf"IDLE = "idle"GC = "gc"BROWSER_INTERNAL = "browser_internal"-# Browser-to-renderer IPC that ran page script. In a harnessed run this is-# dominated by the devtools pipe delivering `Runtime.evaluate`, which is to say-# it is OUR OWN cost and it belongs in a column with its own name so that it can-# be watched for growth with the treatment, not folded into a residual and not-# hidden inside `browser_internal`.+# Browser-to-renderer IPC that ran page script. In a harnessed run this is dominated by the+# devtools pipe delivering `Runtime.evaluate`, which is OUR OWN cost and belongs in a column with+# its own name so it can be watched for growth with the treatment.AGENT_IPC = "agent_ipc"UNCLASSIFIED = "unclassified"@@ -78,12 +76,12 @@ ORIGINS = (UNCLASSIFIED,)-# Origins that are the harness observing, not the app working. Reported, never-# subtracted: a correction you cannot see is a correction you cannot check.+# Origins that are the harness observing, not the app working. Reported, never subtracted: a+# correction you cannot see is a correction you cannot check.HARNESS_ORIGINS = (AGENT_IPC,)-# Above this share of main-thread task time in the unclassified bucket, the cell-# is not quotable. The point of the tool is that we can name things.+# Above this share of main-thread task time in the unclassified bucket, the cell is not quotable.+# The point of the tool is that we can name things.UNCLASSIFIED_FAIL_PCT = 2.0_JS_EVENTS = frozenset(@@ -147,8 +145,8 @@ _INPUT_EVENT_TYPES = frozenset(})-# Network-ish streaming: an SSE frame arrives as a `message` event on an-# EventSource, and a fetch-based stream as a resource event.+# Network-ish streaming: an SSE frame arrives as a `message` event on an EventSource, and a+# fetch-based stream as a resource event._NETWORK_EVENT_TYPES = frozenset({"message", "readystatechange", "load", "error", "open"})_POSTED_FROM_RULES: tuple[tuple[str, str, str], ...] = (@@ -164,8 +162,8 @@ _POSTED_FROM_RULES: tuple[tuple[str, str, str], ...] = (("v8/src/tasks/", "", GC),)-# Tasks the browser posts to itself. Only ever applied to a task that ran NO-# page script, so this can never swallow a JS bottleneck.+# Tasks the browser posts to itself. Only ever applied to a task that ran NO page script, so this+# can never swallow a JS bottleneck._BROWSER_INTERNAL_FILES = ("web_frame_widget_impl.cc","main_thread_scheduler_impl.cc",@@ -189,11 +187,9 @@ _BROWSER_INTERNAL_FILES = (_IDLE_MARKERS = ("DoIdleWork", "RunIdleTask", "IdleTask", "ScheduleIdleTask")# Blink `TaskType` enum names as they appear in-# `args.renderer_main_thread_task_execution.task_type`. Matched by SUBSTRING on-# the part after `TASK_TYPE_`, so that both-# `TASK_TYPE_JAVASCRIPT_TIMER_DELAYED_HIGH_NESTING` and-# `TASK_TYPE_JAVASCRIPT_TIMER_IMMEDIATE` land on `timer` without enumerating-# every nesting variant.+# `args.renderer_main_thread_task_execution.task_type`. Matched by SUBSTRING on the part after+# `TASK_TYPE_`, so both `..._TIMER_DELAYED_HIGH_NESTING` and `..._TIMER_IMMEDIATE` land on+# `timer`._TASK_TYPE_RULES: tuple[tuple[str, str], ...] = (("JAVASCRIPT_TIMER", TIMER),("POSTED_MESSAGE", MESSAGE_CHANNEL),@@ -367,15 +363,10 @@ def classify_task(task: Task) -> ClassifiedTask:queue_name = queue_name,)- # 0. Blink's own label, when the `scheduler` category recorded one. This- # outranks everything below it because it is the scheduler stating what- # it dispatched rather than us reconstructing it. Two exceptions are- # handled first, both cases where the label is true but not what a reader- # wants attributed:- # - a V8 task queue that ran no JS is GC or compilation bookkeeping,- # but one that DID run JS is a real JS task and must not be filed- # under GC;- # - a compositor-queue task with a nested input dispatch is input.+ # 0. Blink's own label, when the `scheduler` category recorded one. It outranks everything below+ # because it is the scheduler stating what it dispatched. Two exceptions are handled first: a V8+ # task queue that ran no JS is GC or compilation bookkeeping while one that DID run JS is a real+ # JS task, and a compositor-queue task with a nested input dispatch is input.if task_type:if types & _INPUT_EVENT_TYPES:return done(@@ -388,16 +379,14 @@ def classify_task(task: Task) -> ClassifiedTask:continuereturn done(origin, f"task_type:{task_type}")- # 1. Input first: a keystroke that also fires a timer is still a keystroke,- # and input latency is the metric a user actually feels.+ # 1. Input first: a keystroke that also fires a timer is still a keystroke, and input latency is what a user feels.if types & _INPUT_EVENT_TYPES:return done(INPUT, f"EventDispatch:{sorted(types & _INPUT_EVENT_TYPES)[0]}")if "main_thread_event_queue.cc" in src_file:return done(INPUT, "posted_from:main_thread_event_queue")- # 2. GC before everything else that could contain it, because a major GC- # posted from the incremental marking job is not a timer task even when- # it happens to run inside one.+ # 2. GC before everything else that could contain it, because a major GC posted from the+ # incremental marking job is not a timer task even when it runs inside one.gc_names = {nfor n in names@@ -422,16 +411,16 @@ def classify_task(task: Task) -> ClassifiedTask:if "dom_timer.cc" in src_file:return done(TIMER, "posted_from:dom_timer")- # 6. Network / SSE. Checked before message-channel because both arrive over- # a mojo pipe and only the resource events tell them apart.+ # 6. Network / SSE. Checked before message-channel because both arrive over a mojo pipe and only+ # the resource events tell them apart.if names & _RESOURCE_EVENTS:return done(NETWORK, f"nested:{sorted(names & _RESOURCE_EVENTS)[0]}")if types & _NETWORK_EVENT_TYPES and "connector.cc" in src_file:return done(NETWORK, f"EventDispatch:{sorted(types & _NETWORK_EVENT_TYPES)[0]}")- # 7. Message channel, that is, the React scheduler. Blink dispatches a- # `MessagePort` message as a mojo connector task, so the discriminator is- # that page script ran and nothing network-shaped happened.+ # 7. Message channel, i.e. the React scheduler. Blink dispatches a `MessagePort` message as a+ # mojo connector task, so the discriminator is that page script ran and nothing network-shaped+ # happened.if ran_js and ("connector.cc" in src_fileor "message_port" in src_file@@ -439,17 +428,16 @@ def classify_task(task: Task) -> ClassifiedTask:):return done(MESSAGE_CHANNEL, "mojo_pipe_running_page_script")- # 8. The devtools / browser IPC channel running page script. This is what- # `Runtime.evaluate` and `Page.*` look like from inside the renderer, so- # it is measurement cost. Named, so it can be watched for correlation- # with the treatment; never subtracted.+ # 8. The devtools / browser IPC channel running page script: what `Runtime.evaluate` and `Page.*`+ # look like from inside the renderer, so it is measurement cost. Named so it can be watched for+ # correlation with the treatment; never subtracted.if ran_js and any(f in src_file for f in ("ipc_mojo_bootstrap.cc", "simple_watcher.cc", "mojo_bootstrap")):return done(AGENT_IPC, f"posted_from:{src_file.rsplit('/', 1)[-1]}")- # 9. Browser bookkeeping, and ONLY when no page script ran inside. This- # guard is what stops this class becoming the residual under a new name.+ # 9. Browser bookkeeping, and ONLY when no page script ran inside. This guard is what stops the+ # class becoming the residual under a new name.if not ran_js and any(f in src_file for f in _BROWSER_INTERNAL_FILES):return done(BROWSER_INTERNAL, f"posted_from:{src_file.rsplit('/', 1)[-1]}")@@ -457,9 +445,8 @@ def classify_task(task: Task) -> ClassifiedTask:if needle in src_file and (not func_needle or func_needle in src_func):return done(origin, f"posted_from:{needle}")- # 10. Last resort before giving up: the queue the task sat on. Weaker than- # the task type, because a queue carries a mix, but still read from the- # scheduler rather than guessed.+ # 10. Last resort before giving up: the queue the task sat on. Weaker than the task type, because+ # a queue carries a mix, but still read from the scheduler rather than guessed.for needle, origin in _QUEUE_NAME_RULES:if needle == queue_name:if origin == GC and ran_js:diff --git a/tests/studio/studiobench/analysis/cpuprofile.py b/tests/studio/studiobench/analysis/cpuprofile.pyindex d7d1054fb25..0c64b6f9765 100644--- a/tests/studio/studiobench/analysis/cpuprofile.py+++ b/tests/studio/studiobench/analysis/cpuprofile.py@@ -37,19 +37,16 @@ from typing import Any, Iterable, Sequencefrom . import CellFailurefrom .traceparse import Trace-# V8 synthetic frames. They are real samples and their time is real, but they-# are not call frames anyone can go and fix, so they are labelled and excluded-# from frame ranking rather than silently dropped from totals.+# V8 synthetic frames. Their time is real but they are not call frames anyone can fix, so they are+# labelled and excluded from frame ranking rather than dropped from totals.SYNTHETIC_FRAMES = frozenset({"(root)", "(program)", "(idle)", "(garbage collector)"})-# How far the summed sample deltas may drift from the wall span of the chunks-# before the cell is unusable.+# How far the summed sample deltas may drift from the wall span of the chunks before the cell is unusable.DELTA_WALL_TOLERANCE = 0.02-# Below this many non-synthetic samples inside a window set, a leaf ranking is-# not a ranking. At a ~150 us sampling interval a 40 us task contributes at most-# one sample, so a set of short tasks can produce a confident-looking table-# built on five data points. Reported as `underpowered`, never hidden.+# Below this many non-synthetic samples inside a window set, a leaf ranking is not a ranking: at a+# ~150 us interval a 40 us task contributes at most one sample, so short tasks can produce a+# confident-looking table built on five data points. Reported as `underpowered`, never hidden.MIN_JS_SAMPLES_FOR_RANKING = 100@@ -104,17 +101,14 @@ class CpuProfile:chunk_ts_first: int = 0chunk_ts_last: int = 0negative_deltas: int = 0- # `Profile.args.data.startTime`, kept only so the clock skew against `ts`- # can be reported. Never used as an anchor.+ # `Profile.args.data.startTime`, kept only so the clock skew against `ts` can be reported. Never used as an anchor.declared_start_time: int = 0source: str = ""- # `sampleTraceId` -> node id, accumulated from- # `ProfileChunk.args.data.cpuProfile.trace_ids`. This is an EXACT stack, not- # a sampled one: when the `disabled-by-default-devtools.timeline.stack`- # category is on, Blink calls `v8::CpuProfiler::CollectSample` with a tagged- # id at specific instrumentation points, so the stack recorded for that id- # is the real stack at that instant rather than whatever the 100 us sampler- # happened to catch.+ # `sampleTraceId` -> node id, from `ProfileChunk.args.data.cpuProfile.trace_ids`. An EXACT stack,+ # not a sampled one: with `disabled-by-default-devtools.timeline.stack` on, Blink calls+ # `v8::CpuProfiler::CollectSample` with a tagged id at specific instrumentation points, so the+ # stack is the real one at that instant.+ # Not whatever the 100 us sampler happened to catch.trace_ids: dict[str, int] = field(default_factory = dict)@property@@ -277,13 +271,11 @@ def parse_cpu_profiles(trace: Trace) -> dict[str, CpuProfile]:profiles[key] = CpuProfile(pid = pid,tid = int(e.get("tid", 0)),- # ANCHOR ON `ts`, NOT ON `args.data.startTime`. V8 writes- # `startTime` straight from `base::TimeTicks` without converting it- # into the trace's clock domain, and both V8's own source comment- # and the DevTools frontend say to use the event timestamp instead.- # The two agree to within a few microseconds on a healthy capture,- # but the drift is unbounded in principle, and it would show up as- # every sample being attributed to the neighbouring task.+ # ANCHOR ON `ts`, NOT ON `args.data.startTime`. V8 writes `startTime` straight from+ # `base::TimeTicks` without converting into the trace's clock domain, and both V8's source+ # comment and the DevTools frontend say to use the event timestamp. They agree to within+ # microseconds on a healthy capture, but the drift is unbounded in principle and would attribute+ # every sample to the neighbouring task.start_time = int(e.get("ts", data.get("startTime", 0))),declared_start_time = int(data.get("startTime", 0)),source = str(data.get("source", "")),@@ -301,8 +293,8 @@ def parse_cpu_profiles(trace: Trace) -> dict[str, CpuProfile]:key = f"{pid}:{e.get('id')}"prof = profiles.get(key)if prof is None:- # A chunk with no matching Profile event cannot be anchored to a- # start time, so its sample timestamps would be meaningless.+ # A chunk with no matching Profile event cannot be anchored to a start time, so its sample+ # timestamps would be meaningless.continuedata = (e.get("args") or {}).get("data") or {}cpu = data.get("cpuProfile") or {}@@ -313,8 +305,8 @@ def parse_cpu_profiles(trace: Trace) -> dict[str, CpuProfile]:parent = raw.get("parent")if parent is not None:prof.parents[nid] = int(parent)- # Some producers give `children` instead of `parent`; honour both so- # ancestry does not silently collapse to a forest of roots.+ # Some producers give `children` instead of `parent`; honour both so ancestry does not silently+ # collapse to a forest of roots.for child in raw.get("children") or ():prof.parents.setdefault(int(child), nid)diff --git a/tests/studio/studiobench/analysis/oracles.py b/tests/studio/studiobench/analysis/oracles.pyindex 1b0afa43335..67f9919f931 100644--- a/tests/studio/studiobench/analysis/oracles.py+++ b/tests/studio/studiobench/analysis/oracles.py@@ -38,9 +38,8 @@ NEAR_MISS = "near_miss"UNEXPLAINED = "unexplained_hot_frame"NOT_MEASURED = "not_measured"-# Integer ratios worth naming when a count is off by a clean factor. Each has a-# specific mechanical meaning, so reporting the ratio hands over a hypothesis-# rather than a mystery.+# Integer ratios worth naming when a count is off by a clean factor. Each has a specific mechanical+# meaning, so reporting the ratio hands over a hypothesis rather than a mystery._KNOWN_RATIOS: dict[Fraction, str] = {Fraction(2, 1@@ -276,35 +275,25 @@ def predicted_next_rung(return quantity_fn(next_structural_input).value-# ═══════════════════════════════════════════════════════════════════════════# M2 and M3: oracle shapes over PAGE-SIDE counters-# ═══════════════════════════════════════════════════════════════════════════-#-# M1 is settled with a call count from precise coverage, because the mechanism-# lives inside react-dom where we can count invocations. M2 and M3 do not work-# that way: their cost is in how much text is rescanned and how much layout is-# forced, and neither is an invocation count. Those must be counted IN THE PAGE.-#-# Layer 3 owns `instruments/layoutcost.js` and already counts `scrollHeight`-# reads, `scrollTop` writes and MutationObserver callbacks. This section does-# NOT duplicate that file. It fixes the KEY NAMES that file should emit and-# supplies the oracles that consume them, so that the counting and the-# interpretation are owned separately and neither has to guess at the other.-#-# ONE HONEST DIFFERENCE, stated plainly rather than buried. The M1 oracle is an-# EXACT INTEGER MATCH and it either holds or it does not. The M2 oracle is a-# REGIME TEST: SSE deltas do not arrive in equal sizes, so the quadratic-# prediction is exact only for a uniform stream and must be compared as a ratio-# with a refusal band. That is weaker evidence and it is labelled as such-# everywhere it is emitted. Do not let a regime verdict sit in a table next to a-# naming as though they were the same thing.+# M1 is settled with a call count from precise coverage, because the mechanism lives inside+# react-dom where invocations can be counted. M2 and M3 do not work that way: their cost is in how+# much text is rescanned and how much layout is forced, and those must be counted IN THE PAGE.+# Layer 3 owns `instruments/layoutcost.js` and already counts `scrollHeight` reads, `scrollTop`+# writes and MutationObserver callbacks. This section does NOT duplicate it: it fixes the KEY+# NAMES that file should emit and supplies the oracles that consume them, so counting and+# interpretation are owned separately.+# ONE HONEST DIFFERENCE: the M1 oracle is an EXACT INTEGER MATCH that either holds or does not,+# while the M2 oracle is a REGIME TEST, since SSE deltas do not arrive in equal sizes, so the+# quadratic prediction is exact only for a uniform stream and must be compared as a ratio with a+# refusal band. That is weaker evidence and is labelled as such everywhere it is emitted.REGIME_QUADRATIC = "cumulative_reparse_quadratic"REGIME_LINEAR = "incremental_parse_linear"REGIME_UNDECIDED = "undecided"-# The counters Layer 3's page-side instrument should emit, per streamed reply.-# Every one is an integer; none is a duration.+# The counters Layer 3's page-side instrument should emit, per streamed reply. Every one is an+# integer; none is a duration.PAGE_COUNTER_CONTRACT: dict[str, dict[str, str]] = {"m2_reparse": {"parse_calls": "invocations of parseAssistantContent during the reply",@@ -394,8 +383,7 @@ def reparse_regime("refusal band. This reply is too short to distinguish the regimes; use a longer rung.")return out- # Closer in log space wins; the ratio is reported either way so a reader can- # see how decisive the call was.+ # Closer in log space wins; the ratio is reported either way so a reader can see how decisive the call was.r_lin = chars_rescanned / linr_quad = chars_rescanned / quadout["ratio_to_linear"] = round(r_lin, 3)diff --git a/tests/studio/studiobench/analysis/parity.py b/tests/studio/studiobench/analysis/parity.pyindex 75b4b100894..d53023aefbd 100644--- a/tests/studio/studiobench/analysis/parity.py+++ b/tests/studio/studiobench/analysis/parity.py@@ -25,75 +25,33 @@ from __future__ import annotationsimport collectionsfrom typing import Any, Iterable, Optional-# What a comparison concluded. Kept as plain strings so a payload row carries them verbatim.-#: WHAT "MATCH" ACTUALLY CLAIMS, because the name promises more than the instrument delivers.-#:-#: Everything in this module is computed from `scene/parity.js`, whose structural digest walks the-#: THREAD and nothing else. It is sidebar-blind and layout-blind by construction: a change confined-#: to the sidebar, or one that alters computed geometry without altering thread structure, passes-#: it while being entirely invisible to it. That is not a theory. A concurrent campaign measuring-#: the sidebar drag found the shipped thread digest returned 0 of 34 differing pairs on a real,-#: visible change -- and its null control returned 0 of 34 as well, so the instrument was not-#: discriminating in either direction. Three purpose-built captures (sidebar-inclusive structure,-#: sidebar inline style, custom-property reach) found the same change 34 of 34, with the null at-#: zero in every category.-#:-#: So MATCH means NO THREAD-STRUCTURE CHANGE WAS DETECTED. It does not mean the UI is unchanged.-#: Not covered, and not detectable here at all:-#:-#: the sidebar outside the digest root-#: computed layout / geometry positions and sizes are never read-#: CSS custom properties never read-#: stylesheet changes only insofar as they alter `display`, `visibility` or-#: `pointer-events` on the <=64 elements the bounded style probe-#: reaches, which is reported separately and as an advisory-#:-#: Anything relying on a stronger reading than that needs its own capture.+#: WHAT "MATCH" ACTUALLY CLAIMS: everything here is computed from `scene/parity.js`, whose+#: structural digest walks the THREAD only, so it is sidebar-blind and layout-blind by+#: construction. Measured: on a real visible sidebar-drag change the thread digest found 0 of 34+#: differing pairs, while three purpose-built captures found it 34 of 34. So MATCH means NO+#: THREAD-STRUCTURE CHANGE WAS DETECTED, not that the UI is unchanged.MATCH = "match"DIFFER = "differ"NOT_COMPARABLE = "not_comparable"NOT_EXERCISED = "not_exercised"-#: THE FOURTH OUTCOME, and it is not a softer NOT_COMPARABLE.-#:-#: NOT_COMPARABLE means the reading failed or the two readings are of different things by-#: accident. NOT_APPLICABLE means the question itself is wrong for this pair: the structural-#: digest asks "is the same DOM on screen", and an arm whose entire purpose is to put less DOM on-#: screen answers "no" by construction. Reporting that as a UI difference would be true and-#: useless -- eighteen red rows, none of them a finding, and the real behavioural questions-#: drowned underneath them.-#:-#: It is kept distinct from NOT_COMPARABLE so a reader can tell "we could not measure this" from-#: "this measurement does not apply here", and so `derive_unstable` counts neither as evidence.+#: THE FOURTH OUTCOME, and not a softer NOT_COMPARABLE: that one means the reading failed, while+#: NOT_APPLICABLE means the question is wrong for this pair -- the digest asks "is the same DOM on+#: screen" and an arm whose purpose is to put less DOM on screen answers "no" by construction.+#: Kept distinct so `derive_unstable` counts neither as evidence.NOT_APPLICABLE = "not_applicable"-#: A KEY ON A NOT_COMPARABLE RESULT, not a fifth verdict.-#:-#: `compare` has one refusal that ALSO carries a complete positive reading: the scaffold, every-#: overlay, the mounted message set and every message not being written at capture time all-#: agreed, and the only leftover is the subtree of a reply that was mid-tail. It stays a refusal-#: because a half-arrived digest names a point in a stream rather than a rendering, and it must-#: never be a pass on the A/B result side: a treatment build whose live message renders wrongly-#: sits in exactly that shape, and `structural_report` files it under `blind`.-#:-#: THE VERDICT DOES NOT MOVE; only `derive_unstable` reads this. The result asks "did this build-#: change the UI", the null asks "does this action differ against ITSELF", and for the second-#: question a pair where everything readable agreed is an observation of non-difference rather-#: than a blank. Calling it blank is what left `keystroke` and `send_turn` permanently undecided-#: at 100K on a runner where the in-flight tail landed 24 characters apart.-#:-#: AND IT CAN ONLY NARROW THE EXCUSE SET: the observation is a NON-difference, so `differed`-#: cannot grow and nothing can be marked unstable on its strength. The worst case is calling an-#: action stable that nothing measured as differing, which costs a loud false alarm on the-#: result and never a silenced regression.+#: A KEY ON A NOT_COMPARABLE RESULT, not a fifth verdict. `compare` has one refusal that also+#: carries a complete positive reading: everything agreed except the subtree of a reply that was+#: mid-tail. It stays a refusal, because a half-arrived digest names a point in a stream. THE+#: VERDICT DOES NOT MOVE; only `derive_unstable` reads this, since for "does this action differ+#: against ITSELF" a pair where everything readable agreed is an observation of non-difference.+#: It can only NARROW the excuse set, so the worst case is a loud false alarm.SETTLED_MATCH = "settled_match"# Actions whose rendered result legitimately differs between two runs of the SAME build, so a-# digest mismatch there carries no information about the pull request under test.-#-# A MECHANISM PER ENTRY, not a hunch, and the value is the mechanism rather than a comment beside-# it so that a test can require one. An action silenced without a stated reason is a hole somebody-# punched in the instrument and nobody can audit afterwards; the null control's empirically-# derived set (`derive_unstable`) is the cross-check that each of these earns its place.+# digest mismatch there says nothing about the pull request. A MECHANISM PER ENTRY, as the value+# rather than a comment beside it so a test can require one: an action silenced without a stated+# reason is an unauditable hole, and `derive_unstable` cross-checks that each earns its place.UNSTABLE_ACTIONS: dict[str, str] = {"stop_generation": "stops a live stream, so how many characters arrived before the stop is a race with the ""network and differs run to run on one build. NOT reproduced by the 100K fast-tier null "@@ -106,8 +64,8 @@ UNSTABLE_ACTIONS: dict[str, str] = {"scroll_after": "same gesture against a settled thread; the resting offset still depends on the observer. ""NOT reproduced by the 100K fast-tier null control (0 of 4); kept on the mechanism alone",# The four below were NOT in the hand-written set and were found by the null control, each- # scoring as a hard UI-change signal on a build compared with itself. The mechanism for each- # was read out of the payload, not guessed.+ # scoring as a hard UI-change signal on a build compared with itself; the mechanism for each was+ # read out of the payload."keystroke": "types characters into the composer over wall clock, and the composer's text is in the ""DOM, so how many keystrokes had landed by the capture deadline is a race. Measured 4 of ""4 differing, by 8 signature characters, with assistant_chars identical",@@ -124,24 +82,14 @@ UNSTABLE_ACTIONS: dict[str, str] = {}-# WHOSE ABILITY TO RUN IS A RACE, which is a different claim from whose DIGEST varies and needs-# its own list. Every entry in UNSTABLE_ACTIONS above describes what makes the CAPTURE move ---# how many characters had arrived, where the scroll came to rest, whether the send button had-# re-enabled. Not one of them says the action sometimes cannot be performed at all. Using that-# list to excuse one-arm-only EXECUTION exempted nine of the sixteen scheduled actions from the-# one regression shape that leaves no digest to differ, on the strength of a measurement about-# something else.-#-# `slot_missed` already covers the runner arriving late, so what is left here is the narrow case-# of an action that legitimately cannot run because the stream it needs is not there. Read out of-# each action's own `not_run` reasons in `scene/actions.py` rather than assumed:-# `action -> (why, the not_run reasons that qualify)`. KEYED ON THE REASON, not just the action,-# because each of these has non-racy failure paths too: send_turn also returns not_run for "no-# composer on the page" (scene/actions.py:1016) and stop_generation for "the stop button is not-# present" (scene/actions.py:501). A treatment build that REMOVES either control records exactly-# the one-arm-only regression this instrument exists to catch, and keyed by name alone the-# exemption swallowed it. The mechanism was always written down here; it just was not what the-# code matched on.+# WHOSE ABILITY TO RUN IS A RACE, a different claim from whose DIGEST varies. Every+# UNSTABLE_ACTIONS entry describes what makes the CAPTURE move, so using that list to excuse+# one-arm-only EXECUTION exempted nine of sixteen actions from the one regression shape that+# leaves no digest to differ. `slot_missed` already covers the runner arriving late, so what is+# left is an action that cannot run because the stream it needs is not there. KEYED ON THE+# REASON, because each has non-racy paths too, and a treatment that REMOVES a control records+# exactly the regression this exists to catch.+# `send_turn` also returns not_run for "no present" (scene/actions.py:501).RACY_EXECUTION: dict[str, tuple[str, tuple[str, ...]]] = {"stop_generation": ("needs a live stream to stop, and returns not_run when nothing was generating and a new "@@ -173,11 +121,9 @@ def racy_execution(action: str, reason: str) -> bool:return any(marker in (reason or "") for marker in entry[1])-# The six that are NOT here, and why, since dropping an exemption needs as much justification as-# granting one. `composer_fill`, `keystroke`, `copy_markdown`, `message_menu` and `select_text`-# fail to run only when the control they need is absent or unresponsive -- "no composer on the-# page", "no Copy button on the last assistant message" -- and that IS the build. `scroll_after`-# has no `not_run` path at all, so a `ran: false` for it cannot be a race under any reading.+# The six that are NOT here: `composer_fill`, `keystroke`, `copy_markdown`, `message_menu` and+# `select_text` fail to run only when the control they need is absent or unresponsive, and that+# IS the build; `scroll_after` has no `not_run` path at all.def _messages(capture: dict) -> dict[int, dict]:@@ -331,9 +277,9 @@ def comparability(base: Optional[dict], treat: Optional[dict]) -> Optional[str]:f"{label} could not be captured: " f"{side.get('reason') or 'no reason recorded'}")assert base is not None and treat is not None- # `root_kind` is absent in captures taken before it was recorded. Missing on BOTH sides is an- # old payload and is allowed through; missing on one side means the two runs were produced by- # two different versions of the instrument, which is not a comparison of two builds.+ # `root_kind` is absent in captures taken before it was recorded. Missing on BOTH sides is an old+ # payload and is allowed through; missing on one side means two different versions of the+ # instrument, which is not a comparison of two builds.bk, tk = base.get("root_kind"), treat.get("root_kind")if (bk is None) != (tk is None):return (@@ -399,9 +345,8 @@ def localise(bm, tm = _messages(base), _messages(treat)for i in sorted(set(bm) | set(tm)):if i in skip and i in bm and i in tm:- # In flight on BOTH sides of the comparison. A message present on one arm only is a- # different statement -- one arm is not rendering a message the other is -- and is- # reported below whatever its status.+ # In flight on BOTH sides of the comparison: a message present on one arm only is a different+ # statement and is reported below whatever its status.continueif i not in bm:moved.append(f"msg{i}({tm[i].get('role', '?')}):only treatment")@@ -415,9 +360,8 @@ def localise(moved.extend(overlays_moved(base, treat))if not moved:- # The whole-thread digest moved but no message and no overlay did. That is the thread- # scaffolding itself -- the viewport, the composer, the empty-state -- and saying so is- # more useful than an empty list, which reads as "nothing differs".+ # The whole-thread digest moved but no message and no overlay did, so the difference is the+ # thread scaffolding itself; saying so beats an empty list that reads as "nothing differs".bc, tc = _scaffold(base)[1], _scaffold(treat)[1]moved.append(f"thread scaffolding outside any message ({bc}->{tc}c)")return moved@@ -437,13 +381,10 @@ def compare_styles(base: dict, treat: dict) -> tuple[str, str]:f"the probe hit its element cap "f"(base={bs.get('elements')}, treatment={ts.get('elements')})")- # A POSITIVE CONTROL ON THE SCAN ITSELF. Two probes that matched no elements have equal- # counts and equal digests -- both are the hash of an empty string -- so a probe that scanned- # NOTHING reports MATCH, which is the strongest verdict this function can return and is- # supported by no observation whatsoever. That is not hypothetical: the selector list is- # written against Unsloth's markup, and a class rename anywhere in it silently empties the- # scan. A DOM or CSSOM scan that can return zero has to be able to tell "I looked and they- # agree" from "I did not look", and this one could not.+ # A POSITIVE CONTROL ON THE SCAN ITSELF: two probes that matched no elements have equal counts+ # and equal digests (both the hash of an empty string), so a probe that scanned NOTHING reports+ # MATCH on no observation whatsoever. The selector list is written against Unsloth's markup, so a+ # class rename anywhere in it silently empties the scan.if not bs.get("elements") or not ts.get("elements"):return NOT_COMPARABLE, (f"the style probe matched no elements (base={bs.get('elements')}, "@@ -451,28 +392,16 @@ def compare_styles(base: dict, treat: dict) -> tuple[str, str]:"selector list is written against Unsloth's markup and does not survive a rename; ""this is a probe that needs fixing, not two arms that agree")- # THE RUN-STATE CONTROL IS IN THE SELECTOR LIST, so this reading straddles the same composer- # swap the structural refusals below are about, and it has to be elided for the same reason.- # `STYLE_SELECTORS` (scene/parity.js) names `button[aria-label="Send message"]` and- # `button[aria-label="Stop generating"]`, and the signature carries the SELECTOR THAT MATCHED- # (`parts.push(sel + "#" + n)`) -- so an arm still generating and a settled arm walk two- # different entries for the one composer button. Measured on two byte-identical fixture threads- # differing only in that control: Send against Stop holds all three properties at- # `inline-block` / `visible` / `auto` on both arms and still moves the digest, and Send against- # either queue control -- neither of which is in the selector list at all -- moves the element- # COUNT from 7 to 6. Both come back DIFFER over identical CSS, and `sweep/ui_parity.report`- # prints that as a style-regression advisory for precisely the ordinary cross-stream-state- # timing the refusals exist to withhold.- #- # THE SAME CORROBORATION THE SCAFFOLD SUPPRESSION REQUIRES, and for the same reason: a- # differing `composer_control` alone is what a treatment that DROPS or renames the control- # produces, and that is a rendering regression this probe should still report. `streaming` and- # `queued_idle` are read off the thread's run state rather than off the composer, so they are- # evidence the composer cannot manufacture.- #- # NOT_COMPARABLE AND NOT MATCH. The probe reads ONE aggregate digest over up to `STYLE_CAP`- # elements, so a genuine CSS difference elsewhere on the page is inside the same number and- # cannot be separated from the swap here. This withholds the reading; it does not pass it.+ # THE RUN-STATE CONTROL IS IN THE SELECTOR LIST, so this reading straddles the same composer swap+ # the structural refusals below are about. `STYLE_SELECTORS` names both the Send and Stop buttons+ # and the signature carries the SELECTOR THAT MATCHED, so a generating arm and a settled one walk+ # two entries for one button: on two byte-identical threads Send against Stop moves the digest+ # with all three properties identical. THE SAME CORROBORATION THE SCAFFOLD SUPPRESSION REQUIRES,+ # because a differing `composer_control` alone is what a treatment that DROPS the control+ # produces. NOT_COMPARABLE AND NOT MATCH: the probe reads ONE aggregate digest, so a genuine CSS+ # difference elsewhere is inside the same number.+ # Over up to `STYLE_CAP` elements.+ # `streaming` and `queued_idle` are read off the run state, not the composer.if (bs.get("elements") != ts.get("elements") or bs.get("digest") != ts.get("digest")) and (generation_disagrees(base, treat) and _run_state_disagrees(base, treat)):@@ -543,7 +472,7 @@ def _messages_moved("""The message half of `_any_moved`, on its own."""skip = skip or set()bm, tm = _messages(base), _messages(treat)- # A message PRESENT ON ONE ARM ONLY is a difference whether or not it was streaming. Being+ # A message PRESENT ON ONE ARM ONLY is a difference whether or not it was streaming: being# mid-reply excuses a digest, never an absence.if set(bm) != set(tm):return True@@ -583,31 +512,20 @@ def settled_messages_moved(base: dict, treat: dict) -> list[str]:if b.get("role") != t.get("role"):out.append(f"msg{i}:role {b.get('role')}->{t.get('role')}")continue- # Flagged in flight by the arm that COULD place its stream. Its digest is a point in a- # stream on that arm whatever role it carries, so it is withheld like any other.+ # Flagged in flight by the arm that COULD place its stream; its digest is a point in a stream on+ # that arm whatever role it carries, so it is withheld like any other.if i in streaming:continueif b.get("role") == "user" and b.get("digest") != t.get("digest"):out.append(f"msg{i}(user):{b.get('chars')}->{t.get('chars')}c")- # WHY THERE IS NO ASSISTANT-ROW RULE HERE, since it looks like the obvious next one to add.- #- # An earlier assistant row on a fully mounted thread really is settled while the tail is being- # written, and a counting argument even makes it provable without knowing WHICH row is in- # flight: a thread writes one reply at a time, so two differing assistant rows cannot both be- # it. Both readings are sound and both are still unusable, because of what BLINDS the probe in- # the first place.- #- # The reachable blind pair is a build that renamed the `data-status` hook. That attribute is on- # the assistant text part and the digest walks attributes, so the rename that blinds the probe- # ALSO moves the digest of every assistant row, by itself. Reporting those rows therefore turns- # every genuinely blind pair into a difference, which is the wall-clock false alarm this file- # exists to remove, wearing the other hat. Measured: implementing the counting rule flipped- # `test_a_settled_queued_idle_pair_is_scored_rather_than_refused`'s blind control from NOT- # COMPARABLE to DIFFER.- #- # A user row is not exposed to that, which is exactly why it is the one shape that qualifies:- # the hook is not on it. Separating "the hook attribute moved" from "the content moved" would- # need a digest that excludes the attribute, which is not something this capture carries.+ # WHY THERE IS NO ASSISTANT-ROW RULE HERE, though it looks like the obvious next one: an earlier+ # assistant row on a fully mounted thread really is settled, and a counting argument even proves+ # which cannot be in flight. Both readings are sound and unusable, because the reachable blind+ # pair is a build that renamed the `data-status` hook -- that attribute is on the assistant text+ # part and the digest walks attributes, so the rename that blinds the probe ALSO moves every+ # assistant row's digest. A user row is not exposed to that.+ # Measured: the counting rule flipped test_a_settled_queued_idle_pair_is_scored_rather_than_refused's+ # blind control from NOT COMPARABLE to DIFFER.return out@@ -628,13 +546,10 @@ def _any_moved(what surfaced it."""skip = skip or set()- # THE SCAFFOLD, not the whole-thread digest. The whole-thread digest serialises the streamed- # message too, so with a reply in flight it differs on essentially every pair -- measured at- # 175 of 175 adjacent 24-character steps of the shipped corpus -- and every check below it- # would be unreachable behind it, exactly as the whole-thread digest once made the overlay walk- # unreachable. The scaffold plus the per-message rows is the same reading taken apart, so- # nothing is lost by decomposing it. On a payload with no scaffold this IS the whole-thread- # digest and the behaviour is the old one.+ # THE SCAFFOLD, not the whole-thread digest: the latter serialises the streamed message too, so+ # with a reply in flight it differs on essentially every pair (175 of 175 adjacent 24-character+ # steps) and every check below would be unreachable behind it. The scaffold plus the per-message+ # rows is the same reading taken apart.if _scaffold(base)[0] != _scaffold(treat)[0]:return Truebo, to = _overlays(base), _overlays(treat)@@ -663,9 +578,8 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:"verdict": NOT_APPLICABLE,"reason": not_applicable,"moved": [],- # The style probe goes with it. Its verdict is `elements` counts matching, and those- # counts are element counts over `[data-role]` among other things, so it reports- # DIFFER for exactly the same reason and with exactly as much meaning.+ # The style probe goes with it: its verdict is `elements` counts matching, and those are element+ # counts over `[data-role]` among other things, so it reports DIFFER for the same reason."style_verdict": NOT_APPLICABLE,"style_reason": not_applicable,}@@ -682,46 +596,24 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:style_verdict, style_reason = compare_styles(base, treat)blind = streaming_probe(base, treat)if blind is not None:- # THE REFUSAL MUST NOT SWALLOW A READING THAT DOES NOT DEPEND ON THE STREAM. An overlay is- # walked from `document`, outside the thread root, so a dialog that mounted when it should- # not, or one whose contents were rewritten, is a finding whether or not the streamed- # message could be identified. Without this it went out with the refusal, and- # `structural_report` buckets a refusal as blind and never consults it for the exit code,- # so the run went green on it.- #- # THE SCAFFOLD IS DELIBERATELY NOT CONSULTED HERE, and this is the part of the review item- # that measurement does not support. `ThreadPrimitive.Root` wraps `ThreadComposerDock`- # (thread.tsx), so the composer is inside `.aui-thread-root` and inside the scaffold -- and- # the composer is exactly the surface that changes when a reply starts and stops. Measured- # on two byte-identical threads differing only in the composer control: Stop against Send- # moves the scaffold from 373 to 381 characters and changes its digest, with no message- # content involved at all. On the very pair this branch is about -- one arm generating with- # a quiet hook, the other finished -- the scaffold therefore differs BECAUSE one arm is- # generating, and reporting that as a rendering difference would manufacture the wall-clock- # false alarm this whole file exists to remove.+ # THE REFUSAL MUST NOT SWALLOW A READING THAT DOES NOT DEPEND ON THE STREAM. An overlay is walked+ # from `document`, outside the thread root, so a dialog that mounted when it should not is a+ # finding either way; without this it went out with the refusal, which `structural_report` buckets+ # as blind, so the run went green. THE SCAFFOLD IS DELIBERATELY NOT CONSULTED HERE:+ # `ThreadPrimitive.Root` wraps `ThreadComposerDock`, so the composer is inside the scaffold and+ # Stop against Send moves it on two byte-identical threads.+ # `compare_visible` already refuses these.+ # thread.tsx; the swap moves the scaffold from 373 to 381 characters.independent = overlays_moved(base, treat)- # AND THE SETTLED MESSAGE ROWS, which are the other half of what this refusal must not- # swallow. A row both arms call the user's, and a row whose role itself changed, cannot be- # the reply being written -- so their meaning does not depend on where the stream had got- # to, and discarding them costs exactly the finding this instrument exists to make. Without- # this a user message rendered differently by the treatment left here as NOT COMPARABLE- # with an empty `moved`, and `structural_report` buckets a refusal as blind and never- # consults it for the exit code, so the run went green on it. `compare_visible` already- # applies this rule to its own rows and says it is the same rule as this one; it has to be.+ # AND THE SETTLED MESSAGE ROWS, the other half of what this refusal must not swallow: a row both+ # arms call the user's, and a row whose role itself changed, cannot be the reply being written,+ # so their meaning does not depend on the stream. Without this a user message rendered+ # differently by the treatment left as NOT COMPARABLE with an empty `moved`.independent = independent + settled_messages_moved(base, treat)- # AND THE SCAFFOLD, but only when the two arms agree about whether a reply was running.- # The composer is inside the scaffold and is a function of exactly that, so the scaffold is- # readable here when they agree and meaningless when they do not. That is the correct form- # of the half of this refusal that measurement did not support.- #- # AND THE DISAGREEMENT HAS TO BE CORROBORATED OFF THE RUN STATE, exactly as the composer- # suppression below requires. `generation_disagrees` reads `composer_control`, which IS the- # composer, so on its own it excuses a composer difference with the composer -- and a- # treatment that DROPS the Stop control, renames it, or selects the wrong one makes the- # tokens differ for that reason and suppressed the whole scaffold with it, while `streaming`- # and `queued_idle` were saying the arms were in the same run state all along. That is a- # rendering regression withheld by the branch written to admit rendering regressions, and- # since a refusal is filed under `blind` the run goes green on it.+ # AND THE SCAFFOLD, but only when the two arms agree about whether a reply was running: the+ # composer is inside the scaffold and is a function of exactly that. AND THE DISAGREEMENT HAS TO+ # BE CORROBORATED OFF THE RUN STATE, since `generation_disagrees` reads `composer_control`, which+ # IS the composer -- so on its own it excuses a composer difference with the composer.if scaffold_moved(base, treat) and not (generation_disagrees(base, treat) and _run_state_disagrees(base, treat)):@@ -748,86 +640,23 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:"style_verdict": style_verdict,"style_reason": style_reason,}- # ── THE STREAMED MESSAGE ────────────────────────────────────────────────────────────────────- #- # `streaming` holds the messages that were still being written on one arm or the other. They- # are scored on NOTHING, and that is a refusal rather than a normalisation: the digest of a- # half-written message names a point in a stream, the two arms are at two different points in- # the same stream by construction, and no amount of text normalisation can turn one into the- # other. See `in_flight`.- #- # THREE OUTCOMES, and the ordering between them is the load-bearing part.- #- # the settled document differs DIFFER. This is the case that used to be lost. An action- # landing inside a stream was silenced wholesale by- # UNSTABLE_ACTIONS, so a real regression anywhere else in- # the thread -- another message, an overlay, the composer --- # printed under "expected to vary" and the run exited 0.- # Now the streamed message is elided and the rest is scored.- # the settled document agrees and- # the in-flight message agrees MATCH, unchanged. Two arms that landed on the same point- # in the stream serialised identically, which is the claim- # this mode makes, and demoting it would cost coverage for- # nothing.- # the settled document agrees and- # the in-flight message differs NOT COMPARABLE. Not a pass. `CLAIM_STRUCTURAL` quantifies- # over the whole thread, one message did not serialise- # identically, and the reason it did not is a reading with- # no defined moment. Calling that MATCH would be the- # instrument certifying a surface it could not look at,- # which is the failure `compare_styles` and- # `compare_visible` each already refuse in their own way.- #- # WHAT THIS GIVES UP, plainly, and both of these are measured rather than reasoned about.- #- # 1. A genuine rendering regression INSIDE the message that happened to be streaming is no- # longer distinguishable here and lands as NOT COMPARABLE. It was not distinguishable before- # either -- every action that lands in a stream is on the declared unstable list -- so- # nothing that used to be caught stops being caught, and the outcome moves from "expected to- # vary, exit 0" to "not measured, not a pass".- # 2. A REORDER that moves the streamed message past another message of the same role is- # demoted from DIFFER to NOT COMPARABLE. The per-message rows are keyed by mounted index, so- # swapping two messages puts a streaming row at one index on one arm and at the other index- # on the other; both indices are then in flight on one side or the other, both are withheld,- # and the scaffold markers that survive carry the same role in both orders. Measured on the- # live-DOM battery: 11 injected rendering differences, 10 still reported DIFFER and this one- # demoted. It is a demotion and not a hole -- NOT COMPARABLE is not a pass and the run does- # not go green on it -- and the only shape that reaches it needs the streamed message not to- # be the last one, which the app does not do today.+ # THE STREAMED MESSAGE. `streaming` holds messages still being written on one arm or the other;+ # they are scored on NOTHING, a refusal rather than a normalisation, because the two arms are at+ # two different points in one stream by construction. THREE OUTCOMES, and the ordering is+ # load-bearing: a settled document that differs is DIFFER; settled agreeing plus in-flight+ # agreeing is MATCH; settled agreeing with the in-flight message differing is NOT COMPARABLE, not+ # a pass, because `CLAIM_STRUCTURAL` quantifies over the whole thread. WHAT THIS GIVES UP: a+ # regression inside the streaming message lands as NOT COMPARABLE, and a REORDER past another+ # message of the same role is demoted from DIFFER (10 of 11 injected differences still DIFFER).streaming = in_flight(base, treat)+ # THE COMPOSER IS NOT A RENDERING DIFFERENCE. One arm finished and one still writing has its+ # MESSAGES withheld correctly, but the dock is inside `.aui-thread-root`, so `digest_scaffold`+ # carries Stop on one arm and Send on the other and `_any_moved` reported DIFFER while every+ # settled row was byte-identical. THE NULL BATTERY CANNOT SEE THIS -- one build against itself+ # has both arms generating and the bias cancels. WITHHELD RATHER THAN IGNORED: if a message or+ # overlay also moved this never runs. AND THE RUN STATE HAS TO SAY SO INDEPENDENTLY, or the+ # suppression argues in a circle.# ── THE COMPOSER IS NOT A RENDERING DIFFERENCE ──────────────────────────────────────────────- #- # The pair this whole change is about is one arm that has finished its reply against one that- # is still writing it. Its MESSAGES are withheld correctly. Its COMPOSER is not: the dock is- # inside `.aui-thread-root`, so `digest_scaffold` carries Stop on the arm that is generating- # and Send on the arm that is not, and `_any_moved` then reports the pair as DIFFER with the- # single claim `thread scaffolding outside any message (373->381c)`. Measured on exactly that- # pair, with every settled message row byte-identical across the two arms.- #- # THE NULL BATTERY CANNOT SEE THIS, which is why it survived a 15-of-15-to-0 null. The null is- # one build against itself at six points in ONE stream, so BOTH arms are generating and both- # render Stop; the bias is symmetric within the control and cancels exactly. A flat null proves- # repeatability, never comparability.- #- # WITHHELD RATHER THAN IGNORED. If a message or an overlay also moved, that is reported as- # usual and this never runs. It is only when the scaffold is the ONLY thing that moved, and the- # arms disagree about generation, that the reading has no defined moment -- and then the honest- # answer is a refusal, not a pass. Calling it MATCH would hide a genuine composer regression.- #- # AND THE RUN STATE HAS TO SAY SO INDEPENDENTLY, or the suppression argues in a circle.- # `generation_disagrees` reads `composer_control`, the token naming which control the composer- # rendered -- so "the arms were at different points in the turn" is being proved by the very- # surface whose difference is in question. A treatment that DROPS the Send button, renames it,- # or selects the wrong control reaches this branch with every message and overlay agreeing, and- # a refusal here is a green run: `report` files NOT COMPARABLE under `blind` and takes its exit- # code from `stable_bad or one_sided`. That is the regression the note above says must not be- # hidden, hidden by the branch written to say so.- #- # `streaming` and `queued_idle` are read off the run state rather than off the composer, so- # they are evidence the composer cannot manufacture. Every legitimate suppression still has- # one: Stop against Send differs in `streaming`, and Queue against Send in the queued-idle- # interval differs in `queued_idle`. What is left, the arms agreeing on both and still- # rendering different controls, has no run-state explanation and is a rendering difference.if (generation_disagrees(base, treat)and _run_state_disagrees(base, treat)@@ -866,12 +695,10 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:f"msg{i}({bm[i].get('role', '?')}):{bm[i].get('chars')}->{tm[i].get('chars')}c"for i in unsettled[:4])- # A ROW WHOSE ROLE CHANGED IS NOT SOMETHING THIS PAIR AGREED ON, and `_any_moved`- # cannot see it: the role sits beside the digest, and an in-flight digest is withheld.- # `settled_messages_moved` reports a role change even in flight -- how far a reply has- # arrived says nothing about whose message it is -- and the two readings must not- # disagree. The verdict is the refusal either way; this only decides whether the- # refusal ALSO carries a positive reading, and a role disagreement carries none.+ # A ROW WHOSE ROLE CHANGED IS NOT SOMETHING THIS PAIR AGREED ON, and `_any_moved` cannot see it+ # because an in-flight digest is withheld while `settled_messages_moved` reports a role change+ # even in flight. The verdict is the refusal either way; this only decides whether the refusal+ # also carries a positive reading.roles_agree = all(bm[i].get("role") == tm[i].get("role") for i in set(bm) & set(tm))out = {"verdict": NOT_COMPARABLE,@@ -891,11 +718,9 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:"style_reason": style_reason,}if roles_agree:- # THE ONE REFUSAL THAT ALSO CARRIES A POSITIVE READING. See `SETTLED_MATCH` for- # why this flag exists and what it is and is not allowed to move. It is set here- # and nowhere else, because this is the only branch reached with `_any_moved`- # already false: the scaffold agreed, every overlay agreed, the two arms mounted- # the same set of messages, and every message that was NOT being written agreed.+ # THE ONE REFUSAL THAT ALSO CARRIES A POSITIVE READING (see `SETTLED_MATCH`). Set here and+ # nowhere else, because this is the only branch reached with `_any_moved` already false: the+ # scaffold, every overlay, the mounted message set and every settled message all agreed.out[SETTLED_MATCH] = Truereturn outreturn {@@ -950,18 +775,11 @@ def execution_verdict(base_row: Optional[dict], treat_row: Optional[dict]) -> Oprow = base_row if label == "base" else treat_rowassert isinstance(row, dict)reason = row.get("reason") or "no reason recorded"- # A MISSED SLOT IS NOT A BUILD DIFFERENCE, even when only one arm missed it, and this- # distinction is the whole load-bearing part of `one_sided`. `ran=false` has two causes- # that look identical in the count and mean opposite things. The runner arriving after- # the slot closed says nothing about the build: the schedule is fixed, the budgets are- # small, and a machine slow enough to miss a slot at 45700ms is slow enough to miss it- # on the next repetition too, so corroboration does not separate them -- the misses are- # correlated through the runner, not independent draws. Treating that as asymmetric- # execution reds the job on machine speed and breaks the invariant the mutation study- # established, that a missed slot cannot move this verdict. A precondition failure is- # the opposite: `image_upload` records `slot_missed=false` with "no visible attachments- # button", and a control that stops opening records exactly that. So the signal is- # `slot_missed`, which the driver already writes, and not merely which arm went idle.+ # A MISSED SLOT IS NOT A BUILD DIFFERENCE, even when only one arm missed it. `ran=false` has two+ # causes that look identical: a runner arriving after the slot closed says nothing about the+ # build, and because misses are correlated through the runner rather than independent draws,+ # corroboration does not separate them. A precondition failure is the opposite, so the signal is+ # `slot_missed` and not merely which arm went idle.missed_slot = bool(row.get("slot_missed"))detail = (f"the action did not run on the {label} arm ({reason}), so any "@@ -978,8 +796,8 @@ def execution_verdict(base_row: Optional[dict], treat_row: Optional[dict]) -> Op"verdict": NOT_EXERCISED,"moved": [],"one_sided": live[0] if live else "",- # The idle arm's OWN not_run string, unwrapped. `reason` above is prose built for a- # reader; the exemption has to match on what the action actually recorded.+ # The idle arm's OWN not_run string, unwrapped: `reason` above is prose built for a reader, while+ # the exemption has to match what the action recorded."idle_reason": reason,"reason": detail,"style_verdict": NOT_EXERCISED,@@ -1095,15 +913,14 @@ def derive_unstable(verdict = result.get("verdict")if verdict not in (MATCH, DIFFER):if verdict == NOT_COMPARABLE and result.get(SETTLED_MATCH):- # Everything this pair could read agreed: an observation of NON-difference, so- # `differed` cannot grow and nothing can be called unstable on its strength.- # Tallied separately so a reader can see how much of a decision rests on it.+ # Everything this pair could read agreed: an observation of NON-difference, so `differed` cannot+ # grow. Tallied separately so a reader can see how much of a decision rests on it.seen[action] += 1settled[action] += 1continue- # Not comparable and not exercised are both "no reading", and neither may be counted- # as an observation of stability. An action derived as stable from four pairs that- # never ran would be permanently trusted on the strength of nothing.+ # Not comparable and not exercised are both "no reading", and neither may count as an observation+ # of stability: an action derived as stable from four pairs that never ran would be permanently+ # trusted on nothing.blind[action] += 1continueseen[action] += 1@@ -1116,23 +933,16 @@ def derive_unstable("observations": n,"differed": d,"not_comparable": blind[action],- # How many of `observations` came from a settled-match refusal rather than a MATCH.- # Reported rather than folded in: an action decided entirely this way was decided on- # the settled thread only, which is the weaker reading.+ # How many of `observations` came from a settled-match refusal rather than a MATCH. Reported+ # rather than folded in: an action decided entirely this way was decided on the settled thread.SETTLED_MATCH: settled[action],- # Unstable only with enough observations to mean it. Below that the honest answer is- # "not enough evidence", which is not the same as "stable" and is not reported as it.- #- # FROM THE COMPLETE COMPARISONS ONLY: a settled-match refusal can decide an action- # and never help CLASSIFY one as unstable. One DIFFER beside one settled match is one- # differing comparison, and letting the refusal supply the second would mint an- # exemption from a reading that never saw the live subtree.+ # Unstable only with enough observations to mean it; below that the honest answer is "not enough+ # evidence". FROM THE COMPLETE COMPARISONS ONLY: a settled-match refusal can decide an action but+ # never help CLASSIFY one as unstable, or it would mint an exemption from a partial reading."unstable": bool(d and (n - settled[action]) >= min_observations),- # THE SAME COUNT `unstable` WAS DECIDED ON, once anything has differed. On the raw- # total the mixed state -- one DIFFER beside one settled match -- read as "decided and- # stable", so `cross_check` filed a declared action under `declared_stable_in_practice`- # ("the null never saw it differ") on a run where it differed once. With nothing- # differing there is no classification to make and the total is the right count.+ # THE SAME COUNT `unstable` WAS DECIDED ON, once anything has differed. On the raw total, one+ # DIFFER beside one settled match read as "decided and stable", so `cross_check` filed a declared+ # action under `declared_stable_in_practice` on a run where it differed once."undetermined": (n - settled[action] if d else n) < min_observations,}return out@@ -1180,36 +990,19 @@ def summarise(results: Iterable[dict[str, Any]]) -> dict[str, int]:return dict(tally)+# THE POLICY: changes must preserve UI and UX idempotency, with three exemptions -- a dramatic+# performance win accepted on the record, a difference that exists only OFF SCREEN, and a+# select-all that need not select all PROVIDED the copy stays complete. The third is what makes+# deferral and virtualization cheap: the copy path may serialise from the message store, since+# completeness is REQUIRED and visual selection fidelity is not. `compare()` cannot express the+# second exemption, so it returns NOT_APPLICABLE and withholds a verdict. This supplies one.++#: The claim each verdict makes, so a reader knows which of the three was checked. NOT+#: "whole-document structural parity", which this instrument cannot support: `scene/parity.js`+#: digests the thread root plus overlay selectors, never the sidebar, geometry or custom+#: properties. Printing the stronger claim is how a sidebar-drag campaign came to be scored 0 of+#: 34 differing pairs under a banner saying the DOM was identical.# ── visible-region parity ───────────────────────────────────────────-#-# THE POLICY. All changes must preserve UI and UX idempotency, with three exemptions: a difference-# may be accepted deliberately when performance improves dramatically; a difference that exists-# only OFF SCREEN is fine by definition, because rendering only what is visible is an accepted-# technique rather than a parity violation; and a select-all need not select all, PROVIDED the copy-# it produces stays complete.-#-# The third is what makes deferral and virtualization cheap: the copy path stops depending on what-# is mounted, so it may serialise the thread from the message store as markdown or plain text-# instead of reproducing a DOM selection. Completeness of the copied content is REQUIRED -- silent-# truncation is data loss -- and visual selection fidelity is NOT.-#-# `compare()` above cannot express the second exemption. It digests the thread whether or not any-# of it is on screen, so deferred off-screen work fails it by construction: virtualization,-# deferred fence highlighting, content-visibility, lazy images. Returning NOT_APPLICABLE for those-# pairs withholds a verdict.-# This supplies one.--#: The claim each verdict makes, so a reader knows which of the three was actually checked. A bare-#: "PARITY OK" has meant three very different things in this file's history and the difference-#: between them is the difference between a strong result and a weak one.-#: NOT "whole-document structural parity: every element in the DOM is identical on both arms",-#: which is what this string used to say and which the instrument cannot support. `scene/parity.js`-#: digests the THREAD ROOT plus a list of overlay selectors; it never walks the sidebar, never reads-#: geometry and never reads CSS custom properties. Printing the stronger claim turns a limited-#: thread-structure reading into an experimental conclusion about the whole UI, which is how a-#: sidebar-drag campaign came to be scored 0 of 34 differing pairs by a digest whose own null-#: control also returned 0 of 34: the instrument was not discriminating in either direction, and-#: the banner said the DOM was identical.CLAIM_STRUCTURAL = ("thread-structure parity: the thread root and the declared overlay selectors serialise ""identically on both arms, on screen and off. It does NOT cover the sidebar, computed layout "@@ -1228,41 +1021,24 @@ CLAIM_BEHAVIOURAL = ("copied")-#: THE POLICY EVERY VERDICT IS JUDGED AGAINST, printed beside the claim.-#:-#: A bare "PARITY OK" reads as "the UI is unchanged" and none of the three modes can support that-#: sentence. Each supports a narrower one, and which one is being made changes what a pass MEANS,-#: so the policy is named next to the claim rather than left in a document somebody has to find.-#:-#: The exemptions are not loopholes. The first is a decision someone makes on the record with a-#: number attached; the second is a definition, because rendering only what is visible is an-#: accepted technique rather than a parity violation; the third is CONDITIONAL, and the condition-#: is the part that is required -- the copy must be complete, and only the visual fidelity of the-#: selection is given up. None of the three removes the need for a floor: a difference that is-#: exempt still has to be shown to be the difference you think it is, which is what the null-#: control is for.-#:-#: NO MODE GRANTS THE THIRD ON A DIGEST. `--mode behaviour` is the only one that speaks to it at-#: all, through `clipboard_carries_the_whole_thread`, which scores the copy against the THREAD-#: rather than against the other arm. Where a payload carries no readable `select_all_copy` the-#: gate records that the exemption exists and does not grant it.-#:-#: And it scores it BY LENGTH. That is a proxy for completeness, not completeness itself, so the-#: printed line says so rather than leaving "complete" to be read as a content comparison. A-#: content comparison is not available here: the base arm's clipboard is the DOM's rendered text-#: while a store-based copy is markdown SOURCE, so the two are different serialisations of one-#: conversation and comparing their characters fails on a CORRECT build. The coverage band is what-#: carries the weight, and it is the band that refused both defects this was written for ---#: truncation at 0.61 of the thread and substitution at 2.16 -- so it is interpolated from the-#: live constants by `behaviour_policy()` rather than written out here, where it could drift.+#: THE POLICY EVERY VERDICT IS JUDGED AGAINST, printed beside the claim, because a bare "PARITY+#: OK" reads as "the UI is unchanged" and no mode supports that sentence. The exemptions are not+#: loopholes: the first is a decision made on the record with a number, the second is a+#: definition, and the third is CONDITIONAL on the copy being complete. NO MODE GRANTS THE THIRD+#: ON A DIGEST: only `--mode behaviour` speaks to it, through+#: `clipboard_carries_the_whole_thread`, which scores the copy against the THREAD rather than the+#: other arm. And it scores BY LENGTH, a proxy for completeness: the base arm's clipboard is+#: rendered text while a store-based copy is markdown SOURCE, so comparing characters fails on a+#: CORRECT build. The coverage band carries the weight -- it refused truncation at 0.61 and+#: substitution at 2.16 -- so it is interpolated from the live constants.POLICY = ("UI and UX idempotency is required, with three exemptions: a deliberate difference accepted ""for a dramatic performance improvement, a difference that exists only OFF SCREEN, and a ""select-all that does not select all PROVIDED the copy it produces stays complete")-#: What each mode can and cannot decide under that policy. The point of the second half of each-#: line is that a reader can tell which sentence they are being handed.+#: What each mode can and cannot decide under that policy; the second half of each line is what+#: tells a reader which sentence they are being handed.POLICY_BY_MODE = {"structural": (f"{POLICY}. This mode judges the FIRST requirement over the thread root and the declared "@@ -1320,9 +1096,9 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:}assert base is not None and treat is not None- # THE POSITIVE CONTROL. A visibility scan that matched nothing has equal (empty) ordinal sets- # and no differing digests, so without this it returns the strongest verdict available on the- # strength of never having seen a single message. Exactly the failure `compare_styles` had.+ # THE POSITIVE CONTROL: a visibility scan that matched nothing has equal (empty) ordinal sets and+ # no differing digests, so without this it returns the strongest verdict available on the strength+ # of never having seen a message. Exactly the failure `compare_styles` had.bn, tn = len(base.get("ever_visible") or []), len(treat.get("ever_visible") or [])if bn == 0 or tn == 0:return {@@ -1336,29 +1112,17 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:"claim": CLAIM_VISIBLE,}- # BEFORE THE TWO SETS ARE COMPARED, because a collision is what makes them untrustworthy.- # `scene/parity.js` keys its per-message digests by ordinal, so two mounted rows sharing an- # `aria-posinset` -- a virtualizer renumbering a recycled row wrongly -- leave one digest where- # two rows were on screen, and `ever_visible` collapses them too. Both quantities below are then- # short by a row, and whether the verdict came out MATCH or DIFFER would turn only on which of- # the two survived, which is DOM order and nothing else.- #- # Read defensively so a payload recorded before the counter existed compares exactly as it did.+ # BEFORE THE TWO SETS ARE COMPARED, because a collision is what makes them untrustworthy:+ # `scene/parity.js` keys per-message digests by ordinal, so two mounted rows sharing an+ # `aria-posinset` leave one digest where two rows were on screen and `ever_visible` collapses+ # them too. Read defensively so a payload recorded before the counter existed compares as it did.collisions = {label: int(side.get("ordinal_collisions") or 0)for label, side in (("base", base), ("treatment", treat))}- # A SEVERE FINDING OUTRANKS THIS REFUSAL, and exactly one finding qualifies. "One arm's- # viewport ended EMPTY and the other's did not" is marked NOT SUPPRESSIBLE where it is raised,- # because losing the thread is a different kind of statement from a capture that could not be- # read -- and a collision provably cannot manufacture it. A collision needs TWO mounted rows- # at one position, so the map it corrupts still has at least one entry in it; it can merge two- # entries and never empty a map. So the one thing a collision cannot have caused is still- # reported rather than swallowed by a blanket refusal.- #- # Only that one. Every other comparison below IS corrupted by a collision: `ever_visible` loses- # an ordinal on one arm alone, so "the two arms put different messages on screen" can be- # entirely an artefact of the collision, and the per-ordinal digests are short by a row.+ # A SEVERE FINDING OUTRANKS THIS REFUSAL, and exactly one qualifies: "one arm's viewport ended+ # EMPTY and the other's did not" is marked NOT SUPPRESSIBLE, because a collision needs TWO mounted+ # rows at one position, so it can merge entries but never empty a map.lost_the_thread = (len(base.get("messages") or {}) == 0) != (len(treat.get("messages") or {}) == 0)@@ -1401,20 +1165,18 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:# never counted as agreement: this is the residue of comparing a windowed arm at one instant.uncomparable = sorted(int(o) for o in map(str, sorted(bev)) if o not in bmsg or o not in tmsg)moved = []- # The subset of `moved` that CANNOT be a point in a stream, kept so the blind-probe refusal- # below does not take it out with the rest. See each `settled.append` for why that row is- # provably not the reply being written.+ # The subset of `moved` that CANNOT be a point in a stream, kept so the blind-probe refusal below+ # does not take it out with the rest. See each `settled.append` for why.settled: list[str] = []for ordinal in sorted(bev):key = str(ordinal)b, t = bmsg.get(key), tmsg.get(key)if b is None or t is None:continue- # A ROLE IS NOT A POSITION IN A STREAM. It is captured beside the digest rather than inside- # it, and how far a reply has arrived says nothing about whether it is the assistant's. So- # a row that changed role is reported even while that row is in flight, where the digest- # itself is withheld: a treatment that renders the live assistant row as `data-role="user"`- # is a structural regression that used to leave here as NOT COMPARABLE.+ # A ROLE IS NOT A POSITION IN A STREAM: it is captured beside the digest, and how far a reply has+ # arrived says nothing about whose message it is. So a row that changed role is reported even+ # while in flight -- a treatment rendering the live assistant row as `data-role="user"` used to+ # leave here as NOT COMPARABLE.if b.get("role") != t.get("role"):claim = f"ordinal {ordinal}:role {b.get('role')}->{t.get('role')}"moved.append(claim)@@ -1422,34 +1184,25 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:continueif b.get("digest") != t.get("digest"):if b.get("in_flight") or t.get("in_flight"):- # STILL BEING WRITTEN ON ONE ARM OR THE OTHER, so its digest names a point in a- # stream. Residue, exactly like an ordinal that was unmounted before the capture:- # it cannot be a difference, and it cannot be an agreement either, so it joins the- # list that refuses the verdict below rather than the list that fails it. Same- # rule as `compare` applies to the structural digest; the two modes must not- # disagree about which readings have a defined moment.+ # STILL BEING WRITTEN ON ONE ARM OR THE OTHER, so its digest names a point in a stream: residue,+ # exactly like an ordinal unmounted before the capture, so it joins the list that refuses the+ # verdict rather than the one that fails it. Same rule `compare` applies to the structural digest.uncomparable.append(ordinal)continueclaim = f"ordinal {ordinal}({b.get('role')}):{b.get('chars')}->{t.get('chars')}c"moved.append(claim)- # A USER ROW IS NEVER THE REPLY BEING WRITTEN. Both arms agree this ordinal is the- # user's, and a stream writes into an assistant message, so this difference survives- # the blind-probe refusal below. The two arms agreeing is what makes it provable: a- # row whose role DISAGREES is reported above as a role change, not silently trusted.+ # A USER ROW IS NEVER THE REPLY BEING WRITTEN, and both arms agreeing on the role is what makes+ # it provable; a row whose role DISAGREES is reported above as a role change.if b.get("role") == "user":settled.append(claim)- # Ordinals that were unmounted before the capture and ordinals that were still streaming are- # the same kind of residue and are counted once each.+ # Ordinals unmounted before the capture and ordinals still streaming are the same kind of residue+ # and are counted once each.uncomparable = sorted(set(uncomparable))- # ONE VIEWPORT ENDED EMPTY AND THE OTHER DID NOT, which is as visible a difference as there is- # and used to be reported as NOT COMPARABLE -- a refusal, not a finding.- #- # This is not hypothetical and it is why the check exists. On the 100K virtualization arm,- # `model_change` took the thread from 12 mounted messages to 0 and it never came back: the- # census reads 0 messages and 2,107 elements for the rest of the film, and three later actions- # could not run at all. Both arms had shown the same ordinals EARLIER in the action, so the- # union matched and every per-ordinal digest was simply missing on one side. Comparing only the- # union cannot see that; comparing what each arm could still show at the end can.+ # ONE VIEWPORT ENDED EMPTY AND THE OTHER DID NOT, as visible a difference as there is, and it used+ # to be reported as NOT COMPARABLE. Not hypothetical: on the 100K virtualization arm+ # `model_change` took the thread from 12 mounted messages to 0 and it never came back. Both arms+ # had shown the same ordinals earlier, so the union matched and every per-ordinal digest was+ # simply missing on one side -- comparing what each arm could still show at the end sees that.b_left, t_left = len(bmsg), len(tmsg)if (b_left == 0) != (t_left == 0):empty, full = ("treatment", "base") if t_left == 0 else ("base", "treatment")@@ -1463,34 +1216,22 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:),"moved": [f"ordinal {o}" for o in sorted(bev)[:8]],"claim": CLAIM_VISIBLE,- # NOT SUPPRESSIBLE BY THE NOISE FLOOR. The floor exists for actions whose visible- # region differs between two runs of the same build, which is a volatile attribute on- # rows of identical length. Losing the entire thread is a different kind of statement,- # and an action can easily be BOTH: on the 100K run `model_change` is in the derived- # unstable set because the null control's copy of it differs on an attribute, and that- # would have silenced "the treatment arm's viewport ended empty" -- suppressing a lost- # conversation on the strength of unrelated jitter in the same action.+ # NOT SUPPRESSIBLE BY THE NOISE FLOOR: the floor exists for actions whose visible region differs+ # between two runs of one build, and losing the entire thread is a different kind of statement.+ # An action can be both -- `model_change` is in the derived unstable set for an unrelated+ # attribute, which would have silenced "the treatment arm's viewport ended empty"."severe": True,}- # THE STREAMING PROBE, on the same footing it has in `compare`, because this mode is scored- # from its own payload and never sees the structural verdict. `in_flight` above walks the- # `data-status` / `aria-busy` hooks, so a build that renames them reports every row settled;- # the two arms are two different builds, so that can be true on ONE of them while the other's- # reply has genuinely finished, and the differing digests below are then two points in a stream- # scored as a rendering difference.- #- # AFTER the two lost-conversation findings above and BEFORE the digest comparison, for the- # reason `compare` puts it after `mount_count_mismatch`: different messages on screen, or one- # viewport emptied, are readings that do not depend on the stream split at all, and a build- # that loses the thread while a reply runs stays a finding rather than a shrug.+ # THE STREAMING PROBE, on the same footing it has in `compare`, because this mode is scored from+ # its own payload: `in_flight` walks the `data-status` / `aria-busy` hooks, so a build that+ # renames them reports every row settled while the other arm's reply has genuinely finished.+ # AFTER the two lost-conversation findings and BEFORE the digest comparison, since different+ # messages on screen do not depend on the stream split at all.blind = streaming_probe(base, treat)if blind is not None:- # SAME RULE AS `compare`: the refusal covers the rows whose meaning depends on where the- # stream had got to, and nothing else. `settled` holds the rows that provably cannot be the- # reply being written -- a row both arms call the user's, and a row whose role itself- # changed -- and those are reported rather than discarded. Without this a changed user- # message left here as NOT COMPARABLE with an empty `moved`, and `visible_report` buckets- # a refusal as blind and never consults it for the exit code.+ # SAME RULE AS `compare`: the refusal covers the rows whose meaning depends on where the stream+ # had got to and nothing else. `settled` holds the rows that provably cannot be the reply being+ # written, and without this a changed user message left as NOT COMPARABLE with an empty `moved`.if settled:return {"verdict": DIFFER,@@ -1519,25 +1260,14 @@ def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:"not_digested": uncomparable,}if uncomparable:- # ANY residue refuses the verdict, not only a total one. This branch used to be entered- # solely when EVERY visible ordinal was undigestable (`len(uncomparable) == len(bev)`), so a- # scrolling or windowed action that put six messages on screen and unmounted one of them- # before the capture returned MATCH on the strength of the other five. The claim printed- # above quantifies over EVERY message the viewport showed; one message missing makes it- # unsupported, and the rendering difference this mode exists to catch could be in exactly- # the message that is missing. `visible_report` never printed the `not_digested` residue- # either, so that pair exited 0 with nothing on screen to say a message went uncompared.- #- # WHY THE VERDICT AND NOT MERELY THE PASS COUNT. Demoting it inside `visible_report` alone- # would leave the row itself reading `match`, and every other consumer of that row counts a- # match as agreement: `visible_unstable_set` reads verdicts to derive the noise floor,- # `summarise` tallies them, and the payload keeps them verbatim for whoever reads it next.- # The honest outcome has to live in the verdict, which is the one thing they all share.- #- # THE COST, MEASURED on the two real 100K films rather than guessed. On the windowed arm 4- # of 39 matching pairs carry a residue, all of them `stop_generation`; on the base-vs-base- # null control 7 of 43 do. The mode keeps a discriminating majority on both, so this buys- # honesty at four pairs in sixty-four.+ # ANY residue refuses the verdict, not only a total one. Entered solely when EVERY visible ordinal+ # was undigestable, a windowed action that put six messages on screen and unmounted one returned+ # MATCH on the strength of the other five, while the printed claim quantifies over EVERY message+ # the viewport showed. WHY THE VERDICT AND NOT MERELY THE PASS COUNT: demoting inside+ # `visible_report`, which never printed the `not_digested` residue either, alone leaves the row+ # reading `match`, which everything downstream counts as+ # agreement. THE COST, MEASURED on two real 100K films: four pairs in sixty-four.+ # `compare` puts this after `mount_count_mismatch`.digested = len(bev) - len(uncomparable)return {"verdict": NOT_COMPARABLE,diff --git a/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py b/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.pyindex 3e0b7469c50..0fc9198f7c8 100644--- a/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py+++ b/tests/studio/studiobench/analysis/selftest/test_studiobench_visible_parity.py@@ -181,8 +181,7 @@ def test_a_pair_where_nothing_visible_could_be_digested_is_not_a_pass():"""Every ordinal the viewport showed had been unmounted by capture time, so the comparisonobserved the visibility but none of the content. That is not agreement."""got = P.compare_visible(_cap({}, ever = [3, 4]), _cap({}, ever = [3, 4]))- # The zero-length scan control fires first, and either refusal is correct; what must not- # happen is a MATCH.+ # The zero-length scan control fires first, and either refusal is correct; what must not happen is a MATCH.assert got["verdict"] == P.NOT_COMPARABLE, got@@ -270,16 +269,15 @@ def test_every_mode_names_the_policy_it_is_judging_against():# The behavioural mode grants neither of the first two and is the only one that speaks to the# third, so "either" would be the wrong word for it now.assert "cannot grant the performance or off-screen exemptions" in P.POLICY_BY_MODE["behaviour"]- # AND IT SAYS HOW IT MEASURES THE CONDITION. "Complete" on its own reads as a comparison of- # the copied content; what the gate does is divide each arm's clipboard length by the thread's- # visible text and require the ratio to land in a band. Saying which of those it is decides- # whether a reader is entitled to conclude the copy was intact, so the weaker wording is not- # allowed back: the line has to name the measure AND disclaim the one it does not perform.+ # AND IT SAYS HOW IT MEASURES THE CONDITION. "Complete" alone reads as a comparison of the copied+ # content; what the gate does is divide each arm's clipboard length by the thread's visible text+ # and require the ratio to land in a band. Which of those it is decides whether a reader may+ # conclude the copy was intact, so the line has to name the measure AND disclaim the other.assert "BY LENGTH" in P.POLICY_BY_MODE["behaviour"]assert "does not compare the copied characters" in P.POLICY_BY_MODE["behaviour"]assert "records the exemption rather than granting it" in P.POLICY_BY_MODE["behaviour"]- # The floor survives the exemption. An exemption changes what counts as a pass; a measurement- # with no floor under it is not a pass in the first place.+ # The floor survives the exemption. An exemption changes what counts as a pass; a measurement with+ # no floor under it is not a pass in the first place.assert "does not remove the floor" in P.POLICY_BY_MODE["visible"]@@ -302,8 +300,8 @@ def test_the_policy_line_is_printed_next_to_every_claim_line():for name in claims:assert f"P.{name}" in source, f"{name} is never printed"for mode in P.POLICY_BY_MODE:- # Either printed straight from the table, or through the per-mode helper that fills in- # the numbers that mode is actually enforcing.+ # Either printed straight from the table, or through the per-mode helper that fills in the numbers+ # that mode is enforcing.assert (f"POLICY_BY_MODE['{mode}']" in source or f"{mode}_policy(" in source), f"the {mode} policy line is never printed"diff --git a/tests/studio/studiobench/analysis/symbols.py b/tests/studio/studiobench/analysis/symbols.pyindex 052deb6e0f5..81daee8d946 100644--- a/tests/studio/studiobench/analysis/symbols.py+++ b/tests/studio/studiobench/analysis/symbols.py@@ -60,28 +60,22 @@ FAILED = "failed"OK = "ok"NOT_BUILT = "not_built"-# A function whose counts are all this small carries almost no information and-# collides with everything. Excluded from matching, and the exclusion is-# reported so the coverage of the bridge is visible.+# A function whose counts are all this small carries almost no information and collides with+# everything. Excluded from matching, and the exclusion is reported so the bridge's coverage is+# visible.MIN_INFORMATIVE_COUNT = 2-# Above this share of resolved mappings being name-identical (`push` -> `push`),-# the two arms are almost certainly the same build. Some identity is EXPECTED-# and healthy: React's release minifier leaves scheduler entry points such as-# `push`, `peek` and `performWorkUntilDeadline` alone, and `keepNames` preserves-# others, so a genuine bridge measured on a real React 19.2.4 pair came in at-# roughly 0.1. A same-build pair comes in at exactly 1.0, so the two regimes are-# not close and this threshold is not delicate.+# Above this share of resolved mappings being name-identical (`push` -> `push`), the two arms are+# almost certainly the same build. Some identity is EXPECTED: React's release minifier leaves+# scheduler entry points alone and `keepNames` preserves others, so a genuine bridge on a real+# React 19.2.4 pair came in at roughly 0.1 against exactly 1.0 for a same-build pair.+# Untouched: `push`, `peek` and `performWorkUntilDeadline`.MAX_IDENTITY_MAPPING_FRACTION = 0.5-# NOTE FOR THE NEXT PERSON, because this is the obvious idea and it DOES NOT-# WORK. You cannot detect "is this the development build?" by looking at how-# long the function names are. Measured on the real pair: the development bundle-# had a median react-dom function-name length of 19 with 97.6% of names at four-# characters or more, and the PRODUCTION bundle had a median of 19 with 95.4%.-# React's own minifier renames only part of react-dom, `keepNames` preserves the-# rest, and V8 reports an inferred name for much of what is left. Name shape-# tells you nothing. The checks below are structural instead.+# NOTE FOR THE NEXT PERSON: you cannot detect 'is this the development build?' from function-name+# length. Measured on the real pair, the development bundle had a median react-dom name length of+# 19 with 97.6% of names four characters or more, and the PRODUCTION bundle a median of 19 with+# 95.4%. The checks below are structural instead.@dataclass(frozen = True)@@ -123,8 +117,8 @@ class Bridge:ambiguous_prod: list[str] = field(default_factory = list)ambiguous_dev: list[str] = field(default_factory = list)unmatched_prod: int = 0- # Resolved functions whose dev name equals their prod name. Expected to be- # small and non-zero; near 100% means both arms were the same build.+ # Resolved functions whose dev name equals their prod name. Expected to be small and non-zero;+ # near 100% means both arms were the same build.identity_mappings: int = 0anchors_checked: int = 0anchor_failures: list[str] = field(default_factory = list)@@ -132,9 +126,8 @@ class Bridge:@staticmethoddef prod_key(url: str, start_offset: int, end_offset: int) -> str:- # The URL is reduced to its basename because an Unsloth install serves the- # same bundle from a hashed path that changes per install, while the- # offsets inside the bundle do not.+ # The URL is reduced to its basename because an Unsloth install serves the same bundle from a+ # hashed path that changes per install, while the offsets inside it do not.return f"{os.path.basename(url)}:{start_offset}:{end_offset}"def resolve(self, url: str, start_offset: int, end_offset: int) -> str | None:@@ -408,16 +401,12 @@ def build_bridge()return bridge+ # The two arms must actually be two different builds. THE FAILURE THIS CLOSES: if both arms are+ # pointed at the same server, every count vector matches trivially, EVERY ANCHOR PASSES because+ # an anchor genuinely does map to itself, and the bridge reports `ok` with a mapping of `Zk` to+ # `Zk`. The anchor check cannot catch it: anchors test invariance and same-build is perfectly+ # invariant.# ---- the two arms must actually be two different builds ------------------ #- # THE FAILURE THIS CLOSES. If both arms are pointed at the same server (a- # port collision, a reused base URL, a dev server that quietly served the- # production dist), every count vector matches trivially, EVERY ANCHOR- # PASSES because an anchor genuinely does map to itself, and the bridge- # reports `ok` with a mapping of `Zk` to `Zk`. That is a clean-looking run- # that proves nothing, and it would put "bridged" next to a minified name.- # The anchor check cannot catch it; it is the one thing anchors are blind- # to, because anchors test invariance and same-build is perfectly invariant.dev_all_keys = {v.key for v in dev_all if v.informative}prod_all_keys = {v.key for v in prod_all if v.informative}if dev_all_keys and dev_all_keys == prod_all_keys:@@ -461,10 +450,8 @@ def build_bridge()return bridge- # Second guard on the same failure, for the case where the two arms differ- # in offsets but are still the same code (a rebuild of one tree, say). If- # nearly every resolved name maps to itself, no minification was undone and- # the bridge is not doing anything.+ # Second guard on the same failure, for arms that differ in offsets but are still the same code:+ # if nearly every resolved name maps to itself, no minification was undone.fraction = identical_names / matchedif fraction > MAX_IDENTITY_MAPPING_FRACTION:bridge.status = FAILEDdiff --git a/tests/studio/studiobench/analysis/test_analysis.py b/tests/studio/studiobench/analysis/test_analysis.pyindex 77e96cf5fed..426fda196e2 100644--- a/tests/studio/studiobench/analysis/test_analysis.py+++ b/tests/studio/studiobench/analysis/test_analysis.py@@ -60,8 +60,8 @@ def test_loads_object_form_trace() -> None:def test_truncated_trace_fails_the_cell_rather_than_parsing_short() -> None:- # A stream that was cut off mid-document must not degrade into "a shorter- # trace"; that reads exactly like the expensive work not happening.+ # A stream cut off mid-document must not degrade into 'a shorter trace'; that reads exactly like+ # the expensive work not happening.try:Trace.from_json_text('{"traceEvents":[{"ph":"X","ts":1,"dur":2,"name":"RunTask"')except CellFailure as exc:@@ -127,8 +127,8 @@ def test_profile_parses_and_deltas_match_wall_clock() -> None:def test_nodes_accumulate_across_chunks() -> None:- # Only a handful of chunks carry a `nodes` array; a parser that reads nodes- # per chunk and forgets them resolves almost nothing.+ # Only a handful of chunks carry a `nodes` array; a parser that reads nodes per chunk and forgets+ # them resolves almost nothing.tr = _trace()chunks_with_nodes = sum(1@@ -178,8 +178,8 @@ def test_stacks_have_ancestry() -> None:def test_clock_anchor_is_the_event_timestamp() -> None:- # `args.data.startTime` is in a different clock domain and is kept only to- # report the skew, never used to anchor samples.+ # `args.data.startTime` is in a different clock domain and is kept only to report the skew, never+ # used to anchor samples.prof = C.main_thread_profile(_trace())assert prof.declared_start_timeassert abs(prof.clock_skew_us) < 10_000@@ -228,8 +228,8 @@ def test_timers_and_frames_and_input_are_all_present() -> None:def test_harness_cost_is_named_not_hidden() -> None:cls = K.classify_thread(_trace())- # The devtools pipe running Runtime.evaluate is our own cost. It must have- # its own column so it can be watched, and must not be inside the app's.+ # The devtools pipe running Runtime.evaluate is our own cost. It must have its own column so it+ # can be watched, and must not be inside the app's.assert K.AGENT_IPC in K.ORIGINSassert K.AGENT_IPC in K.HARNESS_ORIGINS@@ -434,9 +434,8 @@ def test_bridge_resolves_a_minified_name_by_count_vector() -> None:assert b.status == S.OKassert b.resolve("/assets/index-abc123.js", 0, 5) == "cloneChildFibers"assert b.resolve("/assets/index-abc123.js", 10, 15) == "beginWork"- # The anchor legitimately maps to itself, so some identity is expected. What- # must not happen is identity DOMINATING, which would mean no minification- # was undone.+ # The anchor legitimately maps to itself, so some identity is expected; what must not happen is+ # identity DOMINATING, which would mean no minification was undone.assert b.identity_mappings / len(b.mapping) <= S.MAX_IDENTITY_MAPPING_FRACTION@@ -484,10 +483,9 @@ def test_anchor_mismatch_discards_the_whole_bridge() -> None:def test_same_build_on_both_arms_is_refused() -> None:- # The one failure anchors are structurally blind to. If both arms are the- # same build, every anchor maps to itself PERFECTLY, so anchor validation- # passes and the bridge reports ok while mapping minified names to- # themselves. Verified against the real implementation before this guard+ # The one failure anchors are structurally blind to: if both arms are the same build every anchor+ # maps to itself PERFECTLY, so anchor validation passes and the bridge reports ok while mapping+ # minified names to themselves. Verified against the real implementation before this guard# existed: it returned status ok with {"Zk": "Zk"}._, prod = _arms({"a": [1, 1]},@@ -512,8 +510,8 @@ def test_same_build_on_both_arms_is_refused() -> None:def test_an_all_identity_mapping_is_refused() -> None:- # Different scripts and offsets, but no minification was undone: every- # resolved name is its own name, so the bridge is doing nothing.+ # Different scripts and offsets, but no minification was undone: every resolved name is its own+ # name, so the bridge is doing nothing.dev, prod = _arms({"Zk": [340, 3400], "qi": [17, 170]},{"Zk": [340, 3400], "qi": [17, 170]},@@ -645,8 +643,8 @@ def test_missing_page_counters_are_skipped_loudly() -> None:def test_page_counter_contract_is_documented() -> None:- # Layer 3 emits against these exact key names; a rename here is a contract- # change and must be agreed, not discovered at analysis time.+ # Layer 3 emits against these exact key names; a rename here is a contract change and must be+ # agreed, not discovered at analysis time.assert set(O.PAGE_COUNTER_CONTRACT) == {"m2_reparse", "m3_forced_layout"}assert "chars_rescanned" in O.PAGE_COUNTER_CONTRACT["m2_reparse"]assert "forced_layouts" in O.PAGE_COUNTER_CONTRACT["m3_forced_layout"]diff --git a/tests/studio/studiobench/analysis/test_instruments_live.py b/tests/studio/studiobench/analysis/test_instruments_live.pyindex 92a5d842de7..1ecf5484485 100644--- a/tests/studio/studiobench/analysis/test_instruments_live.py+++ b/tests/studio/studiobench/analysis/test_instruments_live.py@@ -45,8 +45,8 @@ from studiobench.runtime.types import ( # noqa: E402new_session_id,)-# A page with a known shape: a message-channel loop (the React scheduler's-# mechanism), a timer loop, and a function whose call count is exactly known.+# A page with a known shape: a message-channel loop (the React scheduler's mechanism), a timer+# loop, and a function whose call count is exactly known.PAGE = """<!doctype html><meta charset=utf-8><body><div id=o></div><script>window.__N = 0;window.__junk = [];@@ -205,8 +205,8 @@ def _check_level(level, wrows, crows, ground_truth) -> int:except Exception as exc: # noqa: BLE001fail(f"window row is not JSON-safe: {exc}")- # Overhead is mandatory from every instrument at level >= 1, and it is what- # the report layer's overhead_growth_with_length gate consumes.+ # Overhead is mandatory from every instrument at level >= 1, and it is what the report layer's+ # overhead_growth_with_length gate consumes.for name, payload in crows.items():if "overhead_ms" not in payload or "overhead_ms_attempted" not in payload:fail(f"{name}.end_cell has no overhead_ms/overhead_ms_attempted")@@ -225,8 +225,8 @@ def _check_level(level, wrows, crows, ground_truth) -> int:if by_origin.get("message_channel", 0) < MESSAGE_ITERATIONS * 0.8:fail(f"expected ~{MESSAGE_ITERATIONS} message-channel tasks, got {by_origin}")- # THE LADDER. Naming requires the v8 profiler, which only L2+ turns on. At- # L1 that must be an explicit null with a reason, never an empty list.+ # THE LADDER. Naming requires the v8 profiler, which only L2+ turns on; at L1 that must be an+ # explicit null with a reason, never an empty list.if level == 1:if tracing.get("named_frames") is not None:fail("L1 must not report named frames; the profiler category is off")diff --git a/tests/studio/studiobench/analysis/traceparse.py b/tests/studio/studiobench/analysis/traceparse.pyindex e9a0a8e9d34..35f3c6a6cb7 100644--- a/tests/studio/studiobench/analysis/traceparse.py+++ b/tests/studio/studiobench/analysis/traceparse.py@@ -37,18 +37,17 @@ from typing import Any, Iterable, Iterator, Sequencefrom . import CellFailure-# Complete-duration events are the only phase that forms the task tree. Async-# (`b`/`e`/`n`), flow (`s`/`t`/`f`), instant (`I`/`R`), sample (`P`), counter-# (`C`) and metadata (`M`) events are carried alongside but never nested, since-# their timestamps do not describe a stack.+# Complete-duration events are the only phase that forms the task tree. Async (`b`/`e`/`n`), flow+# (`s`/`t`/`f`), instant (`I`/`R`), sample (`P`), counter (`C`) and metadata (`M`) events are+# carried alongside but never nested, since their timestamps do not describe a stack._PHASE_COMPLETE = "X"_PHASE_BEGIN = "B"_PHASE_END = "E"-# When two events share both `ts` and `dur`, the trace gives no ordering. On the-# main thread the devtools view (`RunTask`) is conceptually the outer frame and-# the scheduler view (`ThreadControllerImpl::RunTask`) the inner one, so pin-# that order rather than letting dict ordering decide which is the parent.+# When two events share both `ts` and `dur` the trace gives no ordering. On the main thread the+# devtools view (`RunTask`) is conceptually the outer frame and the scheduler view+# (`ThreadControllerImpl::RunTask`) the inner one, so pin that order rather than letting dict+# ordering decide._OUTERMOST_FIRST = {"RunTask": 0,"ThreadControllerImpl::RunTask": 1,@@ -137,7 +136,7 @@ class Trace:self._thread_names[key] = name # type: ignore[index]self._threads: dict[tuple[int, int], Thread] = {}- # ---------------------------------------------------------------- loading+ # loading@classmethoddef from_json_text(cls, text: str) -> "Trace":@@ -154,8 +153,8 @@ class Trace:try:doc = json.loads(text)except json.JSONDecodeError as exc:- # A truncated JSON document is the signature of a drained stream that- # was cut short. That is a failed cell, never a short trace.+ # A truncated JSON document is the signature of a drained stream that was cut short: a failed cell,+ # never a short trace.raise CellFailure("trace_truncated",f"trace JSON did not parse ({exc}); {len(text)} bytes drained",@@ -303,9 +302,9 @@ def build_tree(events: Iterable[dict[str, Any]]) -> list[Task]:for t in complete:while stack and t.ts >= stack[-1].end:stack.pop()- # An event that starts inside its would-be parent but ends after it is- # not nested; the trace is inconsistent there. Treat it as a sibling- # rather than corrupting self-time arithmetic for the whole subtree.+ # An event that starts inside its would-be parent but ends after it is not nested; the trace is+ # inconsistent there, so treat it as a sibling rather than corrupting self-time arithmetic for the+ # whole subtree.while stack and t.end > stack[-1].end:stack.pop()if stack:diff --git a/tests/studio/studiobench/arms/calibration.py b/tests/studio/studiobench/arms/calibration.pyindex 56efe748037..4671b3627ae 100644--- a/tests/studio/studiobench/arms/calibration.py+++ b/tests/studio/studiobench/arms/calibration.py@@ -41,13 +41,13 @@ from ..scoring.schema import Measurefrom .manifest import Arm, ArmOutcome, ArmStatus, Invariance, PotencyCounter#: The spike sizes every batch runs. 0.1 ms is below any plausible per-update mechanism and is-#: expected to be INVISIBLE; it is included precisely so the detection floor has something to sit-#: on. 2.0 ms is comfortably above, and must be seen or the instrument is broken.+#: expected to be INVISIBLE, so the detection floor has something to sit on; 2.0 ms is comfortably+#: above and must be seen or the instrument is broken.SPIKE_SIZES_MS: tuple[float, ...] = (0.1, 0.5, 2.0)-#: A spike is considered recovered when the observed delta is this close to the burned cost.-#: Wide on purpose: the point is "the instrument sees roughly the right amount", not calibration-#: to three digits, and a tight band would fail on a machine that is merely noisy.+#: A spike is considered recovered when the observed delta is this close to the burned cost. Wide on+#: purpose: the point is "roughly the right amount", not calibration to three digits, and a tight+#: band would fail on a merely noisy machine.RECOVERY_MIN = 0.5RECOVERY_MAX = 2.0diff --git a/tests/studio/studiobench/arms/content_visibility_probe.js b/tests/studio/studiobench/arms/content_visibility_probe.jsindex bb4260be4b9..815164a4de7 100644--- a/tests/studio/studiobench/arms/content_visibility_probe.js+++ b/tests/studio/studiobench/arms/content_visibility_probe.js@@ -1,68 +1,34 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-/* POTENCY PROBE for `content-visibility: auto` on Unsloth's chat message roots.+/* POTENCY PROBE for `content-visibility: auto` on Unsloth's chat message roots. Installed through+ * `SBENCH_EXTRA_INIT_SCRIPT`, reporting through `SBENCH_PAGE_CONSOLE`; needs `--password` (and+ * `--password-b`, which defaults to `--password` and is wrong when two Unsloth instances are+ * attached, since each mints its own bootstrap password). A run carrying this is a PROBE RUN and+ * its payload is never scored: the probe forces layout on every sample, including the rendering it+ * asks about.+ * Without it the run dies on an HTTP 401 only after the browser has started. The invocation is+ * SBENCH_EXTRA_INIT_SCRIPT=tests/studio/studiobench/arms/content_visibility_probe.js+ * SBENCH_PAGE_CONSOLE="CVPOT " python -m tests.studio.studiobench --tier fast --ab probe+ * --attach URL --attach-b URL --password ... --password-b ...; see the studiobench README.*- * Installed through `SBENCH_EXTRA_INIT_SCRIPT`, reporting through `SBENCH_PAGE_CONSOLE`:+ * WHY A PROBE AT ALL. A null from an arm that never fired measures the cascade, or a selector, or+ * the fact that nothing was off screen -- a null already produced twice here. So nothing times+ * anything; it answers only whether the browser genuinely skipped rendering an off-screen subtree.+ * Three independent answers, weakest first: `cvAuto` (computed `auto`, proving only that the+ * declaration won the cascade), `skipEvents` (`contentvisibilityautostatechange` with `skipped`,+ * fired by the engine alone, so a non-zero count is proof), and `offUnrendered` (descendants with+ * no layout boxes -- the route everybody reaches for first, which DOES NOT WORK; kept only so its+ * zero is on the record. See `descendantBoxes`).*- * SBENCH_EXTRA_INIT_SCRIPT=tests/studio/studiobench/arms/content_visibility_probe.js \- * SBENCH_PAGE_CONSOLE="CVPOT " \- * python -m tests.studio.studiobench --tier fast --ab probe --out outputs/probe \- * --attach http://127.0.0.1:PORT --attach-b http://127.0.0.1:OTHER \- * --password "$(cat "$STUDIO_HOME_A/auth/.bootstrap_password")" \- * --password-b "$(cat "$STUDIO_HOME_B/auth/.bootstrap_password")"+ * IT ALSO WATCHES THE SIZING TRAP. Size containment while skipping means a skipped root's height is+ * its `contain-intrinsic-size`; with `auto` that is the LAST REMEMBERED SIZE (css-sizing-4 5.2),+ * and a root never rendered without containment falls back to the <length>. Both wreck scroll+ * geometry and they differ, so scrollHeight, the roots on the fallback and the roots on their+ * PADDING ALONE (a remembered size of zero) are reported per sample, read against the unarmed side.*- * The credential flags are spelled out rather than trailed off with an ellipsis. A probe run- * drives a real Unsloth like every other command in the loop, and without `--password` it dies on- * an HTTP 401 only after the browser has already started. See "You need an Unsloth, and you need- * its password" in the studiobench README.- *- * ONE PASSWORD PER ARM when both arms are attached. Two separately booted Unsloth instances mint two- * different bootstrap passwords, so reusing the first for the treatment is a 401 on the second- * arm only, after the browser is up. `--password-b` defaults to `--password`, which is right- * for the single-Unsloth case and wrong for this one.- *- * A run carrying this is a PROBE RUN and its payload is never scored: the probe forces layout on- * every sample, and one of the things it forces is the very rendering it is asking about. See the- * warning on `descendantBoxes`.- *- * WHY A PROBE AT ALL, AND WHY IT RUNS BEFORE ANY TIMING IS COLLECTED.- *- * A null from an arm that never fired is not a measurement of the mechanism; it is a measurement- * of the cascade, or of a selector, or of the fact that nothing was ever off screen. This exact- * null has already been produced twice in this codebase for `content-visibility`. So nothing here- * times anything. It answers one question: did the browser genuinely skip rendering an off-screen- * subtree, on the elements this arm targets, in the running app.- *- * THREE INDEPENDENT ANSWERS, WEAKEST FIRST.- *- * 1. cvAuto elements whose COMPUTED content-visibility is `auto`. This proves the- * declaration won the cascade and nothing more. An element can compute to- * `auto` and be painted on every single frame because it never stopped- * being relevant to the user.- * 2. skipEvents `contentvisibilityautostatechange` events with `skipped === true`. This- * event is fired by the engine's own relevance machinery and by nothing- * else: no stylesheet, no selector and no author code can produce one. A- * non-zero count is proof that a subtree was locked and its rendering- * skipped.- * 3. offUnrendered off-screen armed roots whose element descendants generate NO layout- * boxes. This is the route everybody reaches for first and it DOES NOT- * WORK for `content-visibility: auto`. It is kept, and reported, only so- * that its zero is on the record next to a non-zero skip count rather than- * being rediscovered. See `descendantBoxes`.- *- * IT ALSO WATCHES THE SIZING TRAP. `content-visibility: auto` imposes size containment while- * skipping, so a skipped root's height is its `contain-intrinsic-size`. With the `auto` keyword- * that is the LAST REMEMBERED SIZE (css-sizing-4 5.2, 5.2.1), and an element that has never been- * rendered without size containment has none and falls back to the <length>. Both failure modes- * wreck scroll geometry on a long thread and they are different, so the viewport's scrollHeight,- * the count of roots sitting on the declared fallback, and the count sitting on their PADDING- * ALONE (a remembered size of zero) are all reported every sample, to be read against the- * unarmed side of the same session.- *- * OUT OF THE PAGE VIA THE CONSOLE. Unsloth ships `connect-src 'self'`, so a beacon to a collector- * on another port is blocked by CSP before it is sent. The console is the one channel that costs- * nothing and cannot be silently dropped.+ * Reporting goes out via the console: Unsloth ships `connect-src \'self\'`, so a beacon to another+ * port is blocked by CSP before it is sent.*/(function () {"use strict";@@ -73,32 +39,25 @@var SAMPLE_MS = 2000;var MAX_ROOTS_SCANNED = 60;var MAX_DESCENDANTS_PER_ROOT = 40;- /* Per role, the two heights a skipped root can land on, and they mean OPPOSITE things:- *- * fallback the declared `contain-intrinsic-size` length, used because no last remembered- * size exists yet. The arm is working as designed and the length is a guess.- * padding the root's own padding and nothing else, which is a remembered size of ZERO,- * recorded while the root was mounted and still empty. That is the trap.- *- * They have to be told apart or the probe attributes one to the other, and on this app they- * are close enough to collide: the user root's fallback is 60px against 40px of padding. So- * each is matched against its OWN target height, the fallback first so that it wins ties, and- * a root that is neither is counted as neither. A single `height <= 64` bucket put a user root- * sitting exactly on its fallback into both counters at once. */+ /* Per role, the two heights a skipped root can land on, meaning OPPOSITE things: `fallback` is the+ * declared `contain-intrinsic-size`, used because no last remembered size exists yet (working as+ * designed); `padding` is the root's own padding alone, a remembered size of ZERO recorded while+ * the root was mounted and empty, which is the trap. On this app they collide -- the user root's+ * fallback is 60px against 40px of padding -- so each is matched against its OWN target height,+ * fallback first so it wins ties, and a root that is neither is counted as neither. A single+ * `height <= 64` bucket put a user root sitting exactly on its fallback into both counters. */var ROLE_PX = {assistant: { fallback: 300, padding: 18 },user: { fallback: 60, padding: 40 }};- /* `getBoundingClientRect().height` is the BORDER BOX, so both targets carry the root's own- * padding: an assistant root on its 300px fallback measures 318, and one whose remembered- * size is zero measures 18. Comparing the rect against the bare declared length instead left- * `fallbackBite` structurally pinned at zero, which reads as "the fallback is never used" no- * matter what the browser did. */+ /* `getBoundingClientRect().height` is the BORDER BOX, so both targets carry the root's padding: an+ * assistant root on its 300px fallback measures 318. Comparing against the bare declared length+ * pinned `fallbackBite` at zero, reading as "the fallback is never used". */function targetHeight(px, which) {return (which === "fallback" ? px.fallback : 0) + px.padding;}- /* Half the gap between the two smallest interesting heights, so neither test can reach the- * other's target. Sub-pixel layout means an exact equality test would miss. */+ /* Half the gap between the two smallest interesting heights, so neither test can reach the other's+ * target; sub-pixel layout means an exact equality test would miss. */var PX_EPS = 2;if (typeof window === "undefined" || !window.document) {@@ -112,8 +71,8 @@var doc = window.document;var watched = [];var watchedSet = typeof WeakSet === "function" ? new WeakSet() : null;- /* element -> its record, so a sample can ask whether THIS root is currently skipped. Weak so- * that a root the app has thrown away is not held alive by the probe. */+ /* element -> its record, so a sample can ask whether THIS root is skipped. Weak so a discarded root+ * is not held alive by the probe. */var recOf = typeof WeakMap === "function" ? new WeakMap() : null;var ev = { stateChange: 0, skip: 0, unskip: 0, watchers: 0, listenerErrors: 0 };var seq = 0;@@ -156,8 +115,8 @@return a.bottom > b.top && a.top < b.bottom && a.right > b.left && a.left < b.right;}- /* Attached BEFORE anything is read, and exactly once per element. The event is the only- * signal here that no amount of author CSS can fake. */+ /* Attached BEFORE anything is read and exactly once per element: the one signal here that no author+ * CSS can fake. */function watch(el) {try {if (watchedSet) {@@ -196,13 +155,10 @@/* How many of this element's first `cap` element descendants generate a layout box.*- * A KNOWN FALSE NEGATIVE, KEPT ON PURPOSE. The reasoning is that a skipped subtree is not laid- * out, so its descendants have no boxes. The reasoning is correct and the measurement is still- * useless, because ASKING is what breaks it: `getClientRects()` on content inside a locked- * subtree makes Chromium render that subtree in order to answer, so the probe unlocks exactly- * what it came to observe. Measured on a 100K thread this returned 0 off-screen unrendered- * roots while the event counter recorded 22 roots simultaneously in the skipped state. Read- * alone it is a clean, confident, wrong "the arm did not fire". Use `ev_skip`. */+ * A KNOWN FALSE NEGATIVE, KEPT ON PURPOSE. A skipped subtree is not laid out, so its descendants+ * have no boxes -- but ASKING breaks it: `getClientRects()` inside a locked subtree makes Chromium+ * render it to answer. On a 100K thread this returned 0 off-screen unrendered roots while the event+ * counter recorded 22 roots simultaneously skipped. Use `ev_skip`. */function descendantBoxes(el, cap) {var kids;try {@@ -224,10 +180,8 @@return n;}- /* Whether the browser last told us THIS root is skipping its contents.- *- * `null` means no transition has been seen, which is not the same as "rendered" and must not- * be read as either. Callers that care about the difference check for `true` explicitly. */+ /* Whether the browser last told us THIS root is skipping its contents. `null` means no transition+ * has been seen, which is not "rendered"; callers that care check for `true` explicitly. */function skippedState(el) {var rec = null;try {@@ -309,20 +263,15 @@var r = rect(el);if (r) {heights.push(Math.round(r.height));- /* Two MUTUALLY EXCLUSIVE buckets, in priority order. `content-visibility: auto`- * imposes size containment while skipping, so a skipped root's height is its- * padding plus its intrinsic size. Landing on the declared <length> means no- * remembered size exists yet; landing on the padding alone means one exists and- * it is ZERO, recorded while the root was mounted and still empty. The fallback- * is tested first and wins ties, because a root sitting on its fallback is- * behaving exactly as the declaration asks and must never be charged to the+ /* Two MUTUALLY EXCLUSIVE buckets, in priority order: a skipped root's height is its padding plus+ * its intrinsic size, so landing on the declared <length> means no remembered size exists yet+ * and landing on the padding alone means one exists and is ZERO. The fallback is tested first+ * and wins ties, since such a root behaves exactly as declared and must not be charged to the* trap. Anything else is counted as neither. */- /* ONLY WHILE SKIPPED. `content-visibility: auto` computes to `auto` whether or- * not the element is currently skipping, and size containment applies only while- * it is. An armed root that is on screen has its ordinary rendered height, and if- * that height happens to land within the tolerance of a role target it would be- * charged to the remembered-size trap. Ordinary geometry must not be able to- * masquerade as the finding. */+ /* ONLY WHILE SKIPPED: `content-visibility: auto` computes to `auto` whether or not the element+ * is skipping, and size containment applies only while it is. An on-screen armed root has its+ * ordinary rendered height, which must not be able to land in a role target and masquerade as+ * the finding. */var px = ROLE_PX[roleOf(el)];if (cv === "auto" && px && skippedState(el) === true) {if (Math.abs(r.height - targetHeight(px, "fallback")) <= PX_EPS) {@@ -332,8 +281,8 @@}}}- /* The geometry question is asked only of armed roots, and only of as many as can be- * asked cheaply: this forces layout, and the probe must not become the load. */+ /* Asked only of armed roots, and only of as many as can be asked cheaply: this forces layout, and+ * the probe must not become the load. */if (cv === "auto" && r && vpRect && scanned < MAX_ROOTS_SCANNED) {scanned += 1;var hasOwnBox = r.width > 0 || r.height > 0;@@ -366,13 +315,10 @@}}- /* CONNECTED roots only, and the detached ones are dropped as we go.- *- * `thread_reopen` is in the film: it tears the thread down and rebuilds it, so the old- * roots leave the document. A detached root receives no further transitions, so one whose- * last event said `skipped` would go on being counted in `skippedNow` for the rest of the- * session, reporting roots as skipped that are not in the page at all. Pruning here also- * stops `watched` growing a strong reference per root for the life of a long run. */+ /* CONNECTED roots only, dropping the detached ones as we go. `thread_reopen` rebuilds the thread,+ * so old roots leave the document and receive no further transitions; one whose last event said+ * `skipped` would be counted in `skippedNow` for the rest of the session. Pruning also stops+ * `watched` growing a strong reference per root. */var live = [];for (i = 0; i < watched.length; i++) {var wel = watched[i].el;@@ -430,27 +376,20 @@window.__cvPotSample = sample;- /* THE LISTENER HAS TO EXIST BEFORE THE ELEMENT'S FIRST TRANSITION, and polling cannot- * guarantee that.- *- * `contentvisibilityautostatechange` fires on a CHANGE of state. A root that is inserted- * off screen becomes skipped once, at its first lifecycle update, and then never changes- * again while the user stays where they are. If the listener is attached on a two-second- * tick, every root mounted and skipped inside that first tick emits its only event into a- * void, `ev_skip` stays at zero, and the probe reports precisely the false NOT RUN it was- * written to prevent. A whole thread can mount inside two seconds; a seeded one does.+ /* THE LISTENER HAS TO EXIST BEFORE THE ELEMENT'S FIRST TRANSITION, and polling cannot guarantee+ * that. `contentvisibilityautostatechange` fires on a CHANGE: a root inserted off screen becomes+ * skipped once and never changes again while the user stays put, so on a two-second tick every+ * root mounted inside the first tick emits its only event into a void and `ev_skip` stays zero --+ * the false NOT RUN this file exists to prevent. A whole thread can mount inside two seconds.*- * So roots are adopted at INSERTION, from a MutationObserver installed before the app boots.- * The observer callback runs in a microtask after the mutation and before the next lifecycle- * update, which is early enough. The interval is kept, because a root can also acquire the- * property later without being re-inserted, and `adoptAll` is idempotent.+ * So roots are adopted at INSERTION, from a MutationObserver installed before the app boots; its+ * callback runs in a microtask before the next lifecycle update. The interval is kept because a+ * root can acquire the property later without being re-inserted, and `adoptAll` is idempotent.*- * OBSERVE THE DOCUMENT, NOT `documentElement`. An init script is evaluated after the Document- * exists but before the page's own scripts run, and at that instant the parser may not have- * created the root element yet. `observe(null, ...)` throws, the catch below would swallow it,- * and adoption would silently fall back to the two-second tick: the exact false zero this- * whole arrangement exists to prevent, reintroduced by the fix for it. A Document node is a- * valid target and it always exists here. */+ * OBSERVE THE DOCUMENT, NOT `documentElement`: an init script runs after the Document exists but+ * possibly before the parser creates the root element, and `observe(null, ...)` would throw into+ * the catch below, silently falling back to the tick -- the same false zero, reintroduced by the+ * fix for it. */function adoptAll() {var roots = all(MESSAGE_SELECTOR);for (var i = 0; i < roots.length; i++) {@@ -458,10 +397,9 @@}}- /* Only the ADDED NODES, never a document-wide re-scan. Streaming produces thousands of- * mutations a second, and a `querySelectorAll` per mutation would make this probe the load- * it is trying to observe. `watch` is idempotent, so a node reached twice costs a WeakSet- * lookup. */+ /* Only the ADDED NODES, never a document-wide re-scan: streaming produces thousands of mutations a+ * second and a `querySelectorAll` per mutation would make this probe the load. `watch` is+ * idempotent, so a node reached twice costs a WeakSet lookup. */function adoptAdded(records) {for (var i = 0; i < records.length; i++) {var added = records[i].addedNodes;diff --git a/tests/studio/studiobench/arms/dose.py b/tests/studio/studiobench/arms/dose.pyindex 892a3edf128..12435d25bc7 100644--- a/tests/studio/studiobench/arms/dose.py+++ b/tests/studio/studiobench/arms/dose.py@@ -40,12 +40,12 @@ from typing import Any, Mapping, Sequencefrom ..scoring.schema import Measure-#: The declared doses. Four points spanning three orders of magnitude, because a two-point-#: "line" cannot be distinguished from a step and a three-point one cannot show curvature.+#: The declared doses. Four points spanning three orders of magnitude, because a two-point "line"+#: cannot be distinguished from a step and a three-point one cannot show curvature.DOSES: tuple[int, ...] = (4, 40, 400, 4000)#: A fit this far from the through-origin model is called nonlinear rather than reported as a-#: slope. Loose, because the question is "is this O(children)", not "what is the exact constant".+#: slope. Loose, because the question is "is this O(children)", not the exact constant.MIN_R2_FOR_LINE = 0.90#: An intercept larger than this fraction of the largest-dose cost means most of the cost is not@@ -128,8 +128,8 @@ def _r2(xs: Sequence[float], ys: Sequence[float], predict) -> float | None:ss_tot = sum((y - mean) ** 2 for y in ys)ss_res = sum((y - predict(x)) ** 2 for x, y in zip(xs, ys))if ss_tot <= 0:- # Every reading identical. R2 is undefined, and reporting 1.0 (a perfect fit) for a flat- # line is exactly the wrong way round: this is the case with the least information.+ # Every reading identical. R2 is undefined, and reporting 1.0 for a flat line is exactly the wrong+ # way round: this is the case with the least information.return Nonereturn 1.0 - (ss_res / ss_tot)@@ -213,10 +213,10 @@ def fit_dose_response(largest_cost = max(ys)floor = detection_floor_ms- # Detectability is judged on the FREE slope, not the through-origin one. A perfectly flat- # series still has a large through-origin slope, because forcing the line through zero makes- # the constant offset look like a per-child cost; judging detectability on that number turns- # every null result into a positive identification.+ # Detectability is judged on the FREE slope, not the through-origin one: a perfectly flat series+ # still has a large through-origin slope, because forcing the line through zero makes the+ # constant offset look like a per-child cost, which turns every null result into a positive+ # identification.detectable = (slope_free is not Noneand floor is not None@@ -246,10 +246,10 @@ def fit_dose_response(and largest_cost > 0and abs(intercept_free) / largest_cost > MAX_INTERCEPT_FRACTION):- # Checked BEFORE the through-origin R2, because a straight line with a big offset fits a- # free line perfectly and a through-origin line badly. Reporting that as NONLINEAR would- # be exactly backwards: the data is as linear as data gets, it just does not pass through- # zero, and the offset is the finding.+ # Checked BEFORE the through-origin R2, because a straight line with a big offset fits a free line+ # perfectly and a through-origin line badly. Reporting that as NONLINEAR would be backwards: the+ # data is as linear as data gets, it just does not pass through zero, and the offset is the+ # finding.verdict = "MOSTLY FIXED COST"note = (f"the free fit puts {abs(intercept_free) / largest_cost:.0%} of the largest-dose cost "diff --git a/tests/studio/studiobench/arms/knobs.js b/tests/studio/studiobench/arms/knobs.jsindex 6d4297dbc35..3b8d8b93ed5 100644--- a/tests/studio/studiobench/arms/knobs.js+++ b/tests/studio/studiobench/arms/knobs.js@@ -4,181 +4,80 @@/** knobs.js -- runtime ablation knobs for the SHIPPED Unsloth Studio build.*- * Injected with Playwright's context.add_init_script BEFORE the app boots, so it can patch- * browser APIs the app is about to use. It needs no compiler, no source edit and no rebuild: the- * point is that an external tester can run every ablation against the same production bundle the- * user gets, and that the ablated run and the control run are the same bytes of application code.+ * Injected with Playwright's context.add_init_script BEFORE the app boots, so an external tester+ * can run every ablation against the same production bundle the user gets, with the ablated and+ * control runs sharing the same bytes of application code.*- * WHAT AN ARM IS FOR+ * Each arm removes ONE candidate mechanism and nothing else, so a cheaper thread names the fix:*- * Each arm removes one candidate mechanism and nothing else. If removing it makes the thread- * cheaper, that mechanism was the cost, and the arm names the fix. The arms:- *- * A visibility:hidden on completed messages.- * Removes paint and raster for the prefix. Keeps layout, keeps the DOM, keeps React.- * IF A IS THE WIN: the cost is paint/raster of off-screen content. The fix is to stop- * painting the prefix, not to stop building it.- *- * B content-visibility:auto plus contain-intrinsic-size on completed messages, and the undo- * of the index.css override that force-disables content-visibility on code blocks.- * Removes style, layout AND paint for anything off screen.- * IF B IS THE WIN: the cost is off-screen style plus layout plus paint, and the rule at- * studio/frontend/src/index.css:2537 is wrong -- it was kept for a flicker, and the comment- * above it asserts containment on message roots "was measured as no help", which is exactly- * the claim this arm re-tests. B beating A says the expensive part is not the pixels.- *- * C display:none on completed messages.- * Removes layout geometry and the sibling count as well as everything B removes.- * IF C IS THE WIN AND B IS NOT: the cost is layout geometry and the cost of having N- * siblings in one flow, not per-element style. That points at the thread container's layout- * mode (flex column over N children) rather than at the messages.- *- * D detach the autoscroll subtree MutationObserver.- * use-intent-aware-autoscroll.tsx:502 observes the viewport with subtree+characterData, and- * its callback reads el.scrollHeight, which forces synchronous style+layout of the WHOLE- * thread on every delivery. During streaming those deliveries arrive per streamed character.- * IF D IS THE WIN: the cost is forced synchronous layout inside the autoscroll observer.- * The fix is to stop reading scrollHeight per mutation (rAF-coalesce it, or switch to- * IntersectionObserver / overflow-anchor), not to touch the messages at all.- *- * E neutralise the --aui-scroll-stabilizer custom property write.- * That same callback writes a custom property on the viewport element. A custom property is- * inherited, so writing it on the scroll container invalidates inherited style for the whole- * subtree beneath it, which is every message.- * IF E IS THE WIN: the cost is inherited custom-property subtree style invalidation. The fix- * is to register the property with syntax and inherits:false via CSS.registerProperty, or to- * move the write off the ancestor of the thread.- *- * F freeze React's scheduler while leaving the DOM exactly as it is.- * React's browser scheduler drives work through a MessageChannel: port1.onmessage is- * performWorkUntilDeadline, and port2.postMessage schedules the next slice. Suppressing the- * delivery stops reconciliation, effects and store subscriptions without touching a node.- * IF F IS THE WIN: the cost is React subscriptions and reconciliation over the prefix, and- * the fix is memoisation / virtualisation / narrower context subscriptions, not CSS.- *- * G CONTROL. Identical DOM, no knob, thread scrolled so prior turns are actually IN the- * viewport. Every other arm removes work that only exists when the prefix is off screen and- * cheap; G is the run where the prefix is on screen and expensive. It is the calibration for- * "how much does this page cost when nothing is being skipped", and a G that is not slower- * than the untouched baseline means the baseline was never skipping anything, which would+ * A visibility:hidden on completed messages -- removes paint/raster only. If A wins, stop+ * painting the prefix rather than building it.+ * B content-visibility:auto + contain-intrinsic-size, undoing the index.css:2537 override --+ * removes off-screen style, layout AND paint. B beating A says the pixels are not the cost,+ * and re-tests that override's claim that containment on message roots "was no help".+ * C display:none -- also removes layout geometry and the sibling count. C winning where B does+ * not points at the thread container's layout mode, not at per-element style.+ * D detach the autoscroll subtree MutationObserver (use-intent-aware-autoscroll.tsx:502), whose+ * callback reads el.scrollHeight and so forces synchronous layout of the whole thread per+ * streamed character. If D wins, rAF-coalesce that read rather than touching the messages.+ * Observed with subtree+characterData, so it fires per streamed character.+ * E neutralise the --aui-scroll-stabilizer write: a custom property is inherited, so writing it+ * on the scroll container invalidates inherited style for every message. If E wins, register+ * it with inherits:false or move the write off the thread's ancestor.+ * If E wins, `CSS.registerProperty` with inherits:false is the fix.+ * F freeze React's scheduler (MessageChannel port1.onmessage = performWorkUntilDeadline),+ * leaving the DOM untouched. If F wins the fix is memoisation / virtualisation, not CSS.+ * G CONTROL: identical DOM, no knob, prior turns actually IN the viewport. A G that is not+ * slower than the baseline means the baseline was never skipping anything, which would* invalidate the whole comparison.*- * THE TWO FAILURE MODES THIS FILE EXISTS TO PREVENT- *- * See arms/manifest.py. An ablation lies in exactly two ways and both are silent.- *- * 1. THE ARM CHANGED THE OUTPUT. The knob also removed some of the work, so the two sides did- * not render the same page. The dangerous case is the small invisible difference: an earlier- * stub fixture in this codebase produced 552 syntax-highlighted spans where the real page- * produces 2,561, and it read as a clean 4x win. It was not fast, it was rendering a fifth- * of the content. digest() and counts() exist for that: the canonical serialisation carries- * the code-block count and the span count per message, so a highlighting difference is- * caught even when every character of text is identical.- *- * TWO THINGS THE DIGEST CANNOT SEE, stated here because they cannot be stated in it. It- * walks `[data-message-id]` roots and serialises their attributes and text plus the count of- * code blocks and of spans inside them. It does NOT serialise the attributes of descendants,- * so an inline style written on a code block is invisible to it (arm B does exactly that,- * which is one reason B is EQUIVALENT and not EXACT). And it does not know about geometry,- * so a change that only moves pixels is invisible too. Neither gap is patched by making the- * digest bigger; they are declared, because a normaliser nobody wrote down is just a bug.- *- * Arms A and C therefore go out of their way to mutate NOTHING: they target the completed- * prefix with `:nth-child(-n+K)` against the viewport's direct children rather than by- * tagging elements, so the raw digest is byte-identical across the arm and EXACT is a claim- * they can actually make. When that is impossible they fall back to a marker attribute and- * say in the returned reason that they have dropped to EQUIVALENT.- *- * 2. THE ARM DID NOT FIRE. Selector matched nothing, patch failed to install, run completed,- * difference was zero, and "no effect" got written down as evidence against the mechanism.- * It is evidence of nothing. So every arm has a potency counter that the arm CAUSES: not- * "the stylesheet was appended" but "N elements actually compute to visibility:hidden".- * A patch that could not be installed lands in unavailable[] and never in available[], so- * UNAVAILABLE can never be misread as "had no effect".- *- * CASCADE FACTS THIS FILE DEPENDS ON, VERIFIED AGAINST THE SPEC- *- * The index.css override lives inside `@layer utilities { ... }` (Tailwind v4 emits native- * cascade layers). Per CSS Cascading and Inheritance Level 5 and MDN:- *- * - For IMPORTANT declarations the layer order is REVERSED, and important declarations in ANY- * layer beat important declarations outside every layer. So appending an unlayered- * `!important` rule, at any specificity, LOSES to the layered `!important` rule it is trying- * to undo. The obvious approach silently does nothing, which is failure mode 2.- * - Element-attached (inline) declarations are sorted BEFORE layers in the cascade, so an- * inline `!important` declaration beats every other author declaration whatever its layer.- *- * Therefore each CSS arm applies its stylesheet in two copies, one unlayered and one inside- * `@layer utilities` so it competes in the same layer as the rule it must beat (same layer, same- * importance, higher specificity, later order -> wins), THEN verifies with getComputedStyle, and- * only if the verification says the rule did not take effect does it escalate to inline- * !important, counting the escalation. The verification is the whole point: the stylesheet is a- * hypothesis about the cascade, the computed style is the observation.- *- * ARMS ARE APPLIED ONCE, OVER THE PREFIX THAT EXISTS AT THAT MOMENT- *- * apply() runs at the start of a measured window and never again. Messages created during the- * window are NOT ablated. That is deliberate and not a limitation:- *- * - the hypothesis is about the PREFIX (the N completed turns that are off screen), not about- * the turn being streamed, so ablating the in-flight message would change the thing under- * measurement rather than the thing under test;- * - a live MutationObserver re-applying the knob to each new message would add per-mutation- * work to the hot path, which is the exact category of cost the experiment is trying to- * remove. An ablation that installs its own observer to remove an observer measures itself.- *- * For the same reason arm B marks only code blocks inside COMPLETED messages: touching the code- * block that is still streaming would change how streamdown and shiki finalise it.- *- * PREBOOT VERSUS RUNTIME- *- * D, E and F patch APIs the app captures during boot, so they must be installed before the app- * runs and they are read from window.__sbArmConfig.preboot, set by an earlier init script. An arm- * that is not listed is NOT INSTALLED AT ALL. Installing a patch and leaving it inactive is not- * free: the wrapper costs a call frame on every MutationObserver.observe or setProperty in the- * process, and a control run carrying that overhead is not a control. So the preboot list decides- * per page load, and a run whose config did not arm D reports D as unavailable with a reason,- * never as an arm that ran and did nothing.- *- * D and E are ACTIVE FROM INJECTION when armed; there is no meaningful apply() for them, because- * the autoscroll observer is installed once during mount and the suppression has to be in place- * before that. apply("D") therefore reports the arm as already active and returns the suppression- * count so far. F is different: the port capture is preboot, the freeze is toggled by apply("F").- *- * ARM F IS DOM_CHANGING. Freezing React stops the stream from rendering, so the frozen run is not- * rendering the same page as the control -- it is rendering less of it. Its cost is therefore an- * upper bound on what React reconciliation costs and can never be quoted as a point estimate.- * manifest.py prints it as `<= x`. This is stated here as well because the arm is the one most- * likely to produce a spectacular number that somebody wants to quote.- *- * COUNTERS: EVENT COUNTERS VERSUS GAUGES+ * THE TWO FAILURE MODES THIS FILE EXISTS TO PREVENT (see arms/manifest.py); both are silent.*- * potency() returns integers only, in two kinds, and the distinction is load-bearing:+ * 1. THE ARM CHANGED THE OUTPUT. An earlier stub fixture rendered 552 highlighted spans where+ * the real page renders 2,561 and read as a clean 4x win. digest() and counts() catch that:+ * the canonical form carries code-block and span counts per message, so a highlighting+ * difference is caught even when every character of text matches. Two declared gaps: the+ * digest does not serialise descendants' attributes (so arm B's inline style on a code block+ * is invisible, which is why B is EQUIVALENT and not EXACT) and it knows nothing about+ * geometry. Arms A and C therefore mutate NOTHING, targeting the prefix with+ * `:nth-child(-n+K)` so EXACT is a claim they can make; when that is impossible they fall+ * back to a marker attribute and say in the returned reason that they dropped to EQUIVALENT.+ * 2. THE ARM DID NOT FIRE, and "no effect" got written down as evidence. So every arm has a+ * potency counter the arm CAUSES ("N elements compute to visibility:hidden", not "the+ * stylesheet was appended"), and a patch that could not be installed lands in unavailable[]+ * so UNAVAILABLE can never be misread as "had no effect".*- * EVENT COUNTERS are monotonic and never decrease for the life of the page:- * suppressedViewportObserves, observeCallsTotal, observeCallsPassedThrough,- * suppressedStabilizerSets, customPropSets, setPropertyCallsTotal,- * suppressedSchedulerCallbacks, schedulerCallbacksDelivered, capturedSchedulerPorts, and the- * bookkeeping counters (armsApplied, inlineFallbacks, ...).+ * CASCADE FACTS, verified against CSS Cascading and Inheritance Level 5. The index.css override+ * sits inside `@layer utilities`, and for IMPORTANT declarations layer order is REVERSED, so an+ * unlayered `!important` rule LOSES at any specificity -- the obvious approach silently does+ * nothing, which is failure mode 2. Inline declarations sort BEFORE layers, so an inline+ * `!important` beats every author declaration. Each CSS arm therefore emits its sheet twice (once+ * unlayered, once inside `@layer utilities`), verifies with getComputedStyle, and only escalates+ * to inline !important when the verification says the rule did not take, counting the escalation.*- * GAUGES are point-in-time measurements of the live DOM, re-measured on every potency() call- * unless {live:false} is passed: visibilityHiddenConfirmed, cvAutoMessages, cvAutoCodeBlocks,- * contentVisibilityAutoConfirmed, displayNoneConfirmed, controlVisibleMessages,- * controlVisiblePriorMessages.+ * ARMS ARE APPLIED ONCE, over the prefix that exists at that moment. Messages created during the+ * window are NOT ablated, deliberately: the hypothesis is about the completed off-screen prefix,+ * and a live MutationObserver re-applying the knob would add per-mutation work to the hot path --+ * an ablation that installs an observer to remove an observer measures itself. For the same reason+ * arm B marks only code blocks inside COMPLETED messages.*- * Gauges are deliberately not monotonic. A stored 0 and a measured 0 are different facts, and- * this file exists to keep them different: measuring live means the "before" read is a real- * observation of the untouched page rather than a variable that has not been written yet, and it- * means arm G is checked against what the page looks like at the END of the window, after the- * app's own autoscroll has had every chance to undo the scroll position G set. Re-measuring calls- * getComputedStyle over every message root, which forces style recalc, so potency() must be- * called OUTSIDE the timed window; pass {live:false} to read the last stored values with no DOM- * access at all.+ * PREBOOT VERSUS RUNTIME. D, E and F patch APIs the app captures during boot and are read from+ * window.__sbArmConfig.preboot. An arm not listed is NOT INSTALLED AT ALL, because an inactive+ * wrapper still costs a call frame on every observe or setProperty and a control run carrying that+ * overhead is not a control. D and E are active from injection (apply("D") reports the arm already+ * active); F captures the port at preboot and toggles the freeze in apply("F").*- * HASHING+ * COUNTERS. potency() returns integers in two kinds. EVENT COUNTERS are monotonic for the life of+ * the page (suppressedViewportObserves, observeCallsTotal, suppressedStabilizerSets,+ * suppressedSchedulerCallbacks, capturedSchedulerPorts, the bookkeeping counters, ...). GAUGES are+ * point-in-time reads of the live DOM, re-measured on every call unless {live:false}+ * (visibilityHiddenConfirmed, cvAutoMessages, displayNoneConfirmed, controlVisibleMessages, ...).+ * A stored 0 and a measured 0 are different facts: measuring live keeps the "before" read a real+ * observation, and checks arm G at the END of the window after the app's autoscroll has had every+ * chance to undo the scroll position. Re-measuring calls getComputedStyle over every message root,+ * forcing style recalc, so potency() must be called OUTSIDE the timed window.*- * FNV-1a, 32 bit, hex. NOT CRYPTOGRAPHIC and not a security boundary. It is a change detector- * against an adversary that does not exist, and 32 bits is small enough that a collision on a+ * HASHING: FNV-1a, 32 bit, hex. Not cryptographic, and small enough that a collision on a* multi-megabyte canonical form is not impossible, so digest() also returns canonicalLength and* rawLength. Compare the pair, not the hash alone.*/@@ -192,9 +91,8 @@var W = window;- // Idempotence. add_init_script runs for every document including iframes, and a driver that- // reloads the page must not stack patches: a doubly wrapped observe() would double every- // count, and a doubly wrapped MessageChannel would make the freeze flag ambiguous.+ // Idempotence: add_init_script runs for every document including iframes, and double+ // wrapping would double every count and make the freeze flag ambiguous.if (W.__sbArmsInstalled) {return;}@@ -221,12 +119,8 @@var DEFAULT_CONTROL_VISIBLE_TARGET = 3;var MAX_STYLESHEET_DEPTH = 8;- // ------------------------------------------------------------------------------------- // Config- //- // Read defensively. A missing or malformed __sbArmConfig means no preboot arms, which is the- // correct default: no config is not permission to install patches into a control run.- // ------------------------------------------------------------------------------------+ // A missing or malformed __sbArmConfig means no preboot arms: absent config is not+ // permission to patch a control run.var cfg = {};try {@@ -302,21 +196,15 @@doc = null;}- // ------------------------------------------------------------------------------------- // State- // ------------------------------------------------------------------------------------function freshEventCounters() {return {- // DobserveCallsTotal: 0,observeCallsPassedThrough: 0,suppressedViewportObserves: 0,- // EsetPropertyCallsTotal: 0,customPropSets: 0,suppressedStabilizerSets: 0,- // FcapturedSchedulerPorts: 0,schedulerCallbacksDelivered: 0,suppressedSchedulerCallbacks: 0,@@ -326,7 +214,6 @@droppedSchedulerEvents: 0,schedulerListenerAdds: 0,schedulerCallbacksWithoutHandler: 0,- // bookkeepingarmsApplied: 0,armsReverted: 0,styleSheetsInstalled: 0,@@ -353,13 +240,13 @@var gauges = freshGauges();var state = {- applied: {}, // armId -> {at, affected, reason}+ applied: {},frozen: false,- markers: [], // {el, prev} for MARKER_ATTR restoration- inline: [], // {el, prop, prevValue, prevPriority, hadStyleAttr}- sheets: {}, // armId -> <style> element- channels: [], // captured MessageChannel records- scrollRestore: null, // arm G restoration record+ markers: [],+ inline: [],+ sheets: {},+ channels: [],+ scrollRestore: null,originals: {observe: null,observeDescriptor: null,@@ -402,13 +289,8 @@return { reverted: !!reverted, reason: String(reason || "") };}- // ------------------------------------------------------------------------------------- // DOM helpers- //- // All of them tolerate a missing document, a detached node and a selector that matches- // nothing, and none of them throw. A query that fails returns an empty array, and the caller- // turns that into a reason string rather than into a zero.- // ------------------------------------------------------------------------------------+ // None of these throw on a missing document, detached node or empty match; a failed+ // query returns [] and the caller turns that into a reason, not a zero.function toArray(nodeList) {var out = [];@@ -454,9 +336,8 @@}function isRunning(el) {- // The ONLY DOM signal of an in-flight message is a descendant (or the root itself)- // carrying [data-status="running"]; status.type from assistant-ui is one of- // running|complete|incomplete|requires-action.+ // The only DOM signal of an in-flight message is [data-status="running"] on the root or a+ // descendant (assistant-ui status: running|complete|incomplete|requires-action).try {if (!el) {return false;@@ -466,18 +347,16 @@}return queryOne(RUNNING_SELECTOR, el) !== null;} catch (e) {- // Unknown status is treated as running, i.e. as NOT ablatable. Erring the other way- // would ablate the message being streamed and contaminate the measurement.+ // Unknown status counts as running, i.e. not ablatable: the other way would ablate the+ // streaming message and contaminate the measurement.return true;}}- // The completed-message filter is done in JS, one querySelector per root, and NOT with a CSS- // `:has()` selector. `:has()` is unsupported on older engines and an unsupported compound- // selector makes the WHOLE selector list invalid, so the rule would be dropped at parse time,- // match nothing, and the arm would report a clean zero difference. That is precisely the- // did-not-fire failure this file is built to make impossible, and it is invisible in a- // screenshot, in a trace and in the console.+ // Filtering in JS rather than CSS `:has()`: an unsupported compound selector invalidates the+ // whole selector list, so the rule would be dropped at parse time and the arm would report a+ // clean zero difference.+ // One querySelector per root.function completedMessageRoots() {var roots = messageRoots();var out = [];@@ -567,22 +446,14 @@} catch (e) {/* fall through */}- // No CSS.supports means we cannot prove support. Treat as supported and let the computed- // check decide: a false "unavailable" would hide an arm that works.+ // No CSS.supports means support cannot be proven; assume supported and let the computed+ // check decide, since a false UNAVAILABLE hides an arm that works.return true;}- // ------------------------------------------------------------------------------------- // Markers and inline styles- //- // Arms A, B and C need a CSS hook on specific elements, and the only way to have a stylesheet- // address "these elements and no others" is an attribute the stylesheet can select. One- // attribute name is used for all arms, space separated, so [data-sb-arm~="A"] matches. That- // attribute IS a change to the DOM, so it shows up in digest().raw. It is meant to: an arm- // that mutates attributes is EQUIVALENT, not EXACT, and the Python side declares- // `attr:data-sb-arm` (plus `attr:style` when the inline escalation fires) as its allowed diff.- // Pretending otherwise by hiding the marker from the raw digest would defeat the check.- // ------------------------------------------------------------------------------------+ // One space-separated attribute serves arms A, B and C ([data-sb-arm~="A"]). It IS a DOM+ // change and appears in digest().raw deliberately: such an arm is EQUIVALENT, not EXACT,+ // and Python declares `attr:data-sb-arm` (plus `attr:style`) as its allowed diff.function markElement(el, armId) {try {@@ -693,8 +564,7 @@left = "x";}if (left === "") {- // Leaving style="" behind would be a permanent, invisible difference in- // every future digest of this page.+ // Leaving style="" behind would be a permanent, invisible difference in every future digest of this page.rec.el.removeAttribute("style");}}@@ -705,23 +575,10 @@state.inline = [];}- // ------------------------------------------------------------------------------------- // Stylesheets- //- // Every arm sheet is emitted TWICE: once unlayered, once inside `@layer utilities`.- //- // The rule that has to be beaten (index.css:2536-2538) is inside `@layer utilities`, and for- // important declarations layered beats unlayered no matter the specificity. The layered copy- // therefore competes in the same layer as its target, where the ordinary rules apply again:- // same layer, same importance, higher specificity, later in document order, so it wins. The- // unlayered copy covers the case where the build has no layers at all (then there is nothing- // layered to lose to, and the unlayered copy is the one that applies). Emitting both is safe- // because the two copies carry identical declarations, so whichever wins produces the same- // computed value.- //- // Appending to <head> puts the sheet last among same-layer rules, which settles order of- // appearance. None of this is trusted: applyX() checks getComputedStyle afterwards.- // ------------------------------------------------------------------------------------+ // Every arm sheet is emitted twice, unlayered and inside `@layer utilities`. The rule to beat+ // (index.css:2536-2538) is layered, and for important declarations layered beats unlayered at+ // any specificity; the unlayered copy covers a build with no layers. Nothing is trusted:+ // applyX() rechecks getComputedStyle afterwards.function ensureStyleSheet(armId, cssBody) {try {@@ -763,32 +620,13 @@}}- // ------------------------------------------------------------------------------------- // Two ways to address "the completed messages" from a stylesheet, and why the first one is- // worth the extra code.- //- // PREFIX PATH (preferred). Messages are DIRECT CHILDREN of the viewport; there is no- // message-list wrapper element. The message being streamed is the last one. So the completed- // messages are a leading run of the viewport's children and- //- // .aui-thread-viewport.aui-stream-viewport > [data-message-id]:nth-child(-n+K)- //- // selects exactly them, with K computed in JS as the child index of the first running message- // root. This mutates NOTHING. No attribute is added, no inline style is written, and the raw- // digest of the thread is byte-identical before and after, so arms A and C can honestly claim- // EXACT invariance. The `[data-message-id]` part is not redundant: nth-child alone would also- // match spacers, sentinels and buttons that share the viewport, and hiding one of those is a- // change to the rendered page that the digest cannot see because the digest only walks- // message roots.- //- // MARKER PATH (fallback). If the completed set is not a leading prefix (a message somewhere- // in the middle is still running, which assistant-ui allows in principle), or the viewport- // cannot be found, the arm falls back to a `data-sb-arm` attribute on each completed root.- // That is a real DOM mutation, so the arm drops from EXACT to EQUIVALENT and the returned- // reason string says so in as many words. It is reported rather than hidden because a- // silently-EQUIVALENT arm quoted as EXACT is a false number, while a loudly-EQUIVALENT one is- // just a number with a declared diff.- // ------------------------------------------------------------------------------------+ // PREFIX PATH (preferred): messages are direct children of the viewport and the streamed one+ // is last, so a `> [data-message-id]:nth-child(-n+K)` selector picks exactly the completed+ // run. It mutates nothing, so the raw digest is byte-identical and arms A and C can claim+ // EXACT. `[data-message-id]` is needed: nth-child alone also matches spacers and buttons.+ // MARKER PATH (fallback): when the completed set is not a leading prefix, or no viewport is+ // found, each completed root is marked with `data-sb-arm`; the arm drops to EQUIVALENT and+ // says so in its reason.function cssPrefixA(k) {return (@@ -833,9 +671,8 @@"\tdisplay: none !important;\n" +"}\n";- // Returns the leading run of viewport children that contains no running message, or null when- // the prefix path is not usable. `k` is a 1-based nth-child bound; `roots` are the message- // roots inside it, all of them completed by construction.+ // Returns the leading run of viewport children with no running message, else null. `k` is a+ // 1-based nth-child bound; `roots` are the completed roots inside it.function prefixInfo() {try {var vp = viewportEl();@@ -895,9 +732,6 @@}}- // ------------------------------------------------------------------------------------- // Hashing and canonical serialisation- // ------------------------------------------------------------------------------------var imul =typeof Math.imul === "function"@@ -910,8 +744,8 @@return ((al * bl + (((ah * bl + al * bh) << 16) >>> 0)) | 0);};- // FNV-1a, 32 bit, over UTF-16 code units taken low byte first. Non-cryptographic: this is a- // change detector, not a security boundary. Compare it together with canonicalLength.+ // FNV-1a 32-bit over UTF-16 code units, low byte first. Non-cryptographic: compare it+ // together with canonicalLength.function fnv1a32(str) {var h = 0x811c9dc5;for (var i = 0; i < str.length; i++) {@@ -924,9 +758,8 @@return ("0000000" + (h >>> 0).toString(16)).slice(-8);}- // Field encoding. JSON escapes quotes, backslashes, newlines and control characters; the pipe- // is escaped on top of that so no encoded field can ever contain the field separator, which- // makes the canonical form parseable by a plain split() in diffKeys().+ // JSON escapes quotes, backslashes, newlines and controls; the pipe is escaped on top, so no+ // field can contain the separator and diffKeys() can split() the canonical form.function enc(s) {try {return JSON.stringify(String(s === null || s === undefined ? "" : s))@@ -949,8 +782,8 @@if (!skipStyleProps || !skipStyleProps.length) {return value;}- // Parse with the browser's own CSS parser on a detached element rather than by splitting- // on ";" and ":", which breaks on url(data:...) and on any value containing a semicolon.+ // Parsed by the browser's CSS parser on a detached element: splitting on ";" and ":" breaks+ // on url(data:...) and any value containing a semicolon.try {if (doc && typeof doc.createElement === "function") {var scratch = doc.createElement("div");@@ -986,9 +819,8 @@return kept.join("; ");}- // One DOM walk, reused for both serialisations. Walking twice would be slower and, worse,- // would let the page change between the raw pass and the normalised pass, producing a "raw- // differs, normalised does not" that is an artefact of the instrument.+ // One DOM walk feeds both serialisations: walking twice lets the page change between the raw+ // and normalised passes, faking a raw-only difference.function collectRecords() {var roots = messageRoots();var records = [];@@ -1153,9 +985,8 @@out.codeSpans = collected.codeSpans;var rawKeys = {};- // `raw` skips nothing at all: it is the answer to "did anything about the rendered- // output change", including the arm's own marker attribute. An EXACT arm that touches- // an attribute is not an EXACT arm, and this is where that gets caught.+ // `raw` skips nothing, including the arm's own marker attribute: an EXACT arm that touches an+ // attribute is not EXACT, and this is where that is caught.var rawCanonical = serialise(collected, [], [], rawKeys);var normKeys = {};@@ -1177,9 +1008,8 @@if (keepCanonical) {if (rawCanonical.length > maxCanonicalChars || normCanonical.length > maxCanonicalChars) {- // A prefix is retained because it is useful when debugging, but `truncated`- // is set and diffKeys() refuses to answer from a truncated pair. A quiet cut- // would turn "we did not look at the rest" into "the rest was identical".+ // A prefix is kept for debugging but `truncated` is set and diffKeys() refuses a truncated+ // pair: a quiet cut would turn "not looked at" into "identical".out.truncated = true;}out.canonicalRaw = rawCanonical.slice(0, maxCanonicalChars);@@ -1192,27 +1022,16 @@return out;}- // ------------------------------------------------------------------------------------- // diffKeys- //- // Compares the RAW canonical forms, not the normalised ones. The normalised hashes already- // answer "are these equivalent"; what manifest.py needs from here is the other question,- // "what actually differs", so it can check the observed diff against the diff the arm- // DECLARED it would produce. An arm that declares one difference and produces two is voided,- // and comparing normalised forms would hide the second one, since the normaliser is exactly- // the thing that removes the declared difference.- //- // Keys are collapsed to a stable vocabulary (`attr:<name>`, `text`, `tag`,- // `structure:<what>`) rather than per-message paths, because a declared diff has to be- // writable in advance and `#msg-8fc2.attr.style` is not knowable in advance.- //- // Fails closed: if the canonical form was not retained, or was truncated, or cannot be- // parsed, the returned array contains a key beginning with `__unavailable:` which can never- // appear in a declared diff, so the arm is voided instead of silently passing.- // ------------------------------------------------------------------------------------+ // Compares the RAW canonical forms: the normalised hashes already answer "equivalent?", while+ // manifest.py needs what actually differs to check it against the DECLARED diff. Normalised+ // forms would hide a second difference, since the normaliser removes the declared one.+ // Keys collapse to a stable vocabulary (`attr:<name>`, `text`, `tag`, `structure:<what>`): a+ // declared diff must be writable in advance, and `#msg-8fc2.attr.style` is not.+ // Fails closed: an unretained, truncated or unparseable canonical form yields a+ // `__unavailable:` key, which can never appear in a declared diff, so the arm voids.function parseCanonical(text) {- var entries = {}; // entryId -> {key: value}+ var entries = {};var indexToId = {};var lines = String(text).split("\n");for (var i = 0; i < lines.length; i++) {@@ -1337,7 +1156,7 @@}// Beyond the required API: the same comparison with examples, for a human reading a VOIDED- // verdict who needs to know WHICH message drifted and by how much.+ // verdict who needs to know which message drifted and by how much.function diffDetail(a, b, limit) {var cap = typeof limit === "number" && limit > 0 ? Math.floor(limit) : 20;var out = [];@@ -1397,9 +1216,6 @@return out;}- // ------------------------------------------------------------------------------------- // counts()- // ------------------------------------------------------------------------------------function counts() {var out = {@@ -1414,10 +1230,8 @@chars: 0,domNodes: 0,visibleMessages: 0,- // Beyond the required keys, and required by the zero discipline: visibleMessages == 0- // because the thread viewport was not found is a different fact from- // visibleMessages == 0 because nothing is on screen, and the two must not print the- // same.+ // Zero discipline: visibleMessages == 0 from a missing thread viewport is a different fact+ // from nothing being on screen, and must not print the same.viewportFound: false};try {@@ -1462,10 +1276,8 @@var spans = queryAll("span", blocks[b]);out.codeSpans += spans.length;for (var s = 0; s < spans.length; s++) {- // A shiki token is a leaf span with text. Both numbers are reported: a- // fixture that renders a fifth of the highlighting moves both, and a change- // in span NESTING moves only codeSpans, so keeping them separate says which- // of the two happened.+ // A shiki token is a leaf span with text. Both numbers are reported: partial highlighting+ // moves both, a change in span NESTING moves only codeSpans.var sp = spans[s];var hasElementChild = false;try {@@ -1500,9 +1312,6 @@return out;}- // ------------------------------------------------------------------------------------- // potency()- // ------------------------------------------------------------------------------------function measureGauges() {try {@@ -1565,9 +1374,6 @@return out;}- // ------------------------------------------------------------------------------------- // Arm A -- visibility:hidden on completed messages- // ------------------------------------------------------------------------------------function applyA() {var info = prefixInfo();@@ -1602,8 +1408,8 @@}var confirmed = countComputed(roots, "visibility", "hidden");if (confirmed === 0) {- // The stylesheet lost the cascade. Inline important is sorted before layers and beats- // every author rule, so this is the escalation that cannot lose.+ // The stylesheet lost the cascade. Inline important sorts before layers and beats every+ // author rule, so this escalation cannot lose.counters.inlineFallbacks += 1;for (var i = 0; i < roots.length; i++) {setInline(roots[i], "visibility", "hidden", "important");@@ -1630,22 +1436,14 @@);}- // ------------------------------------------------------------------------------------- // Arm B -- content-visibility:auto plus contain-intrinsic-size- //- // Part (i) restores content-visibility on code blocks, undoing index.css:2536-2538.- // Part (ii) puts content-visibility:auto and contain-intrinsic-size:auto <measured>px inline- // on each completed message root.- //- // THE ORDERING BUG THIS FUNCTION IS SHAPED TO AVOID: content-visibility:auto makes an- // off-screen element skip its contents, so the moment it is set, offsetHeight collapses to the- // contain-intrinsic-size placeholder. Reading offsetHeight after setting the property captures- // the placeholder, which is then fed back as the intrinsic size, and every element ends up- // claiming to be 200px (or 0) tall. The thread's scroll height collapses, the scrollbar jumps,- // the autoscroll observer fires on the reflow, and the run measures a page that is not the- // page. So the heights are all read in one pass FIRST and only then written in a second pass.- // Two passes also avoid read/write layout thrash, but that is a bonus, not the reason.- // ------------------------------------------------------------------------------------+ // (i) restores content-visibility on code blocks, undoing index.css:2536-2538; (ii) puts+ // content-visibility:auto and contain-intrinsic-size inline on each completed message root.+ // THE ORDERING BUG THIS FUNCTION AVOIDS: content-visibility:auto makes an off-screen element+ // skip its contents, so an offsetHeight read AFTER setting it returns the+ // contain-intrinsic-size placeholder, which then feeds back as the intrinsic size: scroll+ // height collapses, the scrollbar jumps, the autoscroll observer fires and the run measures a+ // page that is not the page. So heights are read in one pass and written in a second.+ // The placeholder claims 200px, or 0.function readHeights(els) {var heights = [];@@ -1672,9 +1470,8 @@);}- // Only code blocks inside COMPLETED messages. The block that is still streaming must not- // be touched: changing containment on it would change how streamdown and shiki finalise- // it, which is a change to the output, not an ablation of cost.+ // Only code blocks in COMPLETED messages: changing containment on the streaming block would+ // change how streamdown and shiki finalise it, altering the output rather than its cost.var blocks = [];for (var r = 0; r < roots.length; r++) {var bs = codeBlocksIn(roots[r]);@@ -1684,13 +1481,15 @@}// READ PASS. Every height is captured before any property is written.+ // THE ORDERING BUG THIS AVOIDS: content-visibility:auto makes an off-screen element skip its+ // contents, so offsetHeight collapses to the contain-intrinsic-size placeholder the moment it+ // is set. Read afterwards, that placeholder feeds back as the intrinsic size.var blockHeights = readHeights(blocks);var rootHeights = readHeights(roots);- // WRITE PASS. B writes inline styles on the roots either way, so it is EQUIVALENT- // whatever happens here; the prefix selector is still preferred because it keeps the- // code blocks free of a marker attribute that the digest cannot see (the canonical form- // serialises message-root attributes and DESCENDANT COUNTS, not descendant attributes).+ // B writes inline styles either way, so it is EQUIVALENT regardless; the prefix selector is+ // still preferred because it keeps a marker attribute off the code blocks, which the canonical+ // form cannot see (it serialises root attributes and descendant COUNTS).if (info) {ensureStyleSheet("B", cssPrefixB(info.k));} else {@@ -1699,8 +1498,7 @@}var i;for (i = 0; i < blocks.length; i++) {- // contain-intrinsic-size has to be inline anyway because it is per element, and the- // stylesheet only carries content-visibility.+ // contain-intrinsic-size has to be inline because it is per element; the stylesheet only carries content-visibility.setInline(blocks[i],"contain-intrinsic-size",@@ -1711,10 +1509,9 @@var cvBlocks = countComputed(blocks, "content-visibility", "auto");if (blocks.length > 0 && cvBlocks === 0) {- // Expected on this build: index.css:2537 sets content-visibility:visible !important- // from inside @layer utilities, and for important declarations a layered rule beats- // an unlayered one at any specificity. The layered copy of CSS_B should win; if it- // did not, inline important is the escalation that the cascade guarantees.+ // Expected here: index.css:2537 sets content-visibility:visible !important inside @layer+ // utilities, and a layered important rule beats an unlayered one at any specificity, so the+ // layered copy of CSS_B should win; if not, inline important is the guaranteed escalation.counters.inlineFallbacks += 1;for (i = 0; i < blocks.length; i++) {setInline(blocks[i], "content-visibility", "auto", "important");@@ -1754,9 +1551,6 @@);}- // ------------------------------------------------------------------------------------- // Arm C -- display:none on completed messages- // ------------------------------------------------------------------------------------function applyC() {var info = prefixInfo();@@ -1815,26 +1609,15 @@);}- // ------------------------------------------------------------------------------------- // Arm D -- detach the autoscroll subtree observer (PREBOOT)- //- // Two discriminators identify the autoscroll observer, and both come from- // use-intent-aware-autoscroll.tsx:502: the target carries the aui-stream-viewport class, and- // the options carry "aria-expanded" in attributeFilter. Nothing else in the app observes with- // aria-expanded, and .aui-stream-viewport marks only real streaming-thread viewports.- //- // EVERY OTHER observe() CALL MUST PASS THROUGH UNCHANGED. reasoning.tsx,- // research-activity-panel.tsx, animated-theme-toggler.tsx, tooltip-modal-layer.ts (twice),- // settings-dialog.tsx, monitor-frame-store.ts and use-composer-pill-fit.ts all install- // observers, and breaking any of them would change what the app renders, which converts an- // ablation into a different page.- //- // The wrapper forwards `arguments` verbatim to the original, so calls with missing or invalid- // arguments throw the same TypeError from the same function they always did. The matching- // logic cannot throw: if reading options.attributeFilter fails for any reason, the call is- // treated as not-matching and passes through, because passing an observe() through is always- // safe and suppressing one that should not have been suppressed is not.- // ------------------------------------------------------------------------------------+ // Two discriminators identify the autoscroll observer, both from+ // use-intent-aware-autoscroll.tsx:502: target has .aui-stream-viewport and options carry+ // "aria-expanded" in attributeFilter. Nothing else in the app observes with aria-expanded.+ // EVERY OTHER observe() CALL MUST PASS THROUGH UNCHANGED (reasoning, research panel, theme+ // toggler, tooltip layer, settings dialog, monitor store, composer pill fit): breaking one+ // turns an ablation into a different page.+ // The wrapper forwards `arguments` verbatim, so invalid calls throw the same TypeError from+ // the same function. The matcher cannot throw: an unreadable options.attributeFilter counts as+ // not-matching and passes through, since passing an observe() through is always safe.function targetIsStreamViewport(target) {try {@@ -1945,15 +1728,9 @@return true;}- // ------------------------------------------------------------------------------------- // Arm E -- neutralise the --aui-scroll-stabilizer write (PREBOOT)- //- // One custom property, by exact name. Every other setProperty call, custom property or not,- // passes through with its arguments forwarded verbatim so that coercion and throwing are- // unchanged. customPropSets counts every custom property write including the suppressed ones,- // which is the context that tells a reader whether the stabilizer was one write in ten or one- // in ten thousand.- // ------------------------------------------------------------------------------------+ // One custom property, by exact name; every other setProperty call forwards its arguments+ // verbatim so coercion and throwing are unchanged. customPropSets counts every+ // custom-property write including suppressed ones, so a reader can see the rate.function installE() {var proto = null;@@ -1989,8 +1766,8 @@counters.setPropertyCallsTotal += 1;var name = null;try {- // A Symbol argument makes String() throw. In that case name stays null, the call- // passes through, and the original throws exactly the TypeError it would have.+ // A Symbol argument makes String() throw; name stays null, the call passes through, and the+ // original throws exactly the TypeError it would have.name = typeof property === "string" ? property : String(property);} catch (e) {name = null;@@ -2033,35 +1810,19 @@return true;}- // ------------------------------------------------------------------------------------- // Arm F -- freeze the React scheduler, keep the DOM (PREBOOT capture, apply() freezes)- //- // THIS ARM IS DOM_CHANGING. While frozen, React renders nothing, so the stream stops- // appearing on screen and the frozen side is not rendering the same page as the control. Its- // cost is an UPPER BOUND on what React reconciliation costs and can never be quoted as a point- // estimate; manifest.py prints it as `<= x`. It is here because a bound on React is still- // worth having when the alternative is guessing, not because the number is clean.- //- // React's browser scheduler is the only runtime handle available. The ReactDOMRoot object is- // not exposed on window, so root.unmount() is not reachable, and #root is just an element.- // The scheduler does: const channel = new MessageChannel(); channel.port1.onmessage =- // performWorkUntilDeadline; ... port2.postMessage(null). Intercepting the assignment to- // port1.onmessage puts a dispatcher in front of performWorkUntilDeadline, and the freeze flag- // decides whether the dispatcher forwards.- //- // capturedSchedulerPorts is what separates "React never used MessageChannel on this build"- // (0 ports: NOT RUN, the treatment never happened) from "we froze it and nothing changed"- // (ports captured, callbacks suppressed).- //- // SUPPRESSING SCHEDULER MESSAGES CAN WEDGE REACT PERMANENTLY. The message loop only posts the- // next message from inside the handler it just ran, so a swallowed delivery ends the loop and- // unfreezing alone does not restart it. revert() therefore posts one fresh message per- // channel that had suppressions (schedulerResumeKicks) unless config.redeliverOnUnfreeze is- // false, and says so in the returned reason. The suppressed events themselves are NOT- // replayed: React's performWorkUntilDeadline ignores the event object and reads its own- // queue, so one kick resumes exactly the work the swallowed messages would have done, and- // replaying N events would run N slices that no longer correspond to anything.- // ------------------------------------------------------------------------------------+ // THIS ARM IS DOM_CHANGING: while frozen React renders nothing, so it is not rendering the+ // control's page. Its cost is an UPPER BOUND on reconciliation, never a point estimate;+ // manifest.py prints it as `<= x`.+ // React's browser scheduler is the only runtime handle: ReactDOMRoot is not on window, and the+ // scheduler assigns performWorkUntilDeadline to channel.port1.onmessage, so intercepting that+ // assignment puts a dispatcher in front of it.+ // capturedSchedulerPorts separates "React never used MessageChannel on this build" (0 ports,+ // NOT RUN) from "we froze it and nothing changed".+ // SUPPRESSING SCHEDULER MESSAGES CAN WEDGE REACT PERMANENTLY: the loop posts the next message+ // only from inside the handler it just ran, so revert() posts one fresh kick per channel that+ // had suppressions unless config.redeliverOnUnfreeze is false. Suppressed events are not+ // replayed: performWorkUntilDeadline reads its own queue, so one kick resumes the work.+ // Counted as `schedulerResumeKicks`.function instrumentSchedulerPort(channel) {var port = channel.port1;@@ -2085,15 +1846,13 @@return undefined;}counters.schedulerCallbacksDelivered += 1;- // Called without a try/catch on purpose: an exception thrown by React must propagate- // exactly as it would have. Swallowing it here would make the frozen run behave- // differently from the control in a way that has nothing to do with the ablation.+ // Deliberately not wrapped in try/catch: an exception thrown by React must propagate exactly+ // as it would have, or the frozen run differs for reasons unrelated to the ablation.return fn.call(port, event);}- // Assigning through the prototype accessor first, so the real port ends up calling the- // dispatcher. This implicitly starts port1, which the scheduler does a moment later- // anyway when it assigns its own handler.+ // Assigned through the prototype accessor first so the real port calls the dispatcher; this+ // implicitly starts port1, which the scheduler does a moment later anyway.port.onmessage = dispatch;Object.defineProperty(port, "onmessage", {@@ -2107,9 +1866,9 @@}});- // Some builds attach with addEventListener instead of onmessage. Without this, such a- // build would report captured ports and zero suppressions, which reads as "we froze it- // and React did not care" when the truth is that the freeze never reached the handler.+ // Some builds attach with addEventListener instead of onmessage; without this, such a build+ // reports captured ports and zero suppressions, reading as "React did not care" when the+ // freeze never reached the handler.try {var origAdd = port.addEventListener;var origRemove = port.removeEventListener;@@ -2183,9 +1942,8 @@return false;}- // Probe before committing: if the port instance refuses a redefined onmessage there is no- // way to intercept, and the arm must read UNAVAILABLE rather than run and suppress- // nothing.+ // Probe before committing: a port that refuses a redefined onmessage cannot be intercepted,+ // and the arm must read UNAVAILABLE rather than run and suppress nothing.try {var probe = new Original();Object.defineProperty(probe.port1, "onmessage", {@@ -2212,8 +1970,8 @@try {instrumentSchedulerPort(channel);} catch (e) {- // A channel we failed to instrument is still a working channel. The app must not- // notice, and capturedSchedulerPorts simply does not count it.+ // A channel we failed to instrument is still a working channel: the app must not notice, and+ // capturedSchedulerPorts simply does not count it.debug("could not instrument a MessageChannel", e);}return channel;@@ -2250,8 +2008,8 @@}state.frozen = true;if (counters.capturedSchedulerPorts === 0) {- // Not an error yet: React may create its channel later. It is recorded in the reason- // so that a run which ends with zero captured ports is read as NOT RUN.+ // Not an error yet (React may create its channel later); recorded in the reason so a run+ // ending with zero captured ports reads as NOT RUN.return result(true,"freeze flag set, but no MessageChannel has been created yet; if " +@@ -2313,19 +2071,12 @@);}- // ------------------------------------------------------------------------------------- // Arm G -- control: identical DOM, prior turns in the viewport- //- // The viewport carries the `scroll-smooth` class (thread.tsx:1728), so a scrollTop assignment- // would animate. An animated scroll would still be running when the measured window starts,- // which would put scroll animation cost inside the control and make the control the most- // expensive arm. scrollBehavior is forced to auto for the assignment and restored afterwards.- //- // The app's own autoscroll may pull the thread back to the bottom at any time, which is why- // controlVisibleMessages is a gauge that is re-measured on every potency() call rather than a- // number recorded once at apply time: the honest question is whether the prior turns were- // visible for the WINDOW, not whether they were visible for an instant.- // ------------------------------------------------------------------------------------+ // The viewport carries `scroll-smooth` (thread.tsx:1728), so assigning scrollTop would animate+ // and put scroll-animation cost inside the control; scrollBehavior is forced to auto for the+ // assignment and restored afterwards.+ // The app's own autoscroll may pull the thread back at any time, so controlVisibleMessages is+ // a gauge re-measured on every potency() call: the honest question is whether prior turns were+ // visible for the WINDOW, not for an instant.function applyG() {var vp = viewportEl();@@ -2401,9 +2152,9 @@}seen = visibleCounts();- // Walk upwards until enough prior turns are on screen or the top is reached. Bounded- // iterations: a loop that cannot terminate would hang the measured window, and reporting- // "could not reach the target" is a usable result while a hang is not.+ // Walk up until enough prior turns are on screen or the top is reached. Bounded iterations: a+ // non-terminating loop would hang the measured window, and "could not reach the target" is a+ // usable result while a hang is not.while (seen.prior < target && attempts < 16) {attempts += 1;var before = 0;@@ -2488,9 +2239,6 @@}}- // ------------------------------------------------------------------------------------- // apply / revert- // ------------------------------------------------------------------------------------function apply(armId) {var id;@@ -2506,9 +2254,8 @@return result(false, unavailable[id], 0);}if (Object.prototype.hasOwnProperty.call(state.applied, id) && id !== "F") {- // `applied` means "this call applied it". Arms are applied once per measured window- // by design, so a second call is a no-op and says so rather than re-marking elements- // and inflating the counters.+ // `applied` means "this call applied it": arms apply once per measured window, so a second+ // call is a no-op instead of re-marking elements and inflating the counters.return result(false,"arm " + id + " was already applied at " + state.applied[id].at +@@ -2547,8 +2294,8 @@out = applyG();}} catch (e) {- // An arm that throws has to be reported, not propagated: the app is mid-stream and an- // exception here would end the run with no data at all.+ // An arm that throws is reported, not propagated: the app is mid-stream and an exception here+ // would end the run with no data at all.return result(false, "arm " + id + " threw while applying: " + e, 0);}@@ -2575,9 +2322,8 @@try {if (id === "A" || id === "B" || id === "C") {removeStyleSheet(id);- // The inline and marker ledgers are global rather than per arm: a run applies one- // arm, and unwinding everything is the behaviour least likely to leave a stray- // !important behind on a page that is about to be measured again.+ // The inline and marker ledgers are global, not per arm: a run applies one arm, and unwinding+ // everything is least likely to leave a stray !important on a page about to be re-measured.restoreInline();unmarkAll();out = revertResult(true, "stylesheet removed, inline properties and markers restored");@@ -2646,9 +2392,6 @@return out;}- // ------------------------------------------------------------------------------------- // Install the preboot arms, then publish- // ------------------------------------------------------------------------------------if (armedPreboot.indexOf("D") !== -1) {installD();@@ -2685,8 +2428,8 @@);}- // Feature checks for the runtime arms. These are capability questions about the browser, not- // about the page, so a false here is UNAVAILABLE and never a zero difference.+ // Feature checks for the runtime arms: capability questions about the browser, not the page,+ // so a false here is UNAVAILABLE and never a zero difference.if (!doc) {markUnavailable("A", "no document");markUnavailable("B", "no document");diff --git a/tests/studio/studiobench/arms/ladder.py b/tests/studio/studiobench/arms/ladder.pyindex 8323b526123..2967d4df516 100644--- a/tests/studio/studiobench/arms/ladder.py+++ b/tests/studio/studiobench/arms/ladder.py@@ -50,8 +50,8 @@ from typing import Any, Iterable, Mapping, Sequencefrom ..scoring.schema import Measurefrom .manifest import ArmOutcome, ArmStatus-#: The named mechanisms this ladder can remove. A step must name mechanisms from this set, so a-#: typo becomes an error rather than a new, unexplained row in the report.+#: The named mechanisms this ladder can remove. A step must name mechanisms from this set, so a typo+#: becomes an error rather than a new, unexplained row in the report.MECHANISMS: tuple[str, ...] = ("paint_raster","offscreen_style_layout",@@ -365,8 +365,8 @@ def differences(and result.sum_of_steps):result.residual_ms = float(result.total.value) - float(result.sum_of_steps.value)- # This is an arithmetic identity, not an empirical claim: the interior terms cancel. A- # non-zero residual here means a bug in this function, not a finding about the app.+ # An arithmetic identity, not an empirical claim: the interior terms cancel, so a non-zero residual+ # means a bug in this function, not a finding about the app.result.identity_holds = abs(result.residual_ms) < 1e-9result.identity_note = ("top minus floor equals the sum of the adjacent differences, identically, because the "@@ -495,14 +495,14 @@ def interaction_terms(return terms-# ---------------------------------------------------------------------------------------# the two declared routes+# ---------------------------------------------------------------------------------------_SHIPPING: frozenset[str] = frozenset()-#: Route 1 walks down the rendering pipeline first: stop painting, then stop laying out-#: off-screen, then stop occupying layout at all, and only then touch the observers and React.+#: Route 1 walks down the rendering pipeline first: stop painting, then stop laying out off-screen,+#: then stop occupying layout at all, and only then touch the observers and React.ROUTE_VISUAL_FIRST = LadderRoute(route_id = "visual_first",name = "paint, then off-screen layout, then geometry, then observers, then React",@@ -546,9 +546,9 @@ ROUTE_VISUAL_FIRST = LadderRoute(),)-#: Route 2 reaches the identical floor from the other end: kill the observers and React first,-#: then walk down the rendering pipeline. If the mechanisms are additive both routes report the-#: same per-mechanism numbers. Where they do not, that gap is the interaction term.+#: Route 2 reaches the identical floor from the other end: kill the observers and React first, then+#: walk down the rendering pipeline. If the mechanisms are additive both routes report the same+#: per-mechanism numbers; where they do not, that gap is the interaction term.ROUTE_SCHEDULER_FIRST = LadderRoute(route_id = "scheduler_first",name = "observers, then React, then paint, then off-screen layout, then geometry",diff --git a/tests/studio/studiobench/arms/layoutcost.py b/tests/studio/studiobench/arms/layoutcost.pyindex 918d1be003a..0b4a291e1b8 100644--- a/tests/studio/studiobench/arms/layoutcost.py+++ b/tests/studio/studiobench/arms/layoutcost.py@@ -35,10 +35,10 @@ from ..scoring.schema import MeasureLAYOUTCOST_JS_PATH = Path(__file__).resolve().parents[1] / "instruments" / "layoutcost.js"-#: The instrument level at which this may run. Headline numbers come from level 0 only.+#:The instrument level at which this may run. Headline numbers come from level 0 only.LAYOUTCOST_LEVEL = 3-#: Counter families the browser side reports, each with its own `attempted` flag.+#:Counter families the browser side reports, each with its own `attempted` flag.COUNTER_FAMILIES = ("scrollHeightReads","scrollTopWrites",@@ -187,8 +187,8 @@ class LayoutCostInstrument:self._ctx = ctxdef start_cell(self, cell: Any) -> None:- # `ctx.page` may be replaced between cells when a crashed renderer is recovered, so the- # page is re-read here rather than cached in attach().+ # `ctx.page` may be replaced between cells when a crashed renderer is recovered, so the page is+ # re-read here rather than cached in attach().self._page = getattr(self._ctx, "page", None)def open(self, window: Any) -> None:@@ -214,9 +214,9 @@ class LayoutCostInstrument:return reading_from_snapshot(snapshot).to_json()def end_cell(self, cell: Any) -> dict[str, Any] | None:- # The harness contract requires every instrument at level >= 1 to declare its own cost.- # This one declares the LOWER BOUND it can measure itself, and says so, because the real- # number comes from the paired with/without cell that only the deep tier runs.+ # The harness contract requires every instrument at level >= 1 to declare its own cost. This one+ # declares the LOWER BOUND it can measure itself, because the real number comes from the paired+ # with/without cell that only the deep tier runs.if self._page is None:return {"overhead_ms": None, "overhead_attempted": False}try:diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py b/tests/studio/studiobench/arms/selftest/test_studiobench_arms.pyindex 66d5099e242..3fa395ec485 100644--- a/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_arms.py@@ -72,8 +72,9 @@ def _arm()-# ---------------------------------------------------------------------------------------# manifest: invariance and potency++# ---------------------------------------------------------------------------------------@@ -200,8 +201,9 @@ def test_an_equivalent_arm_must_declare_its_diff_and_an_exact_one_may_not():_arm(declared_diff = DeclaredDiff(normaliser = "n", keys = ("style",)))-# ---------------------------------------------------------------------------------------# ladder++# ---------------------------------------------------------------------------------------@@ -385,8 +387,9 @@ def test_required_rungs_covers_both_routes():assert "D" in keys and "A" in keys-# ---------------------------------------------------------------------------------------# calibration++# ---------------------------------------------------------------------------------------@@ -485,8 +488,9 @@ def test_a_batch_with_no_null_reading_has_no_noise_floor():assert "no measured noise floor" in verdict.reason-# ---------------------------------------------------------------------------------------# dose-response++# ---------------------------------------------------------------------------------------@@ -549,8 +553,9 @@ def test_two_points_do_not_make_a_line():assert fit.verdict == "NO FIT"-# ---------------------------------------------------------------------------------------# armpack++# ---------------------------------------------------------------------------------------@@ -612,8 +617,9 @@ def test_a_matching_armpack_resolves(tmp_path: Path):assert resolution.require().target_dist_digest == "digest-1"-# ---------------------------------------------------------------------------------------# recovery++# ---------------------------------------------------------------------------------------@@ -671,8 +677,9 @@ def test_worse_after_delete_points_at_the_delete_path():assert result.classification == "WORSE AFTER DELETE"-# ---------------------------------------------------------------------------------------# knobs++# ---------------------------------------------------------------------------------------diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py b/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.pyindex f5601f162af..05778cc7ef7 100644--- a/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_end_to_end.py@@ -76,8 +76,8 @@ def _outcome()-#: One coherent synthetic world: the autoscroll observer is the dominant cost on both routes,-#: and the two routes disagree about it, which is the interaction the design exists to surface.+#: One coherent synthetic world: the autoscroll observer is the dominant cost on both routes, and+#: the two routes disagree about it, which is the interaction the design exists to surface._VISUAL = {"shipping": 40.0,"A": 34.0,@@ -140,8 +140,9 @@ def _blind_calibration():)-# ---------------------------------------------------------------------------------------# planning++# ---------------------------------------------------------------------------------------@@ -166,8 +167,9 @@ def test_no_scene_durations_at_all_is_a_refusal_not_a_pass():assert_equal_scene_duration({})-# ---------------------------------------------------------------------------------------# judging and rendering++# ---------------------------------------------------------------------------------------@@ -250,8 +252,8 @@ def test_fix_implications_rank_the_largest_step_and_name_its_fix():)rendered = render_fix_implications(result)top = rendered.splitlines()[1].strip()- # the two routes disagree about the autoscroll observer, so the headline is a RANGE and both- # routes are named; quoting only the larger would let a reader pick the flattering route+ # the two routes disagree about the autoscroll observer, so the headline is a RANGE and both routes+ # are named; quoting only the larger would let a reader pick the flattering routeassert top.startswith("9.000 to 18.000 ms")assert "autoscroll_forced_layout" in topassert "routes disagree: scheduler_first 18.000, visual_first 9.000" in rendered@@ -288,8 +290,9 @@ def test_a_batch_with_a_dose_fit_an_armpack_refusal_and_a_recovery_renders_all_tassert "RETAINED STRUCTURE" in rendered-# ---------------------------------------------------------------------------------------# the harness layer's row stream++# ---------------------------------------------------------------------------------------diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py b/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.pyindex 582b6630abb..713a8d43254 100644--- a/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_probe_hooks.py@@ -61,9 +61,9 @@ def test_the_hooks_are_off_unless_asked_for(main_src: str, monkeypatch):assert os.environ.get("SBENCH_EXTRA_INIT_SCRIPT") is Noneassert os.environ.get("SBENCH_PAGE_CONSOLE") is None- # Both call sites are guarded by a plain truthiness check on the variable, so an unset- # variable cannot reach `add_init_script` or `page.on`. Pinned as source, because the failure- # this guards is a refactor that hoists either call out of its `if`.+ # Both call sites are guarded by a plain truthiness check on the variable, so an unset variable+ # cannot reach `add_init_script` or `page.on`. Pinned as source, because the failure this guards+ # is a refactor that hoists either call out of its `if`.for guarded in ("if extra_init:", "if console_prefix:"):assert guarded in main_src, f"{guarded!r} is gone; the hook may no longer be opt-in"@@ -83,11 +83,10 @@ def test_the_probe_path_is_validated_before_anything_is_started(main_src: str):assert "except (OSError, UnicodeDecodeError) as exc:" in main_src# Read once, used later. A second read at launch time would reintroduce the window.assert main_src.count("Path(extra_init).read_text") == 1- # AND ahead of the archive, which is the other thing a refusal must not arrive after. Reusing- # an --out without --resume moves the payload aside, so reading the probe after that call let a- # path typo take payload.jsonl off its standard path while running no benchmark at all. The- # behaviour is pinned in runtime/selftest/test_studiobench_run_acquisition.py; the order is- # pinned here because that is what makes it true.+ # AND ahead of the archive, the other thing a refusal must not arrive after: reusing an --out+ # without --resume moves the payload aside, so reading the probe after that call let a path typo+ # take payload.jsonl off its standard path while running no benchmark at all. The behaviour is+ # pinned in runtime/selftest/test_studiobench_run_acquisition.py; the order is pinned here.assert read_at < main_src.index("prepare_payload(")@@ -136,9 +135,9 @@ def test_a_probe_run_records_the_gate_that_makes_it_unscorable(main_src: str):"""assert '"probe_init_script": extra_init or None' in main_src- # Matched without pinning the indent. `run()` puts side acquisition under a cleanup guard, so- # everything from the acquisition loop to the end of setup sits one level deeper than it did,- # and the gate moving with it is not the thing this test is here to notice.+ # Matched without pinning the indent: `run()` puts side acquisition under a cleanup guard, so+ # everything from the acquisition loop to the end of setup sits one level deeper, and the gate+ # moving with it is not what this test notices.assert re.search(r"rec\.gate\(\s*\n\s*\"probe_free\",", main_src)@@ -152,8 +151,8 @@ def test_roots_are_adopted_at_insertion_not_on_the_sample_tick(probe_src: str):assert "MutationObserver" in probe_srcassert "adoptAdded" in probe_src- # The DOCUMENT, not documentElement. An init script can run before the parser has created- # the root element, and `observe(null, ...)` throws into the catch that hides it.+ # The DOCUMENT, not documentElement: an init script can run before the parser has created the+ # root element, and `observe(null, ...)` throws into the catch that hides it.assert "observe(doc, {" in probe_src# Added nodes only. A document-wide re-scan per mutation would make the probe the load.assert "addedNodes" in probe_src@@ -171,8 +170,8 @@ def test_the_fallback_and_padding_buckets_cannot_both_count_one_root(probe_src:assert "ROLE_PX" in probe_srcassert "out.fallbackBite += 1;" in probe_srcassert 'targetHeight(px, "padding")' in probe_src- # getBoundingClientRect() is the border box, so BOTH targets carry the root's own padding.- # Comparing against the bare declared length pinned fallbackBite at zero whatever happened.+ # getBoundingClientRect() is the border box, so BOTH targets carry the root's own padding;+ # comparing against the bare declared length pinned fallbackBite at zero whatever happened.assert 'targetHeight(px, "fallback")' in probe_srcassert 'return (which === "fallback" ? px.fallback : 0) + px.padding;' in probe_src@@ -254,8 +253,8 @@ def test_the_probe_source_is_the_first_thing_in_its_script():"semantics than the file it was read from")assert "window.__sbExtraInitScript" in scripts[0]- # Appended after an explicit statement boundary, so neither ASI nor an unterminated- # expression can join it to the probe's last line.+ # Appended after an explicit statement boundary, so neither ASI nor an unterminated expression can+ # join it to the probe's last line.assert (scripts[0][len(source) :].lstrip("\n")@@ -379,8 +378,8 @@ def test_the_event_counter_is_the_one_potency_rests_on(probe_src: str):assert "contentvisibilityautostatechange" in probe_srcassert "ev_skip" in probe_src- # The geometry route is a documented false negative and must stay documented rather than- # quietly removed: someone will reimplement it otherwise.+ # The geometry route is a documented false negative and must stay documented rather than quietly+ # removed: someone will reimplement it otherwise.assert "KNOWN FALSE NEGATIVE" in probe_src@@ -398,9 +397,9 @@ def _node_parses(source: str):if node is None:pytest.skip("no node on PATH; this assertion needs a real JS parser, not a regex")with tempfile.TemporaryDirectory() as tmp:- # `--check` reads a FILE, so the IIFE is written out rather than passed with `-e`: an- # argument would have to survive the shell, and the point of this helper is that nothing- # between the producer and the parser is allowed to edit the bytes.+ # `--check` reads a FILE, so the IIFE is written out rather than passed with `-e`: an argument+ # would have to survive the shell, and the point of this helper is that nothing between the+ # producer and the parser is allowed to edit the bytes.script = Path(tmp) / "init_script.js"script.write_text("(() => {\n" + source + "\n})();", encoding = "utf-8")done = subprocess.run([node, "--check", str(script)], capture_output = True, text = True)diff --git a/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py b/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.pyindex 03c5e674967..7040b0497d8 100644--- a/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py+++ b/tests/studio/studiobench/arms/selftest/test_studiobench_selfcheck.py@@ -35,8 +35,9 @@ from studiobench.instruments.selfcheck import ( # noqa: E402from studiobench.scoring.schema import EXCLUSION_REASONS, check_exclusion_reasons # noqa: E402-# ---------------------------------------------------------------------------------------# the injected stall++# ---------------------------------------------------------------------------------------@@ -58,8 +59,9 @@ def test_no_stall_reading_at_all_is_a_failure_not_a_pass():assert gate.measured.has_reading is False-# ---------------------------------------------------------------------------------------# the injected input delay++# ---------------------------------------------------------------------------------------@@ -73,8 +75,9 @@ def test_an_input_path_that_does_not_move_is_not_measuring_input():assert "will not move when the app gets slower" in gate.detail-# ---------------------------------------------------------------------------------------# scene contrast: the blindness check++# ---------------------------------------------------------------------------------------@@ -88,8 +91,9 @@ def test_an_instrument_that_cannot_tell_heavy_from_trivial_is_blind():assert "BLIND" in gate.detail-# ---------------------------------------------------------------------------------------# longtask support++# ---------------------------------------------------------------------------------------@@ -109,8 +113,9 @@ def test_an_unreadable_support_list_is_recorded_and_not_guessed():assert gate.fatal is False-# ---------------------------------------------------------------------------------------# the clock-pair control ratio++# ---------------------------------------------------------------------------------------@@ -124,8 +129,9 @@ def test_a_moving_control_ratio_means_the_measurement_moved():assert "the MEASUREMENT moved, not the page" in gate.detail-# ---------------------------------------------------------------------------------------# three-clock agreement++# ---------------------------------------------------------------------------------------@@ -188,8 +194,9 @@ def test_clock_disagreement_is_a_declared_exclusion_reason():assert "clock_disagreement" in EXCLUSION_REASONS-# ---------------------------------------------------------------------------------------# the whole gate set++# ---------------------------------------------------------------------------------------@@ -291,7 +298,7 @@ def test_the_injection_script_burns_only_on_sse_chunks():js = stream_cost_injection_init_script(3.0)assert 'indexOf("data:")' in js- # Queued, not inline: the burn has to land inside the measured chain whichever TextDecoder- # wrapper ended up outermost.+ # Queued, not inline: the burn has to land inside the measured chain whichever TextDecoder wrapper+ # ended up outermost.assert "queueMicrotask" in jsassert "3.0" in jsdiff --git a/tests/studio/studiobench/attribution/vite.studiobench.config.ts b/tests/studio/studiobench/attribution/vite.studiobench.config.tsindex 0d2a76ec3b2..0af98566183 100644--- a/tests/studio/studiobench/attribution/vite.studiobench.config.ts+++ b/tests/studio/studiobench/attribution/vite.studiobench.config.ts@@ -1,51 +1,32 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-// studiobench attribution build.-//-// Identical to `studio/frontend/vite.config.ts` in every respect that affects-// what the app DOES, and different only in what it lets us SEE:-//-// 1. `react-dom/client` is aliased to `react-dom/profiling`, which is the-// only build where React's `<Profiler>` `onRender` callback exists at all.-// 2. Hidden sourcemaps, so our own frames get real names without shipping a-// `//# sourceMappingURL` comment.-// 3. Minifier name preservation, so our own functions and classes keep-// their source names instead of being mangled to single letters.-//-// The resulting dist is injected into a REAL Unsloth install through the-// existing `unsloth studio --frontend <dir>` flag (see `studio/backend/run.py`-// around line 2820), so the backend, auth and storage stay real and only the-// frontend bundle is swapped. That matters: the failure this whole tool exists-// to correct was a fixture that did not run the real path.-//-// WHAT THIS BUILD DOES NOT FIX, and cannot. React's production and profiling-// dists are minified by React's own release pipeline BEFORE Vite ever sees-// them. Name preservation and sourcemaps therefore recover OUR names and-// nothing whatsoever inside `react-dom`: the original identifier really is-// `Zk`. Recovering react-dom names is the job of the symbol bridge in-// `tests/studio/studiobench/analysis/symbols.py`, not of this config, and no-// build flag here can do it.+// studiobench attribution build. Identical to `studio/frontend/vite.config.ts` in every respect+// that affects what the app DOES, differing only in what it lets us SEE: `react-dom/client`+// aliased to `react-dom/profiling` (the only build where React's `<Profiler>` `onRender` exists),+// hidden sourcemaps, and minifier name preservation.+// The resulting dist is injected into a REAL Unsloth install through `unsloth studio --frontend+// <dir>`, so backend, auth and storage stay real and only the frontend bundle is swapped: the+// failure this tool exists to correct was a fixture that did not run the real path.+// WHAT THIS BUILD CANNOT FIX: React's production and profiling dists are minified by React's own+// release pipeline before Vite sees them, so name preservation and sourcemaps recover OUR names+// and nothing inside `react-dom` (the original identifier really is `Zk`). That is the job of the+// symbol bridge in `analysis/symbols.py`.+// That is `tests/studio/studiobench/analysis/symbols.py`.import { createRequire } from "node:module";import path from "node:path";const FRONTEND_ROOT = path.resolve(__dirname, "../../../../studio/frontend");-// The plugins are resolved from the FRONTEND package, not from this directory,-// and that is not a stylistic preference. Vite bundles a `--config` file with-// its bare imports left external (`bundleConfigFile` sets `root =-// path.dirname(fileName)`) and then loads the bundle under the config file's-// own path: CJS via `module._compile(code, <config path>)`, ESM via a temp file-// in the nearest ancestor `node_modules`, of which this directory has none. So-// Node starts its `node_modules` walk here and climbs to the repository root,-// and NOTHING on that path has a `node_modules` -- only-// `studio/frontend/node_modules` does. Written with plain top-level imports-// this config could not be loaded from any working directory, by any `vite`-// binary: `Cannot find module '@tailwindcss/vite'`, and the same for the React-// plugin and for `vite` itself. Anchoring the lookup at the frontend package's-// `package.json` is the whole fix, and it points at the same installed copies-// the shipping `studio/frontend/vite.config.ts` uses.+// The plugins are resolved from the FRONTEND package, not this directory. Vite bundles a+// `--config` file with bare imports left external and loads it under the config file's own path,+// so Node starts its `node_modules` walk here and climbs to the repository root, where nothing+// has a `node_modules`: only `studio/frontend/node_modules` does. With plain top-level imports+// this config could not be loaded from any working directory by any `vite` binary. Anchoring the+// lookup at the frontend package's `package.json` points at the same installed copies the+// shipping config uses.+// Vite's `bundleConfigFile` sets root = path.dirname(fileName).const requireFromFrontend = createRequire(path.join(FRONTEND_ROOT, "package.json"));const tailwindcss = requireFromFrontend("@tailwindcss/vite").default;const react = requireFromFrontend("@vitejs/plugin-react").default;@@ -54,8 +35,8 @@ const { defineConfig } = requireFromFrontend("vite");export default defineConfig({root: FRONTEND_ROOT,plugins: [react(), tailwindcss()],- // Keep an unrelated PostCSS config in an ancestor directory from leaking- // into Unsloth installs. Tailwind is provided by its dedicated Vite plugin.+ // Keep an unrelated PostCSS config in an ancestor directory from leaking into Unsloth installs.+ // Tailwind is provided by its dedicated Vite plugin.css: {postcss: {plugins: [],@@ -99,22 +80,14 @@ export default defineConfig({},},resolve: {- // An ARRAY of regex aliases, not the object form, and the react-dom entry- // is deliberately anchored. Two traps, both of which produce a build that- // looks fine and is not:- //- // - Aliasing the bare specifier `react-dom` recurses forever. The- // profiling bundle's own first lines are- // `require("react"), require("react-dom")`, so rewriting `react-dom`- // points it at itself.- // - A plain string `find: "react-dom/client"` is prefix matching in- // Vite, so it would also rewrite `react-dom/profiling` and produce the- // same recursion by a different route.- //- // `/^react-dom\/client$/` matches exactly the specifier the app imports and- // nothing else. Bare `react-dom` is left alone on purpose: it is a small- // shim that dispatches through the injected internals object, so it drives- // whichever renderer actually loaded.+ // An ARRAY of regex aliases, not the object form, and the react-dom entry is deliberately+ // anchored. Aliasing the bare specifier `react-dom` recurses forever, because the profiling+ // bundle's own first lines are `require("react"), require("react-dom")`; and a plain string+ // `find: "react-dom/client"` is prefix matching in Vite, so it would also rewrite+ // `react-dom/profiling` into the same recursion.+ // `/^react-dom\/client$/` matches exactly the specifier the app imports. Bare `react-dom` is left+ // alone on purpose: it is a small shim dispatching through the injected internals object, so it+ // drives whichever renderer actually loaded.alias: [{ find: /^react-dom\/client$/, replacement: "react-dom/profiling" },{ find: /^@\//, replacement: `${path.resolve(FRONTEND_ROOT, "./src")}/` },@@ -130,70 +103,44 @@ export default defineConfig({build: {outDir: path.resolve(__dirname, "dist"),emptyOutDir: true,- // `hidden` emits the .map files but suppresses the sourceMappingURL- // comment, so the bundle is byte-comparable to a shipping one apart from- // the profiling renderer, and devtools does not silently start resolving- // maps mid-measurement.+ // `hidden` emits the .map files but suppresses the sourceMappingURL comment, so the bundle is+ // byte-comparable to a shipping one apart from the profiling renderer, and devtools does not+ // silently start resolving maps mid-measurement.sourcemap: "hidden",- // Vite 8 minifies with oxc by default, and without a name-preservation- // knob the minifier renames our own components too: the app-side frames- // become single letters, which would leave the symbol bridge with no- // anchors to validate against. Measured on this tree, of the 403- // `export function <Component>` names in `studio/frontend/src` only 37- // survive an unconfigured build, against 390 with the option below.- //- // The knob is the OXC MINIFIER's `mangle.keepNames`, reached through- // `output.minify`, and NOT the bundler-level `output.keepNames`. That- // distinction is the whole reason this block looks the way it does.- // `output.keepNames: true` on the rolldown that Vite 8.0.16 pins- // (`[email protected]`, an exact pin in Vite's own `dependencies`) fails the- // build outright:- //- // [MISSING_EXPORT] "toString" is not exported by- // "node_modules/underscore/modules/_setup.js"- //- // and 23 more like it. That is rolldown#9973: under `keepNames` the- // bundler splits a multi-declarator `export var a = 1, b = 2` into- // separate declarations and drops the `export` keyword while doing it, so- // every name after the split silently stops being exported. Underscore's- // `_setup.js` is written almost entirely in that style- // (`export var push = ..., slice = ..., toString = ...`) and we reach it- // through `@dagrejs/graphlib`, so the build aborts. Fixed upstream by- // rolldown#9974, first released in the 1.1.x line; unavailable to us- // without overriding the rolldown that Vite pins, which would change the- // bundler under the shipping UI and is not something an attribution-only- // config gets to do.- //- // `mangle.keepNames` is the option that actually matters here anyway.- // Bundler-level renaming only appends deconflicting suffixes and leaves- // names readable; it is the minifier's mangler that turns `AppSidebar`- // into a letter. `compress.keepNames` is set alongside it because oxc- // documents the pair as belonging together -- DCE has to stop treating- // the name-carrying binding as dead. Vite spreads the user `output` last- // (`buildOutputOptions`), so `minify` here replaces Vite's default- // `minify: true` rather than being clobbered by it; compression and- // mangling both stay on, and the bundle grows only ~3.9% from carrying- // the real names.+ // Vite 8 minifies with oxc by default, and without a name-preservation knob the minifier renames+ // our own components too, leaving the symbol bridge with no anchors: of the 403 `export function+ // <Component>` names in `studio/frontend/src` only 37 survive an unconfigured build, against 390+ // with the option below.+ // The knob is the OXC MINIFIER's `mangle.keepNames`, reached through `output.minify`, and NOT the+ // bundler-level `output.keepNames`, which on the rolldown Vite 8.0.16 pins fails the build+ // outright with `[MISSING_EXPORT] "toString" is not exported by underscore/modules/_setup.js` and+ // 23 more. That is rolldown#9973: under `keepNames` the bundler splits a multi-declarator `export+ // var a = 1, b = 2` and drops the `export` keyword, so every name after the split stops being+ // exported. Underscore's `_setup.js` is written that way and we reach it through+ // `@dagrejs/graphlib`. Fixed upstream by rolldown#9974 in the 1.1.x line, unavailable without+ // overriding the rolldown Vite pins, which an attribution-only config does not get to do.+ // The pin is `[email protected]`.+ // `mangle.keepNames` is the option that matters anyway: bundler-level renaming only appends+ // deconflicting suffixes, while the minifier's mangler is what turns `AppSidebar` into a letter.+ // `compress.keepNames` is set alongside it because oxc documents the pair as belonging together+ // (DCE has to stop treating the name-carrying binding as dead). Vite spreads the user `output`+ // last, so `minify` here replaces Vite's default rather than being clobbered; compression and+ // mangling stay on and the bundle grows only ~3.9%.+ // In `buildOutputOptions`.rolldownOptions: {output: {minify: {compress: { keepNames: { function: true, class: true } },mangle: { keepNames: { function: true, class: true } },},- // Marks the bundle so the harness can prove it is measuring the- // attribution build and not a stale shipping dist left in the- // directory handed to `unsloth studio --frontend`. Read by+ // Marks the bundle so the harness can prove it is measuring the attribution build and not a stale+ // shipping dist left in the directory handed to `unsloth studio --frontend`. Read by// `../analysis/bridge_build.py:assert_attribution_build`.- //- // This is a BANNER and not a `define`, and the difference is not- // cosmetic. `define` performs identifier SUBSTITUTION in source: it- // rewrites occurrences of a token that already appear in the code. No- // Unsloth source file mentions `__STUDIOBENCH_ATTRIBUTION_BUILD__`, so- // a `define` entry substitutes nothing, emits nothing, and the marker- // is simply absent at runtime. Built that way first; the assertion- // could never pass and the bundle was byte-identical with and without- // the option. A banner is prepended unconditionally, so the global- // genuinely exists.+ // This is a BANNER and not a `define`: `define` performs identifier SUBSTITUTION in source, and no+ // Unsloth source file mentions `__STUDIOBENCH_ATTRIBUTION_BUILD__`, so a `define` entry+ // substitutes nothing and the marker is simply absent at runtime. Built that way first, the+ // assertion could never pass and the bundle was byte-identical with and without the option. A+ // banner is prepended unconditionally, so the global genuinely exists.banner: "globalThis.__STUDIOBENCH_ATTRIBUTION_BUILD__ = true;",},},diff --git a/tests/studio/studiobench/build.py b/tests/studio/studiobench/build.pyindex 5000b67d166..2673f388308 100644--- a/tests/studio/studiobench/build.py+++ b/tests/studio/studiobench/build.py@@ -45,7 +45,7 @@ def _copy_package(staging: Path) -> None:target.parent.mkdir(parents = True, exist_ok = True)shutil.copytree(PKG, target, ignore = shutil.ignore_patterns(*EXCLUDE_DIRS, "*.pyc", "*.pyo"))# `tests` and `tests/studio` must be importable packages inside the archive, and they are not- # packages in the repository -- the harnesses there are standalone scripts.+ # packages in the repository: the harnesses there are standalone scripts.for pkg_dir in (staging / "tests", staging / "tests" / "studio"):init = pkg_dir / "__init__.py"if not init.exists():@@ -86,8 +86,8 @@ def build(out: Path = DEFAULT_OUT, compressed: bool = True) -> Path:"artifact ships a benchmark with no content.")- # A __main__.py at the archive root is what `python foo.pyz` executes. It re-exports the- # package's CLI rather than duplicating it.+ # A __main__.py at the archive root is what `python foo.pyz` executes. It re-exports the package's+ # CLI rather than duplicating it.(staging / "__main__.py").write_text("import sys\n""from tests.studio.studiobench.__main__ import main\n"diff --git a/tests/studio/studiobench/fixture/corpus.py b/tests/studio/studiobench/fixture/corpus.pyindex 3139de8678e..03ad263c82a 100644--- a/tests/studio/studiobench/fixture/corpus.py+++ b/tests/studio/studiobench/fixture/corpus.py@@ -48,60 +48,51 @@ UNITS_JSONL = FROZEN_DIR / "units.jsonl"MANIFEST_JSON = FROZEN_DIR / "manifest.json"CORPUS_SEED = 20260819-# 2 added math. A number taken on v1 and a number taken on v2 are measurements of two different-# films, so they are not comparable and `floor_table` refuses to pool them. Bump this whenever the-# generated text changes at all, and never edit the generator without bumping it.+# 2 added math. A number taken on v1 and one on v2 measure two different films, so+# `floor_table` refuses to pool them. Bump this whenever the generated text changes.CORPUS_VERSION = 2-# Span density, calibrated against the field capture. See the module docstring. These are now the-# MEANS of a jittered distribution rather than fixed sizes -- see `_jitter`.+# Span density, calibrated against the field capture (see the module docstring). These are+# MEANS of a jittered distribution rather than fixed sizes; see `_jitter`.PROSE_CHARS = 1250FENCE_CHARS = 1800PREAMBLE_FRACTION = 0.25-# How far each block's size may wander from its mean, as a fraction. Real replies do not emit-# identically sized paragraphs and fences, and a fixture that does lets a cost that is really-# per-block masquerade as per-character (and vice versa): with every block the same size the two-# are perfectly collinear and no measurement can separate them. Jitter breaks that.-#-# Applied through the PER-UNIT rng, so unit 40 is still the same text whether it was reached by-# generating 41 units or by asking for that one. Randomised is not the same as nondeterministic:-# the corpus hash still pins every byte.+# How far each block's size may wander from its mean. With every block the same size,+# per-block and per-character cost are perfectly collinear and no measurement can separate+# them; jitter breaks that. Applied through the PER-UNIT rng, so unit 40 is the same text+# however it was reached, and the corpus hash still pins every byte.BLOCK_JITTER = 0.55-# Unit sizes wander too, around the escalating nominal. Same reason at the turn level: a thread of-# turns that are all exactly 10,000 characters is not the shape of a real session, and the rung-# planner then has no short turns to land its target on.+# Unit sizes wander too, around the escalating nominal: a thread of turns all exactly 10,000+# characters is not the shape of a real session, and the rung planner then has no short turns+# to land its target on.UNIT_JITTER = 0.35-# Tool calls per unit, as a range. The thread renders a tool group for each, which is a component-# with its own mount and update cost and which nothing in the corpus previously exercised.+# Tool calls per unit, as a range. The thread renders a tool group for each, a component with+# its own mount and update cost that nothing in the corpus previously exercised.TOOL_CALLS_PER_UNIT = (0, 3)-#: The stored part shape that actually renders a tool block. VERIFIED, not assumed: seeding a flat-#: `{"type": "tool-call", ...}` and an assistant-ui style `{"type": "tool-invocation", ...}` both-#: rendered their sibling text and NO tool UI, while this shape produced a "Used tool" group. A-#: corpus full of parts the app ignores looks richer and measures nothing.+#: The stored part shape that actually renders a tool block. VERIFIED, not assumed: a flat+#: `{"type": "tool-call"}` and an assistant-ui `{"type": "tool-invocation"}` both rendered+#: their sibling text and NO tool UI, while this shape produced a "Used tool" group.TOOL_NAMES = ("web_search", "code_execution", "python", "terminal", "search_knowledge_base")# The escalating cycle. Reasoning first because a turn's reasoning is what the pane holds open.CYCLE_BASE = ((("reasoning", 10_000), ("code", 8_000)),)-# Doubling stops here. Without a cap the eighth cycle is a single 2.5M-character reply, which is-# not a thread, it is one document -- and it makes the 1M rung one turn deep, where every-# per-message cost this tool exists to measure is multiplied by one.+# Doubling stops here: without a cap the eighth cycle is a single 2.5M-character reply, making+# the 1M rung one turn deep, where every per-message cost this tool measures is x1.MAX_UNIT_CHARS = 320_000-# Text below `units.jsonl` ships for every rung up to and including this many characters. Beyond-# it the units are regenerated on the tester's machine and checked against the manifest hash, so-# drift is still impossible; only the bytes are not carried in the artifact.+# Text below `units.jsonl` ships for every rung up to this many characters; beyond it units+# are regenerated on the tester's machine and checked against the manifest hash.SHIPPED_CHARS_BUDGET = 460_000-# ── the vocabulary ──────────────────────────────────────────────────-#-# Deliberately mundane. The point of the corpus is span density and uniqueness, not realism of-# meaning, and a vocabulary a reader can skim is a vocabulary a reader can spot a bug in.+# Deliberately mundane vocabulary. The point of the corpus is span density and uniqueness, not+# realism of meaning, and a vocabulary a reader can skim is one a reader can spot a bug in.+# ── the vocabulary ──────────────────────────────────────────────────_NOUNS = ("buffer","scheduler",@@ -204,18 +195,12 @@ _SHORT = ("tmp",)+# Corpus v1 contained not one dollar sign across 519,859 characters, making every math cost+# unmeasurable by construction: `preprocessLaTeX` measured as a real cost in isolation and as+# an exact NULL in the browser, because the fixture gave it nothing to do. Both delimiter+# families are here on purpose: `$...$` is what remark-math consumes directly, while `\(...\)`+# is what `preprocessLaTeX` exists to REWRITE.# ── math ────────────────────────────────────────────────────────────-#-# Corpus v1 contained not one dollar sign across 519,859 characters, which made every math cost in-# the app unmeasurable by construction. That is not a hypothetical gap: `preprocessLaTeX` was-# measured as a real cost in isolation and then as an exact NULL in the browser, and the reason was-# that the fixture gave it nothing to do. A benchmark that cannot see a cost cannot be used to-# argue the cost is not there.-#-# Both delimiter families are here on purpose. `$...$` and `$$...$$` are what remark-math consumes-# directly; `\(...\)` and `\[...\]` are what `preprocessLaTeX` exists to REWRITE into that form, so-# a corpus carrying only the first kind exercises the renderer while leaving the preprocessor's-# actual work uncovered._MATH_OPS = ("+", "-", "\\cdot", "\\times", "\\oplus")_MATH_RELS = ("=", "\\le", "\\ge", "\\approx", "\\equiv")_MATH_FUNCS = ("\\log", "\\exp", "\\sin", "\\cos", "\\tanh")@@ -232,25 +217,20 @@ _MATH_GREEK = ("\\omega",)-# Chance that a non-preamble prose BLOCK is a display-math block instead.-#-# Math takes prose's slot rather than being appended alongside it, and is drawn from the same-# jittered size distribution. That keeps the prose/fence alternation exactly as it was: same number-# of fences, same fence sizes, so the Shiki span density the whole corpus is calibrated to is-# untouched by construction rather than by a re-tuning nobody would re-check.+# Chance that a non-preamble prose BLOCK is a display-math block instead. Math takes prose's+# slot rather than being appended alongside it, drawn from the same jittered size+# distribution, so the fence count and sizes, and the Shiki span density the corpus is+# calibrated to, are untouched by construction.MATH_BLOCK_PROB = 0.16-# Chance that a sentence in a non-preamble prose block carries an inline expression. Inline math is-# the common case in real chat replies, and it is the one that interleaves with text rather than-# sitting in its own block, which is a different path through the markdown pipeline.-#-# Inline math is spent from the prose block's own character budget, for the same reason as above.+# Chance that a sentence in a non-preamble prose block carries an inline expression. Inline+# math is the common case in real chat replies and interleaves with text rather than sitting+# in its own block, a different path through the markdown pipeline. Spent from the prose+# block's own character budget.INLINE_MATH_PROB = 0.22-# The PREAMBLE stays pure prose, deliberately. Its whole job is to be the stretch that builds no-# spans and holds 60 fps, so the onset of cost has somewhere to be visible against. Putting math in-# it would give the preamble a rendering cost and destroy the only fence-free, math-free baseline-# the film has.+# The PREAMBLE stays pure prose, deliberately: its job is to be the stretch that builds no+# spans and holds 60 fps, so the onset of cost has somewhere to be visible against.@dataclass(frozen = True)@@ -258,14 +238,14 @@ class Unit:"""One assistant turn's worth of content."""index: int- kind: str # "reasoning" | "code"- reasoning: str # goes out as delta.reasoning_content- content: str # goes out as delta.content+ kind: str+ reasoning: str+ content: strchars: intsha256: str- #: Stored tool-call parts for this turn, in the shape the app renders. Not counted in `chars`:- #: `chars` is the rung's size axis and must stay comparable with every earlier measurement,- #: while a tool call's cost is a mount, not a character count. Reported separately.+ #: Stored tool-call parts for this turn, in the shape the app renders. Not counted in `chars`,+ #: which is the rung's size axis and must stay comparable with every earlier measurement,+ #: while a tool call's cost is a mount.tool_calls: tuple = ()@property@@ -310,8 +290,8 @@ class Unit:size += len(b) + 2return "\n\n".join(out)- # Keep the unit's own reasoning/content split, so a clipped turn has the same shape as a- # whole one and the smallest rung is not accidentally all-visible or all-reasoning.+ # Keep the unit's own reasoning/content split, so a clipped turn has the same shape as a whole+ # one and the smallest rung is not accidentally all-visible or all-reasoning.reasoning_budget = max(1, int(chars * (len(self.reasoning) / self.chars)))reasoning = take(self.reasoning, reasoning_budget)content = take(self.content, max(1, chars - len(reasoning)))@@ -322,13 +302,13 @@ class Unit:reasoning = reasoning,content = content,chars = len(text),- # A DIFFERENT digest, deliberately. A clipped unit is not the frozen unit and must not- # be checked against its hash or reported under it.+ # A DIFFERENT digest, deliberately: a clipped unit is not the frozen unit and must not be+ # checked against its hash.sha256 = "clip:"+ hashlib.sha256(f"{self.sha256}\x00{chars}".encode("utf-8")).hexdigest(),- # Tool calls survive clipping. They carry no `chars` weight, so dropping them would- # silently remove a whole component from exactly the rung -- the smallest -- that every- # growth ratio is taken against.+ # Tool calls survive clipping: they carry no `chars` weight, so dropping them would silently+ # remove a whole component from exactly the rung, the smallest, that every growth ratio is+ # taken against.tool_calls = self.tool_calls,)@@ -477,23 +457,20 @@ def _fence(size = len(lines[0]) + 1i = 0# SHORT identifiers and dense punctuation, because the density target is a SPAN count, not a- # character count. Long descriptive names are cheap to highlight per character: the field- # capture ran at 5.6 characters per span, and a first version of this generator with- # `cumulative_buffer_0007_12`-style names measured 10.1, i.e. half the highlighter work per- # character of the content it is standing in for. Real code is mostly operators.+ # character count: the field capture ran at 5.6 characters per span, and a first generator+ # with `cumulative_buffer_0007_12`-style names measured 10.1, half the work per character.r = rng.randintdef v() -> str:- # A MIX of short and long names. All-short measured 3.95 characters per span against the- # field's 5.6 and all-long measured 10.1, so neither extreme stands in for real code. The- # mix is the tuning knob for the offline proxy; the number that actually gets reported is- # the span density MEASURED in the DOM at run time, because Shiki merges adjacent- # same-scope tokens and no offline count can predict that exactly.+ # A MIX of short and long names: all-short measured 3.95 characters per span against the+ # field's 5.6 and all-long 10.1, so neither extreme stands in for real code. The mix tunes the+ # offline proxy; the reported number is the span density MEASURED in the DOM at run time,+ # because Shiki merges adjacent same-scope tokens.return rng.choice(_SHORT) if rng.random() < 0.55 else rng.choice(_NOUNS)while size < target:- # ONE salted identifier per line is all the uniqueness Shiki's source-keyed cache needs,- # and it leaves the rest of the line free to be dense.+ # ONE salted identifier per line is all the uniqueness Shiki's source-keyed cache needs, and+ # it leaves the rest of the line free to be dense.u = f"{rng.choice(_SHORT)}{salt}{i}"if lang == "python":line = (@@ -553,9 +530,9 @@ def _body(rng: random.Random, target: int, salt: str, *, preamble: bool) -> str:parts.append(head)size += len(head) + 2while size < target:- # Math takes the prose slot rather than being added alongside it, and is drawn from the- # same size distribution, so the fence blocks keep their count and their sizes and the- # span-density calibration in the module docstring still holds.+ # Math takes the prose slot rather than being added alongside it, drawn from the same size+ # distribution, so the fence blocks keep their count and sizes and the span-density+ # calibration still holds.want = _jitter(rng, PROSE_CHARS)if rng.random() < MATH_BLOCK_PROB:p = _math_block(rng, want, f"{salt}{len(parts)}")@@ -674,11 +651,10 @@ def freeze("""Write `units.jsonl` and `manifest.json`. Run deliberately; never on a benchmark run."""out_dir.mkdir(parents = True, exist_ok = True)shipped = units_for_chars(max_chars, seed)- # The manifest covers every unit any rung can reach, including the ones whose text is not- # shipped, so a regenerated unit at the 1M rung is still checked byte for byte. SIZED FROM THE- # LADDER, by `manifest_unit_count`, rather than from a bare character budget: the streamed turn- # and its follow-ups live PAST the seeded prefix, and a manifest that stops where the prefix- # stops leaves them nothing to point at.+ # The manifest covers every unit any rung can reach, including ones whose text is not shipped,+ # so a regenerated unit at 1M is still checked byte for byte. SIZED FROM THE LADDER by+ # `manifest_unit_count` rather than from a character budget: the streamed turn and its+ # follow-ups live PAST the seeded prefix.all_units = [generate_unit(i, seed) for i in range(manifest_unit_count(seed))]with (out_dir / "units.jsonl").open("w", encoding = "utf-8") as fh:for u in shipped:@@ -742,9 +718,8 @@ class Corpus:@classmethoddef load(cls, frozen_dir: Optional[Path] = None) -> "Corpus":- # Through the resource loader, NOT a filesystem path. Inside `studiobench.pyz` the frozen- # corpus is a zip member and every Path built from __file__ points at something that- # cannot exist; the built artifact's own --doctor is what caught it.+ # Through the resource loader, NOT a filesystem path: inside `studiobench.pyz` the frozen+ # corpus is a zip member and every Path built from __file__ points at nothing.from ..runtime import resourcesif frozen_dir is not None:@@ -833,42 +808,37 @@ RUNGS: dict[str, int] = {"1M": 1_000_000,}-# The PROVISIONAL ratio used to size the corpus before anything has been tokenised. It is never-# reported: `chars_per_token` in every row is the MEASURED one, and it is measured per rung.+# The PROVISIONAL ratio used to size the corpus before anything has been tokenised. Never+# reported: `chars_per_token` in every row is the MEASURED one, per rung.PROVISIONAL_CHARS_PER_TOKEN = 4.0-# How much of the last turn STREAMS, at every rung. See the long note in `plan_rung`.-#-# Sized from the film, not chosen round: the quick scene's during-generation slots occupy the-# first 16 s and the first after-generation slot opens at 22.5 s, so the stream has to finish-# inside roughly 20 s. At the field cadence of 24 characters every 73 ms -- 328.8 chars/s -- that-# is about 6,500 characters. Rounded DOWN so the stream reliably drains before the first action-# that claims the reply is complete, rather than finishing exactly as it starts.+# How much of the last turn STREAMS, at every rung (see the long note in `plan_rung`). Sized+# from the film: the quick scene's first after-generation slot opens at 22.5 s, so the stream+# must finish inside roughly 20 s, which at the field cadence of 328.8 chars/s is about 6,500+# characters. Rounded DOWN so the stream drains before the first action that claims the reply+# is complete.STREAM_TAIL_CHARS = 6_000-# How many turns actually stream per cell: the opening one plus the follow-ups the `send_turn`-# action sends mid-film. They SHARE the budget above rather than each getting it, so three turns-# and one turn take the same wall clock. Three because the corpus alternates reasoning-heavy and-# code-heavy units, so three consecutive turns are guaranteed to cover both kinds.+# How many turns stream per cell: the opening one plus the `send_turn` follow-ups. They SHARE+# the budget above rather than each getting it, so three turns and one turn take the same wall+# clock. Three because the corpus alternates reasoning-heavy and code-heavy units.STREAM_TURNS = 3-# Each follow-up's size. Small on purpose: the follow-ups exist to sample "what does a chunk cost-# given what is already on screen" at two more points inside the cell, not to add mass.+# Each follow-up's size. Small on purpose: the follow-ups sample "what does a chunk cost given+# what is already on screen" at two more points, not to add mass.FOLLOW_UP_CHARS = 1_500-# Below this the rung streams ONCE. Three turns need about 9,000 characters of budget and the 1K-# rung is 4,000 characters in total, so splitting it three ways produced an opening stream that-# drained in four seconds -- shorter than the during-generation slots timed against it -- and a-# rung 40% over its own target. A small rung is allowed to be a single exchange, which is also-# what a 1K-token conversation actually is.+# Below this the rung streams ONCE: three turns need about 9,000 characters of budget while+# the 1K rung is 4,000 in total, so splitting it three ways gave an opening stream draining in+# four seconds, shorter than the during-generation slots timed against it, and a rung 40% over+# target.MULTI_TURN_MIN_CHARS = 20_000-# The largest characters-per-token the FROZEN CORPUS is sized for, which is not the same number as-# the provisional ratio used to plan a rung. `plan_rung` takes a measured ratio, and a corpus sized-# at exactly the provisional 4.0 has no material left the moment a machine measures 4.1. A quarter-# of headroom costs a handful of extra manifest entries -- the text of an unshipped unit is-# regenerated only when a rung actually reaches it -- and buys every ratio the seeder can plausibly-# report. Past it `plan_rung` refuses rather than degrades.+# The largest characters-per-token the FROZEN CORPUS is sized for, which is not the+# provisional ratio used to plan a rung: a corpus sized at exactly 4.0 has no material left+# the moment a machine measures 4.1. A quarter of headroom costs a handful of extra manifest+# entries and buys every ratio the seeder can plausibly report; past it `plan_rung` refuses+# rather than degrades.MANIFEST_CHARS_PER_TOKEN = 5.0@@ -902,8 +872,8 @@ class RungPlan:target_chars: intseeded_units: list[Unit] = field(default_factory = list)streamed_unit: Optional[Unit] = None- #: Further turns streamed DURING the film by the `send_turn` action. They come out of the same- #: fixed streaming budget as the first, split between them, so more turns cost no wall clock.+ #: Further turns streamed DURING the film by `send_turn`. They come out of the same fixed+ #: streaming budget as the first, split between them, so more turns cost no wall clock.follow_up_units: list[Unit] = field(default_factory = list)@property@@ -920,9 +890,9 @@ class RungPlan:@propertydef total_chars(self) -> int:- # The follow-ups are part of the rung's mass: they are streamed INTO the thread during the- # film and are on screen for most of it. Leaving them out understated every rung by the- # follow-up budget, which at the 1K rung is most of the rung.+ # The follow-ups are part of the rung's mass: they are streamed INTO the thread and are on+ # screen for most of it, so leaving them out understated every rung by the follow-up budget,+ # which at 1K is most of the rung.return self.seeded_chars + self.streamed_chars + self.follow_up_chars@@ -974,9 +944,9 @@ def dollarise(text: str, salt: str) -> str:fenced = not fencedout.append(line)continue- # Inside a fence, shell-shaped lines. Outside it, prices in prose. Both are what a real- # answer contains, and they exercise different branches: the code-region scan has to- # EXCLUDE the first and the currency heuristic has to escape the second.+ # Inside a fence, shell-shaped lines; outside it, prices in prose. They exercise different+ # branches: the code-region scan must EXCLUDE the first and the currency heuristic must escape+ # the second.if fenced and index % 11 == 0:out.append(f"{line} # $HOME/{salt}{index} costs $1{index % 10}.99")elif not fenced and line and index % 17 == 0:@@ -1109,44 +1079,33 @@ def plan_rung(target_chars = int(tokens * chars_per_token)# THE STREAMED TAIL IS THE SAME SIZE AT EVERY RUNG, and the whole size ladder lives in the- # seeded prefix. This is not tidiness, it is what makes the film mean anything above 10K.- #- # The scene is a FIXED-DURATION film whose slots are wall-clock times: three actions run- # "during generation" and ten run "after the reply is complete". At the field's own cadence- # of 24 characters every 73 ms, a streamed tail that grows with the rung takes 11 s at 1K,- # 54 s at 10K, 354 s at 100K and 811 s at 1M -- against a film 135 s long. Above 10K the- # stream outlasts the entire film, so every action labelled "after the reply is complete"- # runs mid-generation. The run still completes and still prints a full table; the labels are- # simply false, which is worse than a crash.- #- # Holding the tail constant also isolates the variable under investigation. The question is- # what a streamed chunk costs AS A FUNCTION OF THE THREAD ALREADY ON SCREEN, and that needs- # the chunk workload held fixed while the thread grows, not both moving together.- # `stream_tail_chars` overrides the constant, and is the ONLY way to vary reply length in this- # tool. The paragraphs above explain why the tail is pinned: the rung ladder exists to ask what- # a chunk costs as a function of the THREAD, and that needs the chunk workload held fixed. The- # consequence, which cost a whole investigation to rediscover, is that a cost scaling with the- # length of the reply BEING STREAMED is constant across every rung and reads as a floor. Such a- # mechanism is real and this ladder cannot see it at any effect size.- #- # Raising this makes the film's labels false in the way the note above describes, because the- # stream then outlasts the after-generation slots. That is a trade a reply-length investigation- # has to make deliberately, so it is a parameter and not a second constant, and `--assert-- # liveness` on the resulting payload is how the cost of making it shows up.+ # seeded prefix. The scene is a FIXED-DURATION film whose slots are wall-clock times, and at+ # the field cadence a tail that grew with the rung would take 54 s at 10K, 354 s at 100K and+ # 811 s at 1M against a 135 s film, so above 10K every action labelled "after the reply is+ # complete" would run mid-generation while the run still printed a full table under false+ # labels. Holding the tail constant also isolates the variable: what a streamed chunk costs+ # AS A FUNCTION OF THE THREAD ALREADY ON SCREEN.+ # The field's own cadence is 24 characters every 73 ms.+ # `stream_tail_chars` overrides the constant and is the ONLY way to vary reply length in this+ # tool. The consequence, which cost a whole investigation to rediscover, is that a cost+ # scaling with the length of the reply BEING STREAMED is constant across every rung and reads+ # as a floor: such a mechanism is real and this ladder cannot see it at any effect size.+ # Raising this makes the film's labels false as described above, which is why it is a+ # parameter rather than a second constant.turns = STREAM_TURNS if target_chars >= MULTI_TURN_MIN_CHARS else 1tail_budget = STREAM_TAIL_CHARS if stream_tail_chars is None else max(1, stream_tail_chars)tail_target = min(tail_budget, target_chars) if stream_tail_chars is None else tail_budgetfollow_budget = FOLLOW_UP_CHARS * (turns - 1)seed_target = max(0, target_chars - tail_target - follow_budget)- # Size the SEEDED PREFIX to the remainder, then trim its last unit to land on the target.- # Growing it a whole unit at a time instead overshoots by up to one turn, which at the 1K- # rung means 13,333 characters against a 4,000-character budget.+ # Size the SEEDED PREFIX to the remainder, then trim its last unit to land on the target:+ # growing it a whole unit at a time overshoots by up to one turn, which at 1K means 13,333+ # characters against a 4,000-character budget.last_index = max(e["index"] for e in corpus.manifest["units"])try:seeded = corpus.units_for_chars(seed_target) if seed_target > 0 else []except KeyError as exc:- # The prefix alone outgrew the manifest, before any streamed turn was even asked for.+ # The prefix alone outgrew the manifest, before any streamed turn was asked for.raise _too_small(rung, chars_per_token, last_index, "its seeded prefix alone") from excif seeded:overshoot = sum(u.chars for u in seeded) - seed_target@@ -1154,15 +1113,10 @@ def plan_rung(seeded[-1] = seeded[-1].clipped_to(max(1, seeded[-1].chars - overshoot))# The streamed turn is the next one after the prefix, clipped to the fixed tail, and each- # follow-up is the one after that.- #- # NO CLAMP HERE, deliberately. This used to be `min(index, last_index)`, so a rung whose prefix- # reached the end of the manifest silently got the final unit three times over: the opening- # stream re-sent text the seeded prefix already held, both follow-ups were byte-identical, and- # every fence in all three hit Shiki's source-keyed cache. Running off the end is a corpus that- # is too small for the ladder, and the only honest answer is to say so and stop. `--freeze`- # sizes the manifest from the ladder (`manifest_unit_count`) precisely so this cannot be- # reached by any supported configuration.+ # follow-up is the one after that. NO CLAMP HERE, deliberately: `min(index, last_index)`+ # silently gave a rung whose prefix reached the end of the manifest the final unit three times+ # over, so both follow-ups were byte-identical and every fence hit Shiki's source-keyed cache.+ # `--freeze` sizes the manifest from the ladder precisely so this cannot be reached.needed = len(seeded) + turns - 1if needed > last_index:raise _too_small(@@ -1174,25 +1128,17 @@ def plan_rung()# The streaming budget is SPLIT across the opening turn and the follow-ups, so a cell that- # streams three times takes the same wall clock as one that streams once. Turns are taken from- # consecutive corpus units, which alternate reasoning-heavy and code-heavy, so the follow-ups- # exercise the `<think>` re-parse and the Streamdown path rather than repeating one of them.- # The opening turn keeps the FULL tail budget and the follow-ups are extra. The three- # during-generation slots are timed against the opening stream, so it has to outlast them;- # splitting the budget between turns left it draining in four seconds and those slots then ran- # against a finished reply while still being labelled "during generation".+ # streams three times takes the same wall clock as one that streams once. Turns are+ # consecutive corpus units, which alternate reasoning-heavy and code-heavy. The opening turn+ # keeps the FULL tail budget and the follow-ups are extra: the three during-generation slots+ # are timed against the opening stream, and splitting the budget left it draining in four+ # seconds.source = corpus.unit(len(seeded))- # THE REQUESTED TAIL HAS TO BE DELIVERABLE. Keyed on `clipped_to`'s own early return- # (`if chars >= self.chars: return self`) rather than on a tolerance, because the two ways a- # tail lands under its budget are different in kind and only one of them is a defect: clipping- # at a whole BLOCK boundary loses at most one block and is what keeps a prefix from ending- # inside a fence, while exceeding the unit loses everything above it and is unbounded. A- # percentage bound cannot tell those apart, and a bound nobody can derive is an unacknowledged- # bias rather than a tolerance.- #- # The DEFAULT path is exempt on purpose: `STREAM_TAIL_CHARS` is a ceiling the small rungs are- # legitimately under (the 1K rung is 4,000 characters in total, less than one tail), and that- # shortfall is the rung being small rather than the corpus failing to answer.+ # THE REQUESTED TAIL HAS TO BE DELIVERABLE, keyed on `clipped_to`'s own early return rather+ # than a tolerance: clipping at a whole BLOCK boundary loses at most one block and is what+ # keeps a prefix from ending inside a fence, while exceeding the unit loses everything above+ # it and is unbounded, and a percentage bound cannot tell those apart. The DEFAULT path is+ # exempt on purpose, since `STREAM_TAIL_CHARS` is a ceiling the small rungs are under.if stream_tail_chars is not None and tail_target >= source.chars:raise _tail_not_deliverable(rung,@@ -1205,7 +1151,7 @@ def plan_rung(streamed = source.clipped_to(tail_target)follow_ups = [corpus.unit(len(seeded) + i).clipped_to(FOLLOW_UP_CHARS) for i in range(1, turns)]if dollars:- # The streamed turns only. The seeded prefix is rendered once at mount and never+ # The streamed turns only: the seeded prefix is rendered once at mount and never# re-preprocessed, so dollars there would change the corpus without changing what the# per-frame path is asked to do.streamed = replace(diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.pyindex d0951d0e4ea..821cca875f3 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_corpus_math.py@@ -35,14 +35,12 @@ FENCE = re.compile(r"```.*?```", re.S)DISPLAY = (re.compile(r"\$\$\n(.*?)\n\$\$", re.S), re.compile(r"\\\[\n(.*?)\n\\\]", re.S))INLINE = (re.compile(r"\$ ([^$\n]+?) \$"), re.compile(r"\\\( (.+?) \\\)"))-# The v1 corpus, measured before math was added.-#-# The tolerance is set from measurement rather than taste. With MATH_BLOCK_PROB at 0 the share is-# 0.4754, i.e. EXACTLY v1, which is the substitution design working: inline math is spent from the-# prose block's own budget and costs the fence share nothing. At the shipped 0.16 it reads 0.4779,-# a drift of 0.0025. At 0.60 it reads 0.4689, a drift of 0.0065. So 0.005 leaves the shipped value-# a factor of two of headroom while failing well before the parameter could be raised far enough to-# re-weight the film.+# The v1 corpus, measured before math was added. The tolerance is set from measurement: with+# MATH_BLOCK_PROB at 0 the share is 0.4754, EXACTLY v1, which is the substitution design working,+# since inline math is spent from the prose block's own budget. At the shipped 0.16 it reads+# 0.4779 and at 0.60 it reads 0.4689, so 0.005 leaves the shipped value a factor of two of+# headroom while failing well before the parameter could re-weight the film.+# Drifts of 0.0025 and 0.0065 against v1.V1_FENCE_SHARE = 0.4754FENCE_SHARE_TOLERANCE = 0.005@@ -65,10 +63,9 @@ def test_the_corpus_contains_math_at_all():def test_both_delimiter_families_are_present():- # `$...$` is what remark-math consumes directly. `\(...\)` and `\[...\]` reach the renderer- # ONLY if preprocessLaTeX rewrites them first, so a corpus carrying only the first kind- # exercises the renderer while leaving the preprocessor uncovered, which is the situation this- # corpus version exists to end.+ # `$...$` is what remark-math consumes directly. `\(...\)` and `\[...\]` reach the renderer ONLY+ # if preprocessLaTeX rewrites them first, so a corpus carrying only the first kind leaves the+ # preprocessor uncovered.joined = "\n".join(_texts())assert joined.count("$$") > 0assert joined.count("\\[") > 0@@ -76,8 +73,8 @@ def test_both_delimiter_families_are_present():def test_math_is_spread_across_the_thread_not_pooled_into_one_turn():- # A single maths-heavy turn measures one large KaTeX tree. The cost this is meant to expose- # accumulates over a long thread, so it has to be present in most turns.+ # A single maths-heavy turn measures one large KaTeX tree; the cost this exposes accumulates over+ # a long thread, so it has to be present in most turns.texts = _texts()with_math = [t for t in texts if _count(t, DISPLAY) or _count(t, INLINE)]assert len(with_math) >= max(2, int(len(texts) * 0.8))@@ -95,10 +92,10 @@ def test_there_is_both_display_and_inline_math():def test_the_fence_share_is_what_it_was_before_math_existed():- # The span-density calibration in the module docstring is a statement about how much of the- # corpus is fenced code. Math takes the PROSE slot and is drawn from the prose size- # distribution precisely so this number does not move; if it has moved, the 5.6 chars/span- # target no longer describes this film and the docstring is lying.+ # The span-density calibration in the module docstring is a statement about how much of the corpus+ # is fenced code. Math takes the PROSE slot and is drawn from the prose size distribution+ # precisely so this number does not move; if it has moved, the 5.6 chars/span target no longer+ # describes this film.texts = _texts()total = sum(len(t) for t in texts)fenced = sum(len(m.group(0)) for t in texts for m in FENCE.finditer(t))@@ -106,9 +103,9 @@ def test_the_fence_share_is_what_it_was_before_math_existed():def test_the_preamble_stays_free_of_math_and_fences():- # The field capture held a flat 60 fps and exactly zero spans for its first 33,348 characters.- # The preamble stands in for that. Give it math and the film loses the one stretch against- # which the onset of cost is visible, and every rung starts with a rendering cost instead.+ # The field capture held a flat 60 fps and exactly zero spans for its first 33,348 characters, and+ # the preamble stands in for that. Give it math and the film loses the one stretch against which+ # the onset of cost is visible.for index in range(4):unit = generate_unit(index)head = unit.reasoning[: int(len(unit.reasoning) * 0.20)]@@ -119,8 +116,8 @@ def test_the_preamble_stays_free_of_math_and_fences():def test_prose_without_the_math_flag_has_none():- # `_prose` is also what builds tool arguments and results, where a stray dollar sign would be- # rendered by a different component than the one under test.+ # `_prose` also builds tool arguments and results, where a stray dollar sign would be rendered by+ # a different component than the one under test.import randomtext = _prose(random.Random(7), 4_000, "x")@@ -168,8 +165,8 @@ def test_the_shipped_corpus_matches_the_generator_byte_for_byte():def test_the_manifest_records_the_math_parameters():- # The corpus hash covers the generator's parameters as well as its bytes, so two corpora with- # the same text and different declared densities cannot compare as identical.+ # The corpus hash covers the generator's parameters as well as its bytes, so two corpora with the+ # same text and different declared densities cannot compare as identical.manifest = Corpus.load().manifestassert "math_block_prob" in manifestassert "inline_math_prob" in manifestdiff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.pyindex 288d98a0a1f..6af622d701e 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_digest.py@@ -37,11 +37,11 @@ from studiobench.analysis import parity as P # noqa: E402PARITY_JS = Path(__file__).resolve().parents[2] / "scene" / "parity.js"-# A DOM shim, not a DOM. `signature()` touches exactly six things on an element, so those six are-# what the harness provides; anything richer would be a second browser to keep correct. `capture()`-# is deliberately NOT exercised here because it needs querySelectorAll, getComputedStyle and the-# real selector adapter -- that is the live null and spike controls' job, and pretending to cover-# it with a mock is how a test suite comes to assert its own fixtures.+# A DOM shim, not a DOM: `signature()` touches exactly six things on an element, so those six+# are what the harness provides; anything richer would be a second browser to keep correct.+# `capture()` is deliberately NOT exercised here because it needs querySelectorAll,+# getComputedStyle and the real selector adapter, which is the live null and spike controls'+# job.HARNESS_JS = r"""const fs = require("fs");const src = fs.readFileSync(process.argv[2], "utf8");@@ -148,8 +148,8 @@ def test_relative_and_absolute_times_collapse():def test_backend_minted_uuids_collapse():- # The volatile that made the FIRST null control fail on all eighteen actions: every message- # root carries `data-message-id`, and the two arms are two installs with two databases.+ # The volatile that made the FIRST null control fail on all eighteen actions: every message root+ # carries `data-message-id`, and the two arms are two installs with two databases.a, b = norm_text("id 71ad5735-ede4-464d-a36b-44309ef67624", "id f44017dd-f5f7-45dd-9f53-475c115e61ac")@@ -181,12 +181,11 @@ def test_urls_lose_their_origin_but_keep_their_path():assert urls[3] == "#BLOB" and urls[4].startswith("#DATA:"), urls-# ── the normaliser: things that MUST SURVIVE ────────────────────────-## Every test above widens the set of things the digest cannot see. These are the counterweight:# a normaliser that erased them would pass a null control perfectly and detect nothing.+# ── the normaliser: things that MUST SURVIVE ────────────────────────def test_a_bare_number_is_not_a_duration():a, b = norm_text("3 files changed", "4 files changed")assert a != b, (a, b)@@ -237,8 +236,8 @@ def test_adding_or_removing_a_boolean_attribute_moves_the_signature():def test_a_volatile_attribute_keeps_its_presence_even_though_its_value_is_dropped():- # Dropping the value must not drop the fact that the attribute is there: an element that- # gains an `id` has changed, even though which id it gained is noise.+ # Dropping the value must not drop the fact that the attribute is there: an element that gains+ # an `id` has changed, even though which id it gained is noise.plain, with_id = sigs({"tag": "div", "attrs": {}}, {"tag": "div", "attrs": {"id": "radix-:r1a:"}})@@ -286,8 +285,8 @@ def test_added_and_removed_elements_move_the_signature():def test_reordered_siblings_move_the_signature():- # Two elements with identical content in the other order. A digest built from a SET rather- # than a sequence would read these as equal, and a list that renders backwards is a real bug.+ # Two elements with identical content in the other order. A digest built from a SET rather than+ # a sequence would read these as equal, and a list that renders backwards is a real bug.one, two = sigs({"tag": "ul",@@ -311,8 +310,8 @@ def test_nesting_moves_the_signature():def test_attribute_order_does_not_move_the_signature():- # React can emit attributes in either order for the same render. Sorting them is what makes- # the digest a property of the DOM rather than of the serialiser.+ # React can emit attributes in either order for the same render; sorting them makes the digest a+ # property of the DOM rather than of the serialiser.one, two = sigs({"tag": "div", "attrs": {"class": "a", "data-state": "open"}},{"tag": "div", "attrs": {"data-state": "open", "class": "a"}},@@ -337,8 +336,7 @@ def test_the_depth_cap_leaves_a_visible_marker():def test_content_below_the_depth_cap_is_not_compared():- # The honest statement of the limit: past 40 levels the digest stops looking, and this test- # records that as a KNOWN hole rather than leaving somebody to discover it.+ # The honest statement of the limit: past 40 levels the digest stops looking, recorded here as a KNOWN hole.def wrap(inner, n):for _ in range(n):inner = {"tag": "div", "children": [inner]}@@ -385,8 +383,8 @@ def test_identical_captures_match():def test_a_failed_capture_is_never_a_match():- # The single most dangerous confusion in the whole instrument: a capture that threw and a- # capture that agreed both produce no complaint unless they are told apart here.+ # The single most dangerous confusion in the instrument: a capture that threw and a capture that+ # agreed both produce no complaint unless they are told apart here.failed = {"parity_attempted": False, "reason": "threadRoot is not a function"}got = P.compare(failed, capture())assert got["verdict"] == P.NOT_COMPARABLE@@ -395,7 +393,7 @@ def test_a_failed_capture_is_never_a_match():def test_two_different_roots_are_not_comparable():- # A body-root capture carries the sidebar and its relative timestamps. Comparing it with a+ # A body-root capture carries the sidebar and its relative timestamps; comparing it with a# thread-root one produces two plausible hashes and a meaningless verdict.got = P.compare(capture(root = "thread"), capture(root = "body"))assert got["verdict"] == P.NOT_COMPARABLE@@ -439,7 +437,7 @@ def test_an_added_message_is_localised_as_one_sided():def test_an_overlay_that_changes_without_changing_count_is_still_localised():# The bug this pins: comparing only the NUMBER of overlays passes an open menu whose contents- # were rewritten, which is exactly the popover regression the overlay walk was added for.+ # were rewritten, which is the popover regression the overlay walk was added for.one = capture("aaaa", overlays = [{"sel": '[role="menu"]', "digest": "o1", "chars": 40}])two = capture("zzzz", overlays = [{"sel": '[role="menu"]', "digest": "o2", "chars": 44}])got = P.compare(one, two)@@ -448,7 +446,7 @@ def test_an_overlay_that_changes_without_changing_count_is_still_localised():def test_an_overlay_change_alone_is_a_difference():# THE FALSE NEGATIVE THE SPIKE CONTROL FOUND. An overlay lives outside the thread root, so a- # menu that mounts when it should not leaves the whole-thread digest untouched. Testing only+ # menu that mounts when it should not leaves the whole-thread digest untouched; testing only# that digest made the entire overlay walk unreachable and reported a clean pass.one = capture("aaaa", overlays = [])two = capture("aaaa", overlays = [{"sel": '[role="menu"]', "digest": "o1", "chars": 40}])@@ -459,7 +457,7 @@ def test_an_overlay_change_alone_is_a_difference():def test_a_message_change_alone_is_a_difference():# The same shape one level down: if a per-message digest moves while the whole-thread digest- # somehow does not, the pair still differs. Belt and braces on the aggregate hash.+ # somehow does not, the pair still differs.moved = capture("aaaa",messages = [@@ -471,7 +469,7 @@ def test_a_message_change_alone_is_a_difference():def test_a_difference_outside_every_message_is_reported_as_such():- # An empty `moved` list would read as "nothing differs" next to a DIFFER verdict.+ # An empty `moved` list would read as 'nothing differs' next to a DIFFER verdict.got = P.compare(capture("aaaa"), capture("zzzz"))assert got["verdict"] == P.DIFFERassert got["moved"] and "scaffolding" in got["moved"][0]@@ -508,23 +506,24 @@ def test_mutation_detected_does_not_claim_a_detection_it_did_not_make():def test_an_action_that_never_ran_is_not_a_matching_surface():- # MEASURED, not imagined: on a 100K fast-tier null control, five of the eighteen actions did- # not run on either arm -- no attachments button, no Copy button, a missed slot. The window- # still closes and the digest is still captured, so both arms agreed and `image_upload` was- # reported as a stable, matching surface. Nobody had opened it.+ # MEASURED, not imagined: on a 100K fast-tier null control, five of eighteen actions did not run+ # on either arm (no attachments button, no Copy button, a missed slot). The window still closes+ # and the digest is still captured, so both arms agreed and `image_upload` was reported as a+ # stable, matching surface that nobody had opened.idle = {"ran": False, "reason": "no visible attachments button", "parity": capture()}got = P.compare_rows(idle, idle)assert got["verdict"] == P.NOT_EXERCISEDassert "nothing touched" in got["reason"]# NEITHER arm ran it, so nobody opened the surface on either build and the only thing lost is- # coverage. `one_sided` says so, and the caller needs it to keep that apart from the case below.+ # coverage. `one_sided` says so, and the caller needs it to keep that apart from the case+ # below.assert got["one_sided"] == ""def test_an_action_only_one_arm_could_perform_is_named_as_such():- # A control that stops opening leaves NO digest to differ: the arm that cannot reach it- # records `ran: false` and the pair carries no comparison at all. Folding that into the- # missed-slot case is how a button that no longer works reads as lost coverage.+ # A control that stops opening leaves NO digest to differ: the arm that cannot reach it records+ # `ran: false` and the pair carries no comparison. Folding that into the missed-slot case is+ # how a button that no longer works reads as lost coverage.idle = {"ran": False, "reason": "the control never became visible", "parity": capture()}got = P.compare_rows({"ran": True, "parity": capture()}, idle)assert got["verdict"] == P.NOT_EXERCISED@@ -573,8 +572,8 @@ def test_an_action_that_differs_against_itself_is_derived_as_unstable():def test_a_blind_action_is_counted_as_blind_and_not_as_stable():- # An action whose digest could never be captured has an observation count of zero. Reporting- # it as "stable" would be the instrument certifying a surface it never looked at.+ # An action whose digest could never be captured has an observation count of zero, and reporting+ # it as stable would be the instrument certifying a surface it never looked at.got = P.derive_unstable([("image_upload", {"verdict": P.NOT_COMPARABLE}),diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.pyindex 09c9a767940..76fe8b918ce 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_streamed.py@@ -68,13 +68,12 @@ from tests.studio.studiobench.fixture.selftest.test_studiobench_parity_digest imrun_js,)-# ── building a capture out of the SHIPPED signature ──────────────────-#-# Not a hand-written digest. The whole point of the node harness is that the thing under test is-# `scene/parity.js` as it ships, so these fixtures are DOM trees and every digest below is produced-# by walking them with the real `signature()`.+# Building a capture out of the SHIPPED signature. Not a hand-written digest: the thing under+# test is `scene/parity.js` as it ships, so these fixtures are DOM trees and every digest is+# produced by walking them with the real `signature()`.+# ── building a capture out of the SHIPPED signature ──────────────────def message(index: int,*,@@ -141,8 +140,8 @@ def capture(tree: dict, *, streaming_fields: bool = True) -> dict:"""messages = tree["children"][0]["children"]overlays = tree.get("_overlays") or []- # `elide` marks EVERY message, which is what `capture()` in parity.js does: the scaffold has to- # be the same walk on both arms, and which message is in flight is not.+ # `elide` marks EVERY message, as `capture()` in parity.js does: the scaffold has to be the same+ # walk on both arms, and which message is in flight is not.scaffold_tree = _mark_elided(tree)got = run_js({@@ -187,10 +186,10 @@ def capture(tree: dict, *, streaming_fields: bool = True) -> dict:return out-# The body of the reply, at four points in one stream. Not four different documents: the same one,-# with the trailing construct repaired differently at each point, which is what the renderer-# actually does. The `title` is the KaTeX parse error rehype-katex writes while the formula will-# not parse, and its character offset moves with every arriving character.+# The body of the reply at four points in one stream: the same document with the trailing+# construct repaired differently at each point, which is what the renderer does. The `title` is+# the KaTeX parse error rehype-katex writes while the formula will not parse, and its character+# offset moves with every arriving character.def streamed_body(chars: int) -> list:text = "The bounded shard coalesces the retained layout, except that the fibre stays inter"return [@@ -226,7 +225,7 @@ def streaming_arm()-#: Four points in one stream. Adjacent pairs are what two arms one paint apart look like.+#:Four points in one stream. Adjacent pairs are what two arms one paint apart look like.STREAM_POINTS = (12, 24, 48, 81)@@ -242,8 +241,7 @@ def test_the_same_document_at_two_points_in_one_stream_moves_the_raw_digest():"""a, b = streaming_arm(24), streaming_arm(48)assert a["digest"] != b["digest"], "no drift to fix; the fixture is not reproducing the defect"- # ...and the settled thread is byte-identical, which is what makes it a false alarm rather- # than a finding.+ # ...and the settled thread is byte-identical, which makes it a false alarm rather than a finding.assert a["digest_scaffold"] == b["digest_scaffold"]@@ -270,9 +268,9 @@ def test_two_genuinely_different_documents_still_differ_while_a_reply_streams():def test_a_real_difference_survives_the_streamed_message_drifting_at_the_same_time():- # The realistic case: the arms are at different points in the stream AND something real- # changed. Excusing the first must not excuse the second, and the streamed message must not- # appear in `moved` and drown the finding.+ # The realistic case: the arms are at different points in the stream AND something real changed.+ # Excusing the first must not excuse the second, and the streamed message must not appear in+ # `moved` and drown the finding.got = P.compare(streaming_arm(24, settled_body = "settled text"),streaming_arm(81, settled_body = "settled text, rewritten"),@@ -300,7 +298,7 @@ def test_the_null_score_is_zero():differing = [r for r in results if r["verdict"] == P.DIFFER]assert len(results) == 6assert not differing, f"NULL SCORE {len(differing)}/{len(results)}: {differing}"- # And they are refused, not passed. A null that reports MATCH on a pair whose digests plainly+ # And they are refused, not passed: a null reporting MATCH on a pair whose digests plainly# disagree would be the instrument certifying a surface it could not look at.assert all(r["verdict"] == P.NOT_COMPARABLE for r in results)@@ -314,15 +312,15 @@ def test_the_null_battery_scores_the_old_instrument_too():"""before = null_battery(streaming_fields = False)assert all(r["verdict"] == P.DIFFER for r in before), before- # Every one of them localised to the streamed message and to nothing else, which is what made- # them read as a UI change rather than as a clock.+ # Every one localised to the streamed message and nothing else, which is what made them read as+ # a UI change rather than a clock.for r in before:assert [m for m in r["moved"] if m.startswith("msg2(")] == r["moved"], r["moved"]def test_two_arms_that_landed_on_the_same_point_in_the_stream_still_match():- # The coverage this must NOT cost. Two arms at the same point serialise identically, which is- # exactly the claim this mode makes, so it stays a pass.+ # The coverage this must NOT cost: two arms at the same point serialise identically, which is+ # exactly the claim this mode makes.got = P.compare(streaming_arm(24), streaming_arm(24))assert got["verdict"] == P.MATCH, gotassert got["in_flight"] == [2]@@ -332,21 +330,20 @@ def test_a_settled_thread_is_scored_exactly_as_it_was():settled = capture(thread([message(0, role = "user", body = "hi"), message(1)]))assert settled["in_flight"] == [] and settled["streaming"] is Falseassert P.compare(settled, settled)["verdict"] == P.MATCH- # The scaffold is a SHORTER walk than the whole thread, not an equal one, and it has to be:- # every message is elided from it whether or not anything was streaming. What makes the reading- # complete again is the per-message rows, and the mutant battery is what shows they do.+ # The scaffold is a SHORTER walk than the whole thread and has to be, since every message is+ # elided whether or not anything was streaming. The per-message rows are what make the reading+ # complete again, and the mutant battery is what shows they do.assert settled["chars_scaffold"] < settled["chars"]-# ── 3. the mutant score ──────────────────────────────────────────────-#-# Every mutant is a real, visible rendering difference, injected WHILE A REPLY IS IN FLIGHT and at-# the SAME point in the stream on both arms, so the only thing the comparison can be reacting to is-# the mutation. A normaliser that reports "identical" without ever having been shown a difference-# is worthless, and one that stopped seeing these because a stream happened to be running would be-# worse than the drift it was written to fix.+# The mutant score. Every mutant is a real, visible rendering difference injected WHILE A REPLY+# IS IN FLIGHT at the SAME point in the stream on both arms, so the comparison can only be+# reacting to the mutation. A normaliser never shown a difference is worthless, and one that+# stopped seeing these because a stream happened to be running would be worse than the drift it+# was written to fix.+# ── 3. the mutant score ──────────────────────────────────────────────def mutants() -> list[tuple[str, dict, dict]]:at = 24@@ -428,8 +425,8 @@ def mutants() -> list[tuple[str, dict, dict]]:))# An overlay lives outside the thread root, so it has its own walk and its own way of going- # unnoticed. Both shapes, because a menu that mounts and a menu that is rewritten are- # different regressions.+ # unnoticed. Both shapes, because a menu that mounts and a menu that is rewritten are different+ # regressions.menu = {"sel": '[role="menu"]',"tree": {@@ -474,17 +471,16 @@ def test_the_mutant_score_is_total():def test_no_mutant_is_ever_reported_as_a_match():- # The weaker bar, held separately, because it is the one that decides whether a change can ship- # green. DIFFER is the outcome that names the change; NOT COMPARABLE is the outcome that- # refuses to say; only MATCH lets it through, and nothing here may reach it.+ # The weaker bar, held separately, because it decides whether a change can ship green: DIFFER+ # names the change, NOT COMPARABLE refuses to say, and only MATCH lets it through.for name, before, after in mutants():assert P.compare(before, after)["verdict"] != P.MATCH, namedef test_the_mutant_score_is_unchanged_by_the_streaming_fields():# The same battery scored as the OLD instrument would score it. Identical, which is the claim:- # nothing that used to be caught stopped being caught. If this ever diverges from the test- # above, the elision has started hiding something.+ # nothing that used to be caught stopped being caught. Divergence from the test above means the+ # elision has started hiding something.for name, before, after in mutants():old_before = {k: v@@ -573,7 +569,7 @@ def test_a_real_change_inside_the_streamed_message_is_refused_not_caught():def test_a_message_that_is_in_flight_on_one_arm_only_is_still_excused():- # The ordinary case, and it must not read as a build difference: one arm finished the reply+ # The ordinary case, which must not read as a build difference: one arm finished the reply# before its digest was taken and the other did not.a = streaming_arm(81)b = capture(@@ -589,15 +585,14 @@ def test_a_message_that_is_in_flight_on_one_arm_only_is_still_excused():def test_a_message_that_vanished_is_never_excused_by_being_in_flight():- # The elision withholds a subtree, never an element. A streamed message that is absent on one- # arm is a lost message, and no amount of "it was still arriving" makes that not a difference.+ # The elision withholds a subtree, never an element: a streamed message absent on one arm is a+ # lost message, and 'it was still arriving' does not make that not a difference.a = streaming_arm(24)b = capture(thread([message(0, role = "user", body = "the prompt"), message(1)]))got = P.compare(a, b)assert got["verdict"] == P.DIFFER# Caught one level earlier than `localise`, by the mount-count check: neither arm is windowing- # and they mounted different numbers of messages, which is a lost conversation and is reported- # as such rather than as a row that moved.+ # and they mounted different numbers of messages, which is a lost conversation.assert "different numbers of messages (3 vs 2)" in got["reason"], got@@ -814,7 +809,7 @@ def test_the_streamed_row_itself_is_still_withheld_when_the_probe_is_blind():def test_a_user_row_flagged_in_flight_by_the_other_arm_is_still_withheld():"""The arm that COULD place its stream is believed about which rows have no defined moment."""base = _blind_arm(24)- # The non-blind arm names index 0 as in flight. Whatever role it carries, its digest is a point+ # The non-blind arm names index 0 as in flight; whatever role it carries, its digest is a point# in a stream on that arm, so it is not a settled row.treat = dict(_blind_arm(24, prompt = "a different prompt"), in_flight = [0])treat["in_flight_unplaced"] = True@@ -824,8 +819,8 @@ def test_a_user_row_flagged_in_flight_by_the_other_arm_is_still_withheld():def test_an_old_payload_without_the_streaming_fields_is_scored_as_it_always_was():- # Not silently refused, and not silently excused: a capture recorded before the fields existed- # carries no claim about a stream, and falls back to the digest it does carry.+ # Not silently refused and not silently excused: a capture recorded before the fields existed+ # carries no claim about a stream and falls back to the digest it does carry.a = streaming_arm(24, streaming_fields = False)b = streaming_arm(24, streaming_fields = False)assert P.compare(a, b)["verdict"] == P.MATCHdiff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.pyindex cacc5a70cff..91f79483e25 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_parity_text_runs.py@@ -44,8 +44,8 @@ def reasoning_trigger(seconds: str) -> dict:def test_a_duration_split_across_text_nodes_is_normalised():- # The exact regression. Before the fix these two differed by one character and the digest- # moved; the two arms are the same build and the only difference is wall clock.+ # The exact regression. Before the fix these two differed by one character and the digest moved;+ # the two arms are the same build and the only difference is wall clock.assert sig(reasoning_trigger("3")) == sig(reasoning_trigger("2"))@@ -77,8 +77,8 @@ def test_a_split_relative_time_is_normalised():def test_a_bare_number_with_no_unit_still_moves_the_signature():- # A message count, a row count, a badge. No time unit follows it, so it is content and the- # digest has to see it change. This is the false-negative direction and it is the worse one.+ # A message count, a row count, a badge. No time unit follows it, so it is content and the digest+ # has to see it change. This is the false-negative direction and it is the worse one.def badge(n: str) -> dict:return {"tag": "span", "children": [{"tag": "b", "children": [n, " messages"]}]}@@ -97,8 +97,8 @@ def test_an_element_boundary_still_breaks_a_text_run():}joined = {"tag": "div", "children": [{"tag": "span", "children": ["3", " seconds"]}]}assert sig(split) != sig(joined)- # And the genuinely-split-across-elements case keeps its number, because nothing there proves- # the two spans are one rendered phrase.+ # And the genuinely-split-across-elements case keeps its number, because nothing there proves the+ # two spans are one rendered phrase.other = {"tag": "div","children": [diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.pyindex 3591b2e05cb..f6e59f0b266 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_reply_axis.py@@ -77,8 +77,8 @@ def test_the_tail_override_moves_the_reply_and_not_the_thread(corpus: Corpus):for tail in (24_000, 96_000):plan = plan_rung(corpus, "100K", stream_tail_chars = tail)assert abs(plan.streamed_chars - tail) < tail * 0.1, (tail, plan.streamed_chars)- # Within a few percent: the seeded prefix is trimmed to compensate, so the cell measures- # a different SPLIT of the same total rather than a bigger thread.+ # Within a few percent: the seeded prefix is trimmed to compensate, so the cell measures a different+ # SPLIT of the same total rather than a bigger thread.assert abs(plan.total_chars - base.total_chars) < base.total_chars * 0.05, (tail,plan.total_chars,@@ -133,10 +133,10 @@ def test_dollars_reach_the_streamed_turn_and_nothing_else(corpus: Corpus):plain = plan_rung(corpus, "100K", stream_tail_chars = 24_000)salted = plan_rung(corpus, "100K", stream_tail_chars = 24_000, dollars = True)assert _streamed_text(salted).count("$") > _streamed_text(plain).count("$")- # The seeded prefix is rendered once at mount and never re-preprocessed, so dollars there- # would change the corpus without changing what the per-frame path is asked to do. v2's own- # math is expected there; what must not appear is anything THIS added, so the prefix has to- # be byte-identical with the flag on and off.+ # The seeded prefix is rendered once at mount and never re-preprocessed, so dollars there would+ # change the corpus without changing what the per-frame path is asked to do. v2's own math is+ # expected; what must not appear is anything THIS added, so the prefix has to be byte-identical+ # with the flag on and off.assert [(u.reasoning, u.content) for u in salted.seeded_units] == [(u.reasoning, u.content) for u in plain.seeded_units]@@ -233,8 +233,7 @@ class _FakeKeyboard:def press(self, key: str) -> None:self.pressed.append(key)if key == "Enter" and "one more" in self._page.filled:- # Sending the throwaway turn starts a generation, which is what the own-turn path- # then waits for and stops.+ # Sending the throwaway turn starts a generation, which is what the own-turn path then waits for and stops.self._page.running = True@@ -314,8 +313,8 @@ def test_stop_refuses_to_truncate_the_cell_s_own_reply():from studiobench.scene.actions import stop_generationpage = _FakePage(running = True)- # 200 ms: too small to hold the throwaway turn at all, so the drain wait is zero and the- # refusal below is the one this test is about rather than the budget check beside it.+ # 200 ms: too small to hold the throwaway turn at all, so the drain wait is zero and the refusal+ # below is the one this test is about rather than the budget check beside it.result = stop_generation(_stop_ctx(page, budget_ms = 200))assert result.ran is False, "a stop that would truncate the measured reply must not run"diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.pyindex 73e612dfe0a..faa8248f916 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_rung_plan.py@@ -82,10 +82,9 @@ def test_the_stream_drains_before_the_first_after_generation_slot():def test_during_generation_slots_actually_fall_during_generation():from studiobench.scene.schedule import SCENES- # The SHORTEST stream on the ladder above 1K, since a slot has to be inside every rung's- # stream to deserve the name.- # The OPENING turn only: the follow-ups are sent later by `send_turn`, so a slot that has to- # fall during generation has to fall inside the first stream, not inside their sum.+ # The SHORTEST stream on the ladder above 1K, since a slot has to be inside every rung's stream to+ # deserve the name. The OPENING turn only: the follow-ups are sent later by `send_turn`, so a slot+ # that must fall during generation has to fall inside the first stream, not their sum.shortest = min(p.streamed_chars for r, p in _plans().items() if r != "1K") / FIELD_CHARS_PER_SECfor name, scene in SCENES.items():during = [s for s in scene.slots if s.action == "scroll_during_generation"]@@ -134,11 +133,10 @@ def test_the_ladder_is_strictly_increasing_in_seeded_mass():# Actions that need a SETTLED reply. The action bar's More and Copy buttons are not rendered while# a turn is streaming, select-all copies a moving target, and delete is hidden on a running-# message. Each one reports an honest `NOT RUN` rather than a wrong number, which is why this was-# invisible until somebody counted: on a 36 job sweep the fast film recorded-# `message_menu: NOT RUN -- no More button` on 312 of 312 attempts, and the four actions it-# silently stopped exercising include the one carrying the largest measured effect in this-# codebase (the message menu, 1,164.9 ms to 60.7 ms across the merged campaign).+# message. Each reports an honest `NOT RUN`, which is why this was invisible until somebody+# counted: on a 36 job sweep the fast film recorded `message_menu: NOT RUN -- no More button` on+# 312 of 312 attempts, and the four actions it silently stopped exercising include the one+# carrying the largest measured effect in this codebase (1,164.9 ms to 60.7 ms).SETTLED_ACTIONS = ("message_menu", "copy_markdown", "select_all_copy", "delete_message")@@ -165,31 +163,24 @@ def test_settled_actions_open_after_the_follow_up_drains():last_send = Nonefor slot in sorted(scene.slots, key = lambda s: s.t_start_ms):if slot.action == "send_turn":- # THE LATEST THE SEND CAN FIRE, not the earliest. The drain starts when the send- # actually happens, and a send slot is a WINDOW: the action may legitimately begin- # anywhere inside its own budget, which is precisely what it does when the slot- # before it overran and pushed it.- #- # Measured from the start instead, this check reported 1,538 ms of margin on the- # fast film where 38 ms existed, and CI then failed the way the arithmetic says it- # must: `reasoning_toggle` overran its 3,500 ms budget by 934 ms, `send_turn` was- # pushed 1,373 ms late but stayed inside its 1,500 ms budget so nothing recorded a- # miss, the send-to-menu gap collapsed from a nominal 5,300 ms to an actual- # 3,691 ms, and `message_menu` found a reply still streaming and recorded NOT RUN.- # Every slot was individually within budget and the film was still unrunnable.- #- # This is the same defect the rest of this branch is about, in the check rather- # than in the instrument: a quantity computed at a moment whose meaning is not the- # one the reader assumes. A margin measured from the earliest possible send is not- # a margin.+ # THE LATEST THE SEND CAN FIRE, not the earliest. The drain starts when the send actually happens,+ # and a send slot is a WINDOW: the action may legitimately begin anywhere inside its own budget,+ # which is what it does when the slot before it overran.+ # Measured from the start instead, this check reported 1,538 ms of margin on the fast film where+ # 38 ms existed, and CI then failed the way the arithmetic says it must: `reasoning_toggle`+ # overran its 3,500 ms budget by 934 ms, `send_turn` was pushed 1,373 ms late but stayed inside+ # its own budget so nothing recorded a miss, the send-to-menu gap collapsed from a nominal 5,300+ # ms to 3,691 ms, and `message_menu` found a reply still streaming and recorded NOT RUN. Every+ # slot was individually within budget and the film was still unrunnable.+ # Its own budget is 1,500 ms.+ # The same defect the rest of this branch is about, in the check rather than the instrument: a+ # quantity computed at a moment whose meaning is not the one the reader assumes.last_send = slot.t_start_ms + slot.budget_mselif slot.action in SETTLED_ACTIONS and last_send is not None:- # The slot's WINDOW, not its start. A slot may legitimately open a little before- # the follow-up finishes and wait inside its own budget -- the quick film opens- # `message_menu` at 4.5 s against a 4.56 s drain and it runs, because it has 3 s- # of budget to wait in. What is fatal is a window that CLOSES before the reply- # settles, which is what the first fast film did: 1.7 s gap plus 0.8 s budget- # against a 4.56 s drain, so the button had not been rendered yet and never- # would be inside that window.+ # The slot's WINDOW, not its start. A slot may open a little before the follow-up finishes and+ # wait inside its own budget: the quick film opens `message_menu` at 4.5 s against a 4.56 s drain+ # and it runs, because it has 3 s to wait in. What is fatal is a window that CLOSES before the+ # reply settles, which is what the first fast film did.+ # A 1.7 s gap plus a 0.8 s budget against a 4.6 s drain.window_end = (slot.t_start_ms + slot.budget_ms - last_send) / 1000.0assert window_end >= drain_s, (name, slot.action, window_end, drain_s)diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.pyindex d6849dc58a4..0790e0370a8 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_surfaces.py@@ -50,15 +50,15 @@ def test_every_surface_declares_a_reach_a_restore_and_a_root():f"{surface.id} declares no reach, so it must also declare no restore: a surface that "f"is reached by doing nothing cannot need undoing")- # Anything that navigates or opens an overlay has to say how to get back, or the next- # surface is reached from a state nobody declared.+ # Anything that navigates or opens an overlay has to say how to get back, or the next surface is+ # reached from a state nobody declared.opens = any(step[0] in {"goto", "click", "fill", "press"} for step in surface.reach)assert not opens or surface.restore, f"{surface.id} opens something and never closes it"def test_every_surface_declares_a_settle_condition():- # A sweep with no settle condition is a sweep that digests whatever is on screen when the- # sleep expires, which is the one result this tool may not produce.+ # A sweep with no settle condition digests whatever is on screen when the sleep expires, which+ # is the one result this tool may not produce.for surface in surfaces.surfaces():assert surface.settle is not None, surface.idassert isinstance(surface.settle, dict)@@ -88,9 +88,8 @@ def test_every_step_uses_a_known_verb_with_the_right_arity():def test_the_sweep_can_execute_every_verb_the_registry_uses():- # The registry and the interpreter are in different files, and a verb added to one and not the- # other fails only at run time, on the surface that uses it, as a reach failure that reads- # like a broken selector.+ # The registry and the interpreter are in different files, so a verb added to one and not the+ # other fails only at run time, on the surface that uses it, and reads like a broken selector.used = {step[0] for s in surfaces.surfaces() for steps in (s.reach, s.restore) for step in steps}@@ -125,9 +124,9 @@ def test_the_registry_covers_every_major_group(group):def test_the_settings_dialog_covers_every_shipped_tab():- # The twelve panels are lazily imported chunks, so a tab that stopped rendering fails only- # for the tab. Pinned here so a panel added to the app without a surface is visible as a- # missing id rather than as a coverage figure that quietly stayed at 100%.+ # The twelve panels are lazily imported chunks, so a tab that stopped rendering fails only for+ # the tab. Pinned so a panel added without a surface shows up as a missing id rather than a+ # coverage figure that quietly stayed at 100%.shipped = {"general","profile",@@ -147,9 +146,8 @@ def test_the_settings_dialog_covers_every_shipped_tab():def test_the_registry_covers_every_routed_path():- # From app/router.tsx. `/` and `/settings` are not page surfaces -- both redirect on- # beforeLoad -- and the two auth-flow routes are in KNOWN_UNCOVERED with the guard that- # redirects them.+ # From app/router.tsx. `/` and `/settings` are not page surfaces (both redirect on beforeLoad)+ # and the two auth-flow routes are in KNOWN_UNCOVERED with the guard that redirects them.routed = {"/chat","/projects",@@ -216,8 +214,8 @@ def test_the_surface_row_type_is_registered():def test_a_surface_row_must_carry_its_reason():- # The single defect this project cares most about: a check that cannot distinguish "did not- # run" from "passed". A surface row without `reason` is exactly that row.+ # The single defect this project cares most about: a check that cannot distinguish 'did not run'+ # from 'passed'. A surface row without `reason` is exactly that row.assert "reason" in ROW_REQUIRED["surface"]assert "reached" in ROW_REQUIRED["surface"]assert "parity" in ROW_REQUIRED["surface"]@@ -261,14 +259,14 @@ def test_surface_rows_land_in_their_own_payload_section(tmp_path):)payload = assemble_rows(path)assert len(payload["surfaces"]) == 1- # NOT in unknown_rows. A new row type that lands there is carried through the whole report+ # NOT in unknown_rows: a new row type that lands there is carried through the whole report# without anyone noticing it is not being read.assert payload["unknown_rows"] == []def test_a_payload_carrying_surface_rows_has_no_bare_zeros():# The schema bans a naked numeric zero, because a zero outside a measure is indistinguishable- # from "we never ran that". Surface rows carry counts, so they have to attest.+ # from 'we never ran that'. Surface rows carry counts, so they have to attest.rows = surface_sweep.sweep(_FakePage(reach_ok = True), "http://x")[0]validate_payload({"excluded_cells": [], "surfaces": rows})@@ -318,8 +316,8 @@ def test_a_conditional_miss_is_counted_apart_from_a_hard_miss():def test_a_volatile_surface_is_reached_but_not_counted_as_comparable():- # Reached and volatile are different facts and the manifest has to keep them apart. Folding- # them together is how a live memory gauge ends up quoted as a UI change.+ # Reached and volatile are different facts the manifest must keep apart; folding them together+ # is how a live memory gauge ends up quoted as a UI change.entries = [_one(id = "a"), _one(id = "b", volatile = "shows a live memory gauge")]rows = _rows_for(entries, {"a", "b"})manifest = surface_sweep.build_manifest(rows, entries, {"scoped": True})@@ -337,10 +335,9 @@ def test_every_volatile_surface_states_its_mechanism():def test_the_measured_volatile_surfaces_are_declared():- # Three consecutive sweeps against one Unsloth agreed on 44 of 53 surfaces. These are the ones- # that did not, each with a mechanism established from source or from a text diff. Pinned so- # a settle condition that is tightened later shows up as a flag that can now be dropped,- # rather than as a flag nobody revisits.+ # Three consecutive sweeps against one Unsloth agreed on 44 of 53 surfaces; these are the ones+ # that did not, each with a mechanism established from source or a text diff. Pinned so a+ # later-tightened settle condition shows up as a flag that can now be dropped.declared = {s.id for s in surfaces.surfaces() if s.volatile}assert {"route:chat",@@ -358,8 +355,8 @@ def test_the_measured_volatile_surfaces_are_declared():def test_the_unexplained_movers_say_so_rather_than_offering_a_cause():- # A plausible-sounding mechanism that has not been established is worse than an admitted gap:- # it is the sentence somebody uses to wave away a real difference.+ # A plausible-sounding mechanism that has not been established is worse than an admitted gap: it+ # is the sentence somebody uses to wave away a real difference.unexplained = [sfor s in surfaces.surfaces()@@ -397,9 +394,8 @@ def test_the_rendered_manifest_names_every_failure_and_every_known_gap():def test_an_unscoped_sweep_says_so_in_the_rendered_manifest():- # An unscoped digest is a whole-page reading taken forty times, and every one of them agrees- # with every other for reasons that have nothing to do with the surfaces. That has to be at- # the top of the artefact, not inferable from a field somebody might read.+ # An unscoped digest is a whole-page reading taken forty times, every one agreeing for reasons+ # that have nothing to do with the surfaces. That has to be at the top of the artefact.entries = surfaces.surfaces()[:1]manifest = surface_sweep.build_manifest(_rows_for(entries, {entries[0].id}),@@ -511,8 +507,8 @@ def test_a_reach_that_fails_records_a_reason_and_never_a_digest():for row in interactive:assert row["reached"] is Falseassert "the reach failed" in row["reason"]- # Never a digest it did not take. A parity block that claimed `attempted` here would make- # a failed sweep read as a passing parity check downstream.+ # Never a digest it did not take: a parity block claiming `attempted` here would make a failed+ # sweep read as a passing parity check downstream.assert row["parity"]["parity_attempted"] is False@@ -559,8 +555,8 @@ def test_the_sweep_streams_rows_to_a_recorder_as_it_goes():def test_the_sweep_returns_to_the_known_state_before_every_surface():- # Not "after every surface": an after-only reset trusts each surface's own restore, and the- # one that fails is the one whose restore did not run.+ # Not 'after every surface': an after-only reset trusts each surface's own restore, and the one+ # that fails is the one whose restore did not run.page = _FakePage()surface_sweep.sweep(page, "http://x")assert surfaces.KNOWN_STATE_PATH == "/chat"@@ -570,9 +566,9 @@ def test_the_sweep_returns_to_the_known_state_before_every_surface():def test_the_sweep_takes_its_digest_through_parity_capture():- # If surfaces.js ever grew its own DOM walk, surface digests and action digests would stop- # being comparable -- and nothing downstream would notice, because both would still be hex- # strings of the same length.+ # If surfaces.js grew its own DOM walk, surface digests and action digests would stop being+ # comparable and nothing downstream would notice, because both would still be hex strings of+ # the same length.text = (Path(__file__).resolve().parents[2] / "scene" / "surfaces.js").read_text(encoding = "utf-8")@@ -581,8 +577,8 @@ def test_the_sweep_takes_its_digest_through_parity_capture():def test_surfaces_js_does_not_edit_the_shared_chat_adapter():- # dom.js is the film's adapter and every action reads it. The surface layer restores the root- # it moved, so a sweep cannot leave the film digesting the wrong element.+ # dom.js is the film's adapter and every action reads it. The surface layer restores the root it+ # moved, so a sweep cannot leave the film digesting the wrong element.text = (Path(__file__).resolve().parents[2] / "scene" / "surfaces.js").read_text(encoding = "utf-8")diff --git a/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py b/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.pyindex 135f0ef3312..657a5d26102 100644--- a/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py+++ b/tests/studio/studiobench/fixture/selftest/test_studiobench_tail_deliverable.py@@ -40,10 +40,10 @@ def corpus() -> Corpus:return Corpus.load()-#: Every pair the axis is plausibly asked for, and what the corpus can actually do with it.-#: `True` means deliverable, `False` means the request has to be refused. Derived by running the-#: reply-axis test's own two assertions across the ladder, not chosen: every `False` here is a-#: pair that silently under-delivered before the guard, and every `True` is one that did not.+#: Every pair the axis is plausibly asked for, and what the corpus can actually do with it. `True`+#: means deliverable, `False` means the request has to be refused. Derived by running the+#: reply-axis test's own two assertions across the ladder, not chosen: every `False` is a pair that+#: silently under-delivered before the guard.MATRIX: list[tuple[str, int, bool]] = [("1K", 24_000, False),("1K", 96_000, False),diff --git a/tests/studio/studiobench/instruments/coverage.py b/tests/studio/studiobench/instruments/coverage.pyindex 26754e6b782..1eea1d12366 100644--- a/tests/studio/studiobench/instruments/coverage.py+++ b/tests/studio/studiobench/instruments/coverage.py@@ -121,9 +121,8 @@ def _parse(result: dict[str, Any]) -> CoverageSnapshot:ranges = fn.get("ranges") or []if not ranges:continue- # With `detailed: false` there is one range per function and it- # spans the function; even with block coverage the FIRST range is- # the function-level one, so this stays correct either way.+ # With `detailed: false` there is one range per function spanning the function; even with block+ # coverage the FIRST range is the function-level one, so this stays correct either way.head = ranges[0]snap.functions.append(FunctionCount(@@ -288,19 +287,13 @@ def ambiguity(snapshot: CoverageSnapshot, name: str) -> int:return len(snapshot.find(name))-# ═══════════════════════════════════════════════════════════════════════════# Harness adapter (INTERFACES.md section 3)-# ═══════════════════════════════════════════════════════════════════════════-#-# Level 3, and every window this instrument touches is TIMING-VOID by-# construction. Precise coverage keeps count-collecting bytecode alive, which-# disables TurboFan and Maglev for the whole isolate, so the durations that-# `tracing` reports in the same cell describe a program nobody ships.-#-# The payload therefore carries `timings_void: true` at both window and cell-# level. That flag exists so the report layer can refuse to quote a duration-# from this cell without having to know why. Only integers cross the boundary,-# enforced by `assert_integers_only` on the way out rather than by convention.+# Level 3, and every window this instrument touches is TIMING-VOID by construction: precise+# coverage keeps count-collecting bytecode alive, which disables TurboFan and Maglev for the whole+# isolate, so the durations `tracing` reports in the same cell describe a program nobody ships.+# The payload therefore carries `timings_void: true` at both window and cell level, so the report+# layer can refuse to quote a duration from this cell without knowing why. Only integers cross the+# boundary, enforced by `assert_integers_only`.import time # noqa: E402from typing import Any # noqa: E402@@ -338,10 +331,9 @@ class CoverageInstrument:returnt0 = time.perf_counter()try:- # Started ONCE per cell, never per window. Restarting coverage- # re-runs V8's DeoptimizeAll, which changes what gets compiled and- # therefore what gets counted, so per-window restarts would make the- # counts depend on the window boundaries.+ # Started ONCE per cell, never per window: restarting coverage re-runs V8's DeoptimizeAll, which+ # changes what gets compiled and therefore counted, so per-window restarts would make the counts+ # depend on the window boundaries.self.cov = PreciseCoverage(self.cdp)self.cov.start()except Exception as exc: # noqa: BLE001@@ -394,8 +386,8 @@ class CoverageInstrument:),},)- # The boundary guard. A float here would be a category error, not a- # rounding problem, so it raises rather than warns.+ # The boundary guard. A float here would be a category error, not a rounding problem, so it raises+ # rather than warns.assert_integers_only({k: vdiff --git a/tests/studio/studiobench/instruments/cpuprofile.py b/tests/studio/studiobench/instruments/cpuprofile.pyindex a1553291c51..f883737f126 100644--- a/tests/studio/studiobench/instruments/cpuprofile.py+++ b/tests/studio/studiobench/instruments/cpuprofile.py@@ -34,12 +34,10 @@ from typing import Anyfrom ..analysis import CellFailurefrom ..analysis.cpuprofile import CallFrame, CpuProfile, Sample-# V8's own default is 1000 us. 100 us resolves a sub-millisecond frame without-# the sampler itself becoming the workload.+# V8's own default is 1000 us. 100 us resolves a sub-millisecond frame without the sampler becoming the workload.DEFAULT_SAMPLING_INTERVAL_US = 100-# Minimum non-synthetic samples on BOTH sides before the two extraction routes-# may be declared to agree or disagree.+# Minimum non-synthetic samples on BOTH sides before the two extraction routes may be declared to agree or disagree.MIN_JS_SAMPLES_FOR_COMPARISON = 200@@ -160,13 +158,10 @@ def compare_with_trace_profile(1 for s in p.samples if (f := p.nodes.get(s.node_id)) is not None and not f.is_synthetic)- # A comparison built on a handful of JS samples cannot distinguish the two- # parsers from sampling noise. Saying "agrees" there would be worse than- # useless and saying "disagrees" is a false alarm, so it declines to rule.- # V8 has ONE CpuProfiler. If the trace carried the profiler categories, the- # "standalone" profile IS the in-trace profile and comparing them is- # comparing a thing with itself, which always agrees and therefore proves- # nothing. Detected structurally rather than trusted to the caller.+ # A comparison built on a handful of JS samples cannot distinguish the two parsers from sampling+ # noise, so it declines to rule. And V8 has ONE CpuProfiler: if the trace carried the profiler+ # categories, the 'standalone' profile IS the in-trace profile and comparing them always agrees.+ # Detected structurally rather than trusted to the caller.if (len(standalone.samples) == len(in_trace.samples)and len(standalone.nodes) == len(in_trace.nodes)@@ -221,30 +216,17 @@ def compare_with_trace_profile(}-# ═══════════════════════════════════════════════════════════════════════════# Harness adapter (INTERFACES.md section 3)-# ═══════════════════════════════════════════════════════════════════════════-#-# MEASURED, NOT ASSUMED, and it changed this design: V8 HAS ONE CPU PROFILER.-# Starting `Profiler.start` while a trace with `disabled-by-default-v8.cpu_profiler`-# is already running returns THE SAME SAMPLES. On a real capture both routes-# reported identical counts, 2169 samples over 17 nodes, because the inspector-# profiler and the tracing profiler share one `CpuProfiler` on the isolate.-#-# Two consequences, both load-bearing:-#-# 1. `compare_with_trace_profile` must refuse to compare a profile with-# itself. A cross-check that can never fail is worse than no cross-check,-# because it reads as corroboration.-# 2. This instrument is only meaningful at instrument level 1, where the trace-# carries NO profiler categories. There it is a genuine capability: stacks-# without the ProfileChunk volume that dominates an L2 trace buffer. A real-# L1 capture confirmed the two clocks agree, with 1748 of 1748 standalone-# samples landing inside the trace's own `RunTask` span, so its samples can-# still be joined to classified task windows.-#-# At level 2 and above it stands down and says so, rather than quietly-# duplicating what `tracing` already has.+# MEASURED, NOT ASSUMED, and it changed this design: V8 HAS ONE CPU PROFILER. Starting+# `Profiler.start` while a trace with `disabled-by-default-v8.cpu_profiler` is running returns THE+# SAME SAMPLES: on a real capture both routes reported 2169 samples over 17 nodes, because the+# inspector and tracing profilers share one `CpuProfiler` on the isolate.+# Two consequences: `compare_with_trace_profile` must refuse to compare a profile with itself,+# since a cross-check that can never fail reads as corroboration; and this instrument is only+# meaningful at level 1, where the trace carries NO profiler categories and it gives stacks+# without the ProfileChunk volume that dominates an L2 buffer (a real L1 capture had 1748 of 1748+# standalone samples inside the trace's own `RunTask` span). At level 2 and above it stands down+# and says so.import time # noqa: E402diff --git a/tests/studio/studiobench/instruments/frames.js b/tests/studio/studiobench/instruments/frames.jsindex 013586c2c00..0f3a7c3e88b 100644--- a/tests/studio/studiobench/instruments/frames.js+++ b/tests/studio/studiobench/instruments/frames.js@@ -1,35 +1,26 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//// The frame recorder. Installed as an init script before any app code runs.-//-// SALVAGED from playwright_reasoning_pane.py's RECORDER_INIT, with the two things that version-// got right kept exactly and the one thing it got wrong fixed.-//+// SALVAGED from playwright_reasoning_pane.py's RECORDER_INIT, keeping the two things that version+// got right and fixing the one it got wrong.// KEPT: ONE self-rescheduling rAF loop as the frame counter, and requestAnimationFrame is NOT// wrapped. A wrapper that increments per callback counts the page's frame once for the loop and-// once more for every rAF the app happened to schedule in that frame, so the reported frame rate-// RISES with how busy the app is. The first version of that file reported 888 fps on a 60Hz page-// for exactly this reason, which is not a small error, it is the metric inverted.-//-// KEPT: blocked time from a 1ms setTimeout, not a MessageChannel ping-pong. The MessageChannel-// version ticks about 150,000 times a second and halves Firefox's frame rate before any app code-// runs. This ticks about 150 times a second and costs nothing on any engine. Blocked time is the-// column that MOVES WHEN FPS DOES NOT: a page with 65% idle still paints every frame on time, so-// fps stays pinned at 60 while the work per chunk quietly triples.-//-// FIXED: the clamp is calibrated during an ENFORCED IDLE WINDOW that the driver opens, not from-// the first 60 ticks of whatever the page was doing. `setTimeout(fn, 1)` has a floor of about 4ms-// by spec, the real floor differs by engine and build, and blocked time is a SUBTRACTION against-// it -- so a wrong clamp invents block on one engine and hides it on another. Calibrating from-// the first 60 ticks means calibrating while the app is booting, or worse, on a rung where 31,637-// elements are already standing and the "idle" floor is really the app's own steady-state load.-// That produces a clamp far above 4ms, which then subtracts real block out of every window and-// reports a page pinned at 100% busy as 0.2% busy.-//-// And when the calibrated clamp comes out ABOVE 10ms, the answer is not a number. A 10ms floor-// under a 1ms timer means the machine could not answer an idle timer promptly, so nothing was-// idle and there is no floor to subtract. busy_pct is then null with a reason, never 0.2%.+// once more for every rAF the app scheduled, so the reported frame rate RISES with how busy the+// app is: the first version reported 888 fps on a 60Hz page, which is the metric inverted.+// KEPT: blocked time from a 1ms setTimeout, not a MessageChannel ping-pong, which ticks about+// 150,000 times a second and halves Firefox's frame rate before any app code runs. This ticks+// about 150 times a second. Blocked time is the column that MOVES WHEN FPS DOES NOT: a page with+// 65% idle still paints every frame on time, so fps stays pinned at 60 while the work per chunk+// triples.+// FIXED: the clamp is calibrated during an ENFORCED IDLE WINDOW the driver opens, not from the+// first 60 ticks of whatever the page was doing. `setTimeout(fn, 1)` has a ~4ms spec floor that+// differs by engine and build, and blocked time is a SUBTRACTION against it, so a wrong clamp+// invents block on one engine and hides it on another. Calibrating from the first 60 ticks means+// calibrating while the app is booting, or on a rung where the 'idle' floor is really the app's+// steady-state load, which reports a page pinned at 100% busy as 0.2% busy.+// And when the calibrated clamp comes out ABOVE 10ms the answer is not a number: the machine+// could not answer an idle timer promptly, so nothing was idle and there is no floor to subtract.+// busy_pct is then null with a reason.(() => {if (window.__sb && window.__sb.frames) return;@@ -39,8 +30,8 @@const MAX_CLAMP_MS = 10.0;const CALIBRATION_TICKS = 60;- // A window long enough to exceed this is minutes of 60 Hz, which no slot in the scene is. The- // cap exists so a pathological window cannot balloon the payload, not as a routine path.+ // A window long enough to exceed this is minutes of 60 Hz, which no slot in the scene is. The cap+ // stops a pathological window ballooning the payload; it is not a routine path.const GAPS_CAP = 50000;const R = {@@ -50,9 +41,9 @@lagTicks: 0,lagSumMs: 0,blockedMs: 0,- // The same blocked time again, NEVER reset. `blockedMs` is drained by the window reader, and- // a settle watch has to read blocked time on its own cadence while that reader is running.- // Two readers draining one accumulator would each see a fraction of the block.+ // The same blocked time again, NEVER reset. `blockedMs` is drained by the window reader, and a+ // settle watch has to read blocked time on its own cadence while that reader runs; two readers+ // draining one accumulator would each see a fraction of the block.blockedTotalMs: 0,clampMs: null,clampReason: "not calibrated",@@ -64,8 +55,8 @@),longTasks: 0,longTaskMs: 0,- // Every rAF the APP schedules, counted separately from the loop's own. Not a frame rate; it- // is how the tri-clock check tells "the page is idle" from "the loop is starved".+ // Every rAF the APP schedules, counted separately from the loop's own. Not a frame rate: it is+ // how the tri-clock check tells 'the page is idle' from 'the loop is starved'.appRafs: 0,};@@ -79,8 +70,8 @@};nativeRaf(frame);- // Count the app's own rAF traffic without pumping it. A pass-through wrapper is safe here- // BECAUSE it is not the frame counter: the loop above is.+ // Count the app's own rAF traffic without pumping it. A pass-through wrapper is safe here BECAUSE+ // it is not the frame counter.window.requestAnimationFrame = function (cb) {R.appRafs += 1;return nativeRaf(cb);@@ -122,8 +113,8 @@sorted.length === 0 ? null : sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * q))];window.__sb.frames = {- // Opened by the driver during an ENFORCED IDLE WINDOW: nothing streaming, no action running,- // the page at rest. Returns what it measured so the driver can record it rather than trust it.+ // Opened by the driver during an ENFORCED IDLE WINDOW: nothing streaming, no action running, the+ // page at rest. Returns what it measured so the driver can record it rather than trust it.beginCalibration() {R.calibrating = true;R.calibration = [];@@ -141,9 +132,9 @@}const median = samples[Math.floor(samples.length / 2)];if (median > MAX_CLAMP_MS) {- // NOT a clamp. A 1ms timer that takes longer than 10ms to come back on an idle page means- // the page was not idle, so there is no floor here to subtract and every blocked-time- // figure derived from it would be a subtraction against the app's own steady load.+ // NOT a clamp. A 1ms timer taking longer than 10ms on an idle page means the page was not idle,+ // so there is no floor to subtract and every blocked-time figure would be a subtraction against+ // the app's own steady load.R.clampMs = null;R.clampReason ="the calibrated timer clamp came out at " +@@ -177,9 +168,8 @@return performance.now();},- // Drain the window. `elapsedMs` is the DRIVER's measure of the window, passed in rather than- // computed here, so fps is per real elapsed time even when the page could not run its own- // clock reads promptly.+ // Drain the window. `elapsedMs` is the DRIVER's measure, passed in rather than computed here, so+ // fps is per real elapsed time even when the page could not run its own clock reads promptly.read(elapsedMs) {const gaps = R.frameGaps.slice().sort((a, b) => a - b);let over33 = 0;@@ -191,21 +181,18 @@app_rafs: R.appRafs,fps: elapsed === null ? null : Math.round((R.frames / (elapsed / 1000)) * 10) / 10,frames_over_33: over33,- // As a SHARE of the frames observed, because the denominator is not fixed: headless- // Chromium has no vsync and runs the loop as fast as it can, so a raw count is not- // comparable across engines or loads.+ // As a SHARE of the frames observed, because the denominator is not fixed: headless Chromium has+ // no vsync and runs the loop as fast as it can, so a raw count is not comparable across engines+ // or loads.frames_over_33_pct:gaps.length === 0 ? null : Math.round((over33 / gaps.length) * 1000) / 10,p50_frame_ms: quantile(gaps, 0.5),p95_frame_ms: quantile(gaps, 0.95),- // The RAW deltas, not only the summary. time_in_jank_pct and jank_index are defined over- // the whole distribution (a share of wall time, and a sum of squared over-budget time),- // and neither can be recovered from percentiles. Without this the scoring layer has to- // either skip two of its six metrics or invent them from p95, which is the kind of- // plausible-but-unfounded number this tool exists to refuse.- // `gaps` is sorted ASCENDING, so a head slice would drop exactly the janky frames and- // score a stuttering window as clean. Over the cap this emits null and says why: the- // scoring layer then reads "failed", not a number built from the fastest frames.+ // The RAW deltas, not only the summary: time_in_jank_pct and jank_index are defined over the+ // whole distribution and neither can be recovered from percentiles, so without this the scoring+ // layer would skip two of its six metrics or invent them from p95. `gaps` is sorted ASCENDING,+ // so a head slice would drop exactly the janky frames; over the cap this emits null and says+ // why, and the scoring layer reads 'failed' rather than a number built from the fastest frames.frame_gaps_ms:gaps.length > GAPS_CAP ? null : gaps.map((g) => Math.round(g * 10) / 10),frame_gaps_truncated: gaps.length > GAPS_CAP,@@ -218,8 +205,8 @@clamp_reason: R.clampReason,long_tasks: R.longTaskSupported ? R.longTasks : null,long_task_ms: R.longTaskSupported ? Math.round(R.longTaskMs) : null,- // The point of the flag: without it an engine with no Long Tasks API reports zero jank in- // exactly the same shape as an engine that had none.+ // The point of the flag: without it an engine with no Long Tasks API reports zero jank in the+ // same shape as an engine that had none.long_task_supported: R.longTaskSupported,};if (R.clampMs === null) {@@ -248,8 +235,8 @@},};- // Two rAFs: the second is the frame that has PAINTED the first's work. Every action's timing- // that is clocked across a paint uses this, so the paint floor is one shared constant.+ // Two rAFs: the second is the frame that has PAINTED the first's work. Every action timing+ // clocked across a paint uses this, so the paint floor is one shared constant.window.__sbNextPaint = () =>new Promise((resolve) => nativeRaf(() => nativeRaf(() => resolve(performance.now()))));})();diff --git a/tests/studio/studiobench/instruments/glass.js b/tests/studio/studiobench/instruments/glass.jsindex 7931139e23b..4698fbc1705 100644--- a/tests/studio/studiobench/instruments/glass.js+++ b/tests/studio/studiobench/instruments/glass.js@@ -1,21 +1,15 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//-// The glass: instrumented accessors that turn "the autoscroll observer forces a synchronous-// layout on every streamed character" from a reading of the source into a counted, timed number.-//-// This is the direct instrument for M3. `use-intent-aware-autoscroll.tsx` installs a-// MutationObserver with `subtree: true, characterData: true` over the whole viewport; its callback-// synchronously reads `scrollHeight` (stabilize(), :435), writes the inherited custom property-// `--aui-scroll-stabilizer` on the scroll container (:466), which invalidates style for every-// descendant, and calls `scrollTo` (:487). All three are invisible to a React Profiler, to-// markdown timing and to a DOM node census. They are visible HERE.-//-// PERTURBING BY CONSTRUCTION, and so off at level 0. Wrapping a hot getter on Element.prototype-// costs something on every call site in the app, not only the ones under suspicion, and the-// wrapper's own `performance.now()` pair is a large fraction of a cheap read. Level 1 and above,-// and the headline numbers come from level 0, where none of this is installed.-//+// The glass: instrumented accessors that turn "the autoscroll observer forces a synchronous layout+// on every streamed character" from a reading of the source into a counted, timed number.+// The direct instrument for M3. `use-intent-aware-autoscroll.tsx` installs a MutationObserver with+// `subtree: true, characterData: true` over the whole viewport; its callback synchronously reads+// `scrollHeight` (:435), writes the inherited custom property `--aui-scroll-stabilizer` on the+// scroll container (:466), which invalidates style for every descendant, and calls `scrollTo`+// (:487). All three are invisible to a React Profiler, to markdown timing and to a node census.+// PERTURBING BY CONSTRUCTION, and so off at level 0: wrapping a hot getter on Element.prototype+// costs something on every call site in the app, and the wrapper's own `performance.now()` pair is+// a large fraction of a cheap read. Level 1 and above; the headline numbers come from level 0.// Split by WHOSE read it is. "Layout was forced 4,000 times" is not attribution; "3,980 of them// were on the scroll viewport, from inside a MutationObserver callback" is.@@ -28,8 +22,8 @@scroll_height_ms: 0,viewport_reads: 0,viewport_ms: 0,- // Reads taken while a MutationObserver callback is on the stack. This is the number that- // separates "the app reads scrollHeight" from "the app reads scrollHeight per mutation".+ // Reads taken while a MutationObserver callback is on the stack: the number that separates "the app+ // reads scrollHeight" from "the app reads scrollHeight per mutation".reads_in_observer: 0,reads_in_observer_ms: 0,scroll_top_writes: 0,diff --git a/tests/studio/studiobench/instruments/heap.py b/tests/studio/studiobench/instruments/heap.pyindex 28c82dd6506..1c479f7825e 100644--- a/tests/studio/studiobench/instruments/heap.py+++ b/tests/studio/studiobench/instruments/heap.py@@ -28,12 +28,12 @@ from typing import Any, Iterablefrom ..analysis import CellFailure-# 4096 bytes between samples. Small enough to resolve a per-sibling allocation-# at a few hundred siblings, large enough not to perturb the allocation path.+# 4096 bytes between samples. Small enough to resolve a per-sibling allocation at a few hundred+# siblings, large enough not to perturb the allocation path.DEFAULT_SAMPLING_INTERVAL = 4096-# `includeObjectsCollectedByMajorGC` / `MinorGC` landed in V8 10.8, which-# shipped in Chrome 108. Below that the parameter is accepted and ignored.+# `includeObjectsCollectedByMajorGC` / `MinorGC` landed in V8 10.8, which shipped in Chrome 108.+# Below that the parameter is accepted and ignored.MIN_CHROME_FOR_GC_FLAGS = 108@@ -253,18 +253,13 @@ def survival_ratio(}-# ═══════════════════════════════════════════════════════════════════════════# Harness adapter (INTERFACES.md section 3)-# ═══════════════════════════════════════════════════════════════════════════-#-# Level 3. One sampling session per window, started with-# `includeObjectsCollectedByMajorGC` so that transient garbage is visible; the-# hypothesis is per-render churn, and a survivors-only profile of that is empty.-#-# The survivors arm is NOT taken here. It needs a second run of the identical-# workload with the flag off, which is an ablation arm and belongs to Layer 3.-# What this instrument emits is the allocation side plus the site breakdown, so-# `analysis.heap.survival_ratio` can be applied across two arms afterwards.+# Level 3. One sampling session per window, started with `includeObjectsCollectedByMajorGC` so+# transient garbage is visible: the hypothesis is per-render churn, and a survivors-only profile+# of that is empty.+# The survivors arm is NOT taken here. It needs a second run of the identical workload with the+# flag off, which is an ablation arm belonging to Layer 3. This emits the allocation side plus the+# site breakdown, so `analysis.heap.survival_ratio` can be applied across two arms afterwards.import time # noqa: E402@@ -311,9 +306,8 @@ class HeapInstrument:try:self.profiler.start()except CellFailure as exc:- # The version gate. Refusing is correct: an older browser ignores- # the flag silently and hands back survivors only, which for a- # transient-allocation hypothesis reads as "no allocation here".+ # The version gate. Refusing is correct: an older browser ignores the flag silently and hands back+ # survivors only, which for a transient-allocation hypothesis reads as "no allocation here".self._reason = f"{exc.gate}: {exc.detail}"self.profiler = Noneexcept Exception as exc: # noqa: BLE001@@ -360,9 +354,8 @@ class HeapInstrument:return payloaddef end_cell(self, cell: Any) -> dict | None:- # A prose key is OMITTED when there is nothing to say, never set to- # None. `None` is reserved for a QUANTITY that could not be measured,- # and overloading it for "no comment" would make the two indistinguishable.+ # A prose key is OMITTED when there is nothing to say, never set to None. `None` is reserved for a+ # QUANTITY that could not be measured.out = merge(measured("overhead_ms", round(self._overhead_ms, 3)),measured("windows_sampled", self._windows),diff --git a/tests/studio/studiobench/instruments/input.js b/tests/studio/studiobench/instruments/input.jsindex 02096b60cc4..5b8352dfce4 100644--- a/tests/studio/studiobench/instruments/input.js+++ b/tests/studio/studiobench/instruments/input.js@@ -1,37 +1,27 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//// Keystroke-to-paint, measured from the page side of a REAL key event.-//// WHY THIS IS NOT THE SALVAGED KEYSTROKE_JS. The old harness typed by calling the native value-// setter on the textarea and dispatching a synthetic `input` event. That reaches React's-// controlled input, so it measured something -- but it enters the pipeline AFTER hit testing,-// after the browser's own event routing, and it carries no `latencyInfo`, which is the trace-// field that says when the OS delivered the keypress. A synthetic event therefore cannot show-// input queueing delay at all: it is dispatched from a task that is by definition already running,-// so the queue is empty by construction and the number reads clean exactly when a user would be-// waiting longest.-//+// setter and dispatching a synthetic `input` event. That reaches React's controlled input, but+// enters the pipeline AFTER hit testing and event routing and carries no `latencyInfo`, so it+// cannot show input queueing delay at all: it is dispatched from a task already running, so the+// queue is empty by construction and the number reads clean exactly when a user would be waiting+// longest.// So the driver types with `page.keyboard`, which goes in through CDP as a real input event, and-// this file's job is only to observe from the page: mark when the key arrived, and mark the first-// paint after the character landed in the value. The subtraction is done here because a-// driver-side clock would include the CDP round trip, which is not something a user experiences.-//-// THE CLOCK STARTS AT THE KEYDOWN, NOT AT THE INPUT HANDLER, and that is the whole point of using-// a real key event. `input` is dispatched as the default action of `keydown`, so any handler that-// blocks the main thread on the way in has ALREADY finished by the time `input` fires: a start-// taken here reads `performance.now()` after the queue drained and subtracts the wait out of the-// number. Measured directly against the harness's own 400 ms injected keydown stall, an-// input-anchored clock moved keystroke p95 by -14.8 ms while the user waited 400 ms, and the-// integrity gate in `instruments/selfcheck.py` -- which requires that stall to move p95 by at-// least 350 ms -- can never pass on it.-//+// this file only observes from the page: mark when the key arrived, and mark the first paint+// after the character landed in the value. The subtraction is done here because a driver-side+// clock would include the CDP round trip.+// THE CLOCK STARTS AT THE KEYDOWN, NOT AT THE INPUT HANDLER. `input` is dispatched as the default+// action of `keydown`, so a handler that blocks the main thread on the way in has ALREADY+// finished when `input` fires and a start taken there subtracts the wait out of the number.+// Against the harness's own 400 ms injected keydown stall, an input-anchored clock moved keystroke+// p95 by -14.8 ms, and the integrity gate in `instruments/selfcheck.py`, which requires 350 ms of+// movement, can never pass on it.// A trusted event's `timeStamp` is a `DOMHighResTimeStamp` on the same origin as-// `performance.now()`, set when the occurrence the event signals happened rather than when it was-// dispatched, so `keydown.timeStamp` is the hardware arrival time and carries the queueing delay.-// Verified on all three engines: with the 400 ms stall armed, `performance.now()` inside the-// `input` handler is ~400 ms past `keydown.timeStamp` and ~0 ms past the INPUT event's own-// timeStamp, which is why the anchor is the keydown and not this event.+// `performance.now()`, set when the occurrence happened rather than when it was dispatched, so+// `keydown.timeStamp` is the hardware arrival time and carries the queueing delay. Verified on all+// three engines: with the stall armed, `performance.now()` inside the `input` handler is ~400 ms+// past `keydown.timeStamp` and ~0 ms past the input event's own timeStamp.(() => {if (window.__sb && window.__sb.input) return;@@ -46,9 +36,9 @@dropped: 0,keyAt: null,unanchored: 0,- // Every `input` event this instrument saw, whether it became a sample or was coalesced behind- // a paint that had not finished. Without it there is no denominator: `samples` alone cannot- // distinguish "the page painted every keystroke" from "most of them never reached here".+ // Every `input` event this instrument saw, whether it became a sample or was coalesced behind an+ // unfinished paint. Without it there is no denominator: `samples` alone cannot distinguish 'the+ // page painted every keystroke' from 'most never reached here'.seen: 0,};@@ -57,18 +47,18 @@? window.__sbNextPaint(): new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(() => r(performance.now()))));- // The key that produced the character being measured. The LAST unconsumed keydown, because a- // key that produced no character (an arrow, a modifier) must not anchor the next one that did.+ // The key that produced the character being measured: the LAST unconsumed keydown, because a key+ // that produced no character must not anchor the next one that did.const onKeyDown = (ev) => {if (!S.armed || (S.target && ev.target !== S.target)) return;S.keyAt = ev.timeStamp;};- // Consume the anchor, and refuse an implausible one rather than quoting it. A synthetic event- // constructed by a page script carries the time of its CONSTRUCTOR, an engine that does not put- // key events on the performance timeline could report 0, and a composition commit produces an- // `input` with no keydown at all. Each of those falls back to this handler's own clock, which is- // the old behaviour, and is counted so the report can say how many samples were not anchored.+ // Consume the anchor, and refuse an implausible one rather than quoting it. A page-constructed+ // synthetic event carries its CONSTRUCTOR's time, an engine that does not put key events on the+ // performance timeline could report 0, and a composition commit produces an `input` with no+ // keydown. Each falls back to this handler's own clock, the old behaviour, and is counted so the+ // report can say how many samples were unanchored.const anchor = (now) => {const at = S.keyAt;S.keyAt = null;@@ -82,8 +72,8 @@const onInput = (ev) => {if (!S.armed || ev.target !== S.target) return;S.seen += 1;- // One in flight at a time. A burst of characters typed faster than the page can paint would- // otherwise attribute one paint to several keystrokes and report each of them as fast.+ // One in flight at a time: a burst typed faster than the page can paint would otherwise attribute+ // one paint to several keystrokes and report each as fast.if (S.pending !== null) {S.dropped += 1;S.keyAt = null;@@ -99,8 +89,8 @@at_ms: Math.round(at * 10) / 10,// Keystroke to paint: from the key arriving to the frame that shows it.latency_ms: Math.round((paintedAt - started) * 10) / 10,- // The two halves, kept separately so a regression can be attributed rather than guessed:- // how long the key waited to be handled, and how long the page then took to paint it.+ // The two halves, kept separately so a regression can be attributed: how long the key waited to be+ // handled, and how long the page then took to paint it.input_delay_ms: keyAt === null ? null : Math.round((at - keyAt) * 10) / 10,paint_ms: Math.round((paintedAt - at) * 10) / 10,anchored_on: keyAt === null ? "input" : "keydown",@@ -111,8 +101,8 @@};window.__sb.input = {- // `selector` is resolved here rather than passed as a handle, so the driver can re-arm across- // a page that re-rendered its composer without holding a stale node.+ // `selector` is resolved here rather than passed as a handle, so the driver can re-arm across a+ // page that re-rendered its composer without holding a stale node.arm(selector) {const el = document.querySelector(selector);if (!el) return { armed: false, reason: "no element matched " + selector };@@ -127,28 +117,24 @@S.seen = 0;S.armed = true;el.addEventListener("input", onInput, true);- // On the WINDOW, in capture, so the anchor is taken however the app routes the key and even- // if a handler on the way in stops propagation. Idempotent: re-arming across a re-rendered- // composer must not leave two of these behind, each overwriting the other's anchor.+ // On the WINDOW, in capture, so the anchor is taken however the app routes the key and even if a+ // handler stops propagation. Idempotent: re-arming across a re-rendered composer must not leave+ // two behind, each overwriting the other's anchor.window.removeEventListener("keydown", onKeyDown, true);window.addEventListener("keydown", onKeyDown, true);return { armed: true, baseline_length: S.baseline.length };},- // IS ANYTHING STILL IN FLIGHT? The driver polls this instead of waiting a fixed interval.- //- // A fixed wait loses whichever keystroke had not painted when it expired, and the keystroke- // that has not painted yet is the SLOWEST one -- so the metric dropped precisely the sample it- // exists to catch, and a build that made typing worse read faster. A bigger constant has the- // same defect on a slower machine or a heavier rung; the only wait that does not is one that- // ends when the work does.+ // IS ANYTHING STILL IN FLIGHT? The driver polls this instead of waiting a fixed interval, which+ // would lose whichever keystroke had not painted when it expired: the SLOWEST one, so the metric+ // would drop precisely the sample it exists to catch and a build that made typing worse would read+ // faster. A bigger constant has the same defect on a slower machine.settled() {return { pending: S.pending !== null, samples: S.samples.length, seen: S.seen };},- // Drain. `expected` is how many characters the driver actually sent, so the report can say- // "27 of 30 keystrokes produced a measurement" instead of quoting a median over an unknown- // denominator.+ // Drain. `expected` is how many characters the driver actually sent, so the report can say '27 of+ // 30 keystrokes produced a measurement' instead of quoting a median over an unknown denominator.collect(expected) {const samples = S.samples.slice();const latencies = samples.map((s) => s.latency_ms).sort((a, b) => a - b);@@ -161,8 +147,8 @@.sort((a, b) => a - b);const unanchored = S.unanchored;const seen = S.seen;- // A sample still in flight AT THIS MOMENT is one the collect is about to lose. Reported so- // the driver can fail the reading rather than publish a percentile over what survived.+ // A sample still in flight AT THIS MOMENT is one the collect is about to lose. Reported so the+ // driver can fail the reading rather than publish a percentile over what survived.const pendingNow = S.pending !== null;S.samples = [];S.unanchored = 0;@@ -171,25 +157,25 @@samples: samples.length,samples_attempted: true,expected: expected === undefined ? null : expected,- // The denominator. `inputs_seen` is every keystroke that reached this instrument;- // `samples + coalesced` must account for all of them, and `pending_at_collect` says- // whether one was thrown away by the drain itself.+ // The denominator. `inputs_seen` is every keystroke that reached this instrument; `samples ++ // coalesced` must account for all of them, and `pending_at_collect` says whether one was thrown+ // away by the drain.inputs_seen: seen,pending_at_collect: pendingNow,- // How many samples could not be anchored on their own key event and fell back to the- // input handler's clock. A number quoted from those understates the wait by the queueing- // delay, so it is reported rather than blended in silently.+ // How many samples could not be anchored on their own key event and fell back to the input+ // handler's clock. A number quoted from those understates the wait by the queueing delay, so it+ // is reported rather than blended in.unanchored: unanchored,input_delay_p95_ms:delays.length === 0 ? null : delays[Math.min(delays.length - 1, Math.floor(delays.length * 0.95))],- // Not the same question as `samples`. A dropped sample is a keystroke that arrived while- // a previous one had not painted yet, which is itself the symptom.+ // Not the same question as `samples`: a dropped sample is a keystroke that arrived while a+ // previous one had not painted, which is itself the symptom.coalesced: S.dropped,p50_ms: at(0.5),p95_ms: at(0.95),max_ms: latencies.length === 0 ? null : latencies[latencies.length - 1],- // The FIRST sample is systematically a cold outlier and is reported separately rather- // than dropped, because on a jammed page it is also the largest real number in the set.+ // The FIRST sample is systematically a cold outlier and is reported separately rather than+ // dropped, because on a jammed page it is also the largest real number in the set.first_ms: samples.length === 0 ? null : samples[0].latency_ms,// Proof the characters reached the controlled component and not only the DOM node.text_length: observedText === null ? null : observedText.length,diff --git a/tests/studio/studiobench/instruments/layoutcost.js b/tests/studio/studiobench/instruments/layoutcost.jsindex 8612e5a7f33..271db52cd90 100644--- a/tests/studio/studiobench/instruments/layoutcost.js+++ b/tests/studio/studiobench/instruments/layoutcost.js@@ -2,96 +2,46 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0/*- * layoutcost.js -- counts the DOM operations suspected of forcing synchronous layout- * during streaming, and times them.+ * layoutcost.js -- counts and times the DOM operations suspected of forcing synchronous layout+ * during streaming.*- * WHAT IT COUNTS AND WHY THESE FIVE+ * Under investigation: use-intent-aware-autoscroll.tsx, whose MutationObserver on the thread+ * viewport reads `scrollHeight`, writes `--aui-scroll-stabilizer` and calls `scrollTo` on every+ * delivery -- per streamed character, at a cost proportional to the whole thread. So the five+ * instrumented operations are exactly the ones that shape is made of: scrollHeight reads (the+ * forced-layout trigger), scrollTop writes, scrollTo calls, MutationObserver callbacks AND+ * records per callback (one callback with 400 records and 400 callbacks cost wildly differently+ * and look identical in a callback-only profile), and custom-property writes with+ * `--aui-scroll-stabilizer` counted separately. Timings are here because a count cannot tell+ * 4,000 cheap reads from 4,000 that each walk a 300-message thread.+ * Configured childList + subtree + characterData + an attributeFilter.*- * The mechanism under investigation is- * studio/frontend/src/components/assistant-ui/use-intent-aware-autoscroll.tsx. It installs a- * MutationObserver on the thread viewport (`.aui-thread-viewport.aui-stream-viewport`) with- * `{childList, subtree, characterData, attributes, attributeFilter:["class","hidden",- * "aria-hidden","aria-expanded","data-state"]}`. Every delivery of that observer runs- * `el.scrollHeight` (a read that forces style plus layout when the DOM is dirty, and the DOM is- * always dirty here because the mutation is what woke the observer), then- * `el.style.setProperty("--aui-scroll-stabilizer", "<n>px")`, then- * `el.scrollTo({top, behavior:"instant"})`. During token streaming the characterData mutations- * arrive per streamed character, and each one costs something proportional to the size of the- * whole thread, not to the size of the change. That is the shape of the hypothesis, so the- * instrument records exactly the five operations that shape is made of:+ * SELF COST. Wrapping a getter to time it makes it slower, so the distortion is measured rather+ * than assumed: `selfCostEstimate()` times N wrapped reads against N through the ORIGINAL+ * descriptor on a detached clean element, giving the per-call wrapper overhead; and the Python+ * driver runs the same cell with and without injection and compares frame statistics, which+ * catches cache effects and lost inlining that no microbenchmark sees. If the two runs disagree+ * about the app, the counts stay usable and the timings do not.+ * Reported as `overheadMsPerCall`.*- * scrollHeight READS getter on Element.prototype. The forced-layout trigger.- * scrollTop WRITES setter on Element.prototype. Cheap to issue, expensive later.- * scrollTo CALLS method on Element.prototype. The pin.- * MutationObserver callback invocations, and the number of MutationRecords handed to- * each one. Records per callback is the number that separates "the- * observer fires once with 400 records" from "it fires 400 times",- * which are wildly different costs and look identical in a profile- * that only counts callbacks.- * setProperty("--..") custom property writes, with `--aui-scroll-stabilizer` counted on- * its own so the autoscroll path is separable from every other CSS- * variable the app writes.+ * `clockGranularityMs` exists because a clean read can be faster than a clamped+ * `performance.now()` can resolve: maxMs 0 means "below the clock", not "free".+ * A genuine 0.003 ms read is indistinguishable from zero.*- * Counts alone would be enough to test the "per character" claim. The timings are here because- * a count cannot distinguish 4,000 cheap reads from 4,000 reads that each walk a 300-message- * thread, and the whole question is which of those is happening.+ * OFF BY DEFAULT because it perturbs the measurement; the driver injects it only for the deep+ * tier, where the question has narrowed to "which operation". `window.__sbLayoutCostDisabled`+ * is a secondary escape hatch for bisecting the instrument itself.*- * THE SELF-COST PROBLEM, STATED PLAINLY+ * It does not measure layout time, only how often the app asks for something that can force it;+ * attribution comes from the trace. `window.scrollY`, `getBoundingClientRect`, `offsetHeight`+ * and `getComputedStyle` also force layout and are deliberately NOT wrapped: they are not in the+ * path under investigation and every wrapper makes the run less like the app.*- * Wrapping the `scrollHeight` getter to time it makes `scrollHeight` slower. This instrument- * changes the thing it measures. There is no version of this technique that does not, so the- * honest move is to measure the distortion instead of hoping it is small:- *- * 1. `selfCostEstimate()` times N wrapped reads and N reads through the ORIGINAL descriptor,- * which this file keeps a reference to for exactly this purpose, both on a detached and- * clean element so no real layout work is included. The difference is the per-call- * overhead of the wrapper as installed, including its own bookkeeping. The deep tier can- * subtract `overheadMsPerCall * scrollHeightReads` from the total, or at minimum print it- * next to the total so a reader can see whether the instrument is a rounding error or a- * third of the number.- * 2. The Python driver runs the SAME benchmark cell twice, once with this instrument injected- * and once without, and compares the frame statistics of the two. That is the real check.- * Step 1 measures what the wrapper costs per call in isolation; step 2 measures what the- * whole instrument costs the app in situ, including cache effects and lost inlining that- * no microbenchmark can see. If the two runs disagree about the app, the counts from this- * file are still usable and its timings are not, and that has to be discovered rather than- * assumed.- *- * `clockGranularityMs`, also reported by `selfCostEstimate()`, exists because a single- * `scrollHeight` read on a clean DOM can be faster than `performance.now()` can resolve. In a- * browser without cross-origin isolation the timer is clamped, so a genuine 0.003 ms read is- * indistinguishable from zero. A `maxMs` of 0 therefore means "below the clock", not "free",- * and the granularity number is what lets the report say so.- *- * WHY IT IS OFF BY DEFAULT- *- * See above: it perturbs the measurement. The default benchmark run must be as close to the- * shipping app as the harness can manage, so the driver injects this file only for the deep- * tier, where the question has already narrowed to "which operation" and a known, quantified- * distortion is worth paying for. Injection is the opt-in. Setting- * `window.__sbLayoutCostDisabled = true` before this script runs is a secondary escape hatch- * for bisecting the instrument itself.- *- * WHAT IT DOES NOT MEASURE- *- * It does not measure layout time. It measures how often the app asks for something that can- * force layout, and how long the asking took from JS. Actual layout attribution comes from the- * trace, and the point of this file is to tell the trace where to look.- *- * Reads through `window.scrollY`, `document.documentElement.scrollHeight` in the getter's own- * frame, `getBoundingClientRect`, `offsetHeight` and `getComputedStyle` are NOT wrapped. They- * force layout too. They are not in the code path under investigation, and every wrapper added- * here makes the run less like the app.- *- * ZERO DISCIPLINE- *- * A count of 0 from this file is a real observation: the operation did not happen. A count of 0- * because the patch could not be installed is not, and the two must never print the same. So- * `snapshot()` carries `attempted` per instrumented family, `unavailable` lists the names whose- * original descriptor was missing or non-configurable, and a WebKit build that refuses the- * patch reads as "not attempted" rather than as a quiet zero. The Python side turns those into- * Measure objects. Failing to install is not an error and never throws: a partial instrument- * that is honest about which half ran is more useful than a page that fails to boot.+ * ZERO DISCIPLINE. A 0 because the operation did not happen and a 0 because the patch could not+ * be installed must never print the same, so `snapshot()` carries `attempted` per family and+ * `unavailable` lists names whose descriptor was missing or non-configurable. Failing to install+ * is not an error and never throws.+ * A WebKit build that refuses the patch reads as not attempted.*/(function () {@@ -103,9 +53,7 @@var W = window;- // Idempotence. add_init_script runs per document, and a page that creates an iframe or that- // the driver reloads must not stack wrappers: a doubly wrapped getter would double every- // count and square nothing useful.+ // Idempotence: add_init_script runs per document; double wrapping would double every count.if (W.__sbLayoutCostInstalled) {return;}@@ -116,8 +64,7 @@var MAX_KEYS = 24;var OTHER_KEY = "__other__";- // Capture these before the app can touch them. An app that replaces performance.now (some- // test doubles do) would otherwise silently redefine every timing in this file.+ // Capture before the app can replace performance.now.var perf = W.performance;var rawNow =perf && typeof perf.now === "function" ? perf.now.bind(perf) : null;@@ -138,9 +85,6 @@}}- // ---------------------------------------------------------------------------------------- // State- // ---------------------------------------------------------------------------------------function freshTiming() {return { totalMs: 0, maxMs: 0, samples: 0 };@@ -199,8 +143,7 @@unavailable: [],selfCost: null,clockGranularityMs: null,- // Sink for the self-cost loops. A read whose result is thrown away can in principle be- // optimised out; assigning it somewhere observable keeps both loops honest.+ // Sink so the discarded read cannot be optimised out.sink: 0};@@ -228,18 +171,10 @@}}- // ---------------------------------------------------------------------------------------- // Per-target breakdown- //- // Keyed by a short, stable descriptor rather than by element identity. Holding elements in a- // Map would pin detached DOM alive for the whole run, which on a streaming thread means- // holding every message ever rendered. The key is deliberately coarse: the question is "which- // KIND of element is being read", and a 300-message thread has maybe six kinds.- //- // The cap matters. A page that reads scrollHeight on thousands of distinct elements would- // otherwise grow an unbounded object inside the hot path, so key 25 and beyond all land in- // one bucket and `breakdownCapped` says the truncation happened.- // ---------------------------------------------------------------------------------------+ // Keyed by a coarse stable descriptor, not element identity, so detached DOM is not pinned alive.+ // Coarse because the question is which KIND of element is read, and a 300-message thread has+ // about six kinds.+ // Cap keys so a page reading thousands of distinct elements cannot grow an unbounded object.function keyFor(el) {try {@@ -260,8 +195,7 @@if (list.length > 0) {cls = String(list[0] || "");}- // contains(), not matches(): a class test cannot invoke the selector engine and- // cannot be tricked into anything expensive, and this runs inside the getter.+ // contains(), not matches(): no selector engine inside the getter.isViewport = !!list.contains && list.contains(VIEWPORT_CLASS);}} catch (e) {@@ -300,18 +234,13 @@};state.breakdown[key] = b;if (!isOther) {- // The overflow bucket is not charged to the budget, so `breakdownKeyCount` is the number- // of REAL keys and the object holds at most MAX_KEYS + 1 entries. Counting the bucket- // would make a capped run report 25 keys and a reader would reasonably conclude the cap- // does not work.+ // The overflow bucket is not charged to the budget, so breakdownKeyCount counts REAL keys.+ // At most MAX_KEYS + 1 entries.state.breakdownKeyCount += 1;}return b;}- // ---------------------------------------------------------------------------------------- // Element.prototype patches- // ---------------------------------------------------------------------------------------var ElementProto =typeof W.Element === "function" && W.Element.prototype@@ -360,8 +289,7 @@return origGet.call(this);}var t0 = now();- // Outside try/catch on purpose. If the native getter throws, the app must see the- // same exception it would have seen without this file.+ // Outside try/catch: a throwing native getter must reach the app unchanged.var value = origGet.call(this);try {var dt = now() - t0;@@ -401,18 +329,14 @@Object.defineProperty(ElementProto, "scrollTop", {configurable: true,enumerable: d.enumerable,- // The getter is left exactly as it was. Reading scrollTop also forces layout, but it is- // not the operation under investigation and wrapping it would add cost to a path the- // hypothesis does not name.+ // scrollTop's getter is left alone: not the operation under investigation.get: d.get,set: function (v) {if (!state.active) {origSet.call(this, v);return;}- // Writes are counted, not timed. Issuing a scroll write is cheap; the cost lands in- // the next layout, where a timer wrapped around the setter would not see it. A number- // that looks like a duration but is not one is worse than no number.+ // Writes are counted, not timed; the cost lands in the next layout.try {state.counters.scrollTopWrites += 1;bucketFor(this).scrollTopWrites += 1;@@ -477,9 +401,6 @@}}- // ---------------------------------------------------------------------------------------- // CSSStyleDeclaration.prototype.setProperty- // ---------------------------------------------------------------------------------------function installSetProperty() {var proto =@@ -516,8 +437,7 @@value: function (name) {if (state.active) {try {- // Only custom properties. Ordinary style writes are far more numerous and are not- // part of the mechanism being tested; counting them would bury the signal.+ // Only custom properties: ordinary style writes would bury the signal.if (typeof name === "string" && name.charCodeAt(0) === 45 && name.charCodeAt(1) === 45) {state.counters.customPropSets += 1;if (name === STABILIZER_PROP) {@@ -538,22 +458,10 @@}}- // ---------------------------------------------------------------------------------------- // MutationObserver- //- // A subclass rather than a Proxy or a plain function: `instanceof MutationObserver` keeps- // working, `observe` / `disconnect` / `takeRecords` stay native (they are inherited, and- // `observe` is overridden only to read its arguments before delegating), and the callback is- // the single thing that changes.- //- // The split into viewportObserver and other exists because the app runs several observers and- // an aggregate count cannot tell the autoscroll one from React's own. Two independent- // discriminators are recorded per observer: the observe() target carrying the- // `.aui-stream-viewport` class, and an attributeFilter containing "aria-expanded", which no- // other observer in this app requests. An observer counts as the viewport observer if EITHER- // fires, because the classes are renamed more often than the filter is, and both booleans are- // reported separately so a reader can see when they disagree instead of trusting the merge.- // ---------------------------------------------------------------------------------------+ // A subclass, not a Proxy: instanceof and the native methods keep working.+ // Two independent discriminators (viewport class, aria-expanded filter) separate the autoscroll+ // observer from React's, reported separately so disagreement is visible.+ // The split key is `viewportObserver`; no other observer requests that attributeFilter.function installMutationObserver() {var Native = W.MutationObserver;@@ -577,7 +485,7 @@Wrapped = class extends Native {constructor(callback) {if (typeof callback !== "function") {- // Let the native constructor produce its own TypeError, unchanged.+ // Let the native constructor produce its own TypeError.super(callback);return;}@@ -596,8 +504,7 @@try {return callback.call(this, records, observer);} finally {- // finally, not a trailing statement: a callback that throws still consumed the- // time, and dropping the sample would make a broken build look fast.+ // finally: a throwing callback still consumed the time.try {var dt = now() - t0;state.counters.moCallbacks += 1;@@ -655,8 +562,7 @@} catch (e) {/* classification is optional, observing is not */}- // apply(arguments), not (target, init): observe() with a missing options argument has- // its own spec behaviour and must keep it.+ // apply(arguments), not (target, init): observe() with no options has its own spec behaviour.return super.observe.apply(this, arguments);}};@@ -684,9 +590,6 @@}}- // ---------------------------------------------------------------------------------------- // Self cost- // ---------------------------------------------------------------------------------------function measureClockGranularityMs() {var best = null;@@ -726,9 +629,7 @@state.breakdown = saved.breakdown;state.breakdownKeyCount = saved.breakdownKeyCount;state.breakdownCapped = saved.breakdownCapped;- // The live objects were replaced, so the convenience aliases on the public API have to be- // rebound or a caller reading __sbLayoutCost.counters directly would watch a detached copy- // that never increments again.+ // Rebind the aliases or a caller reading __sbLayoutCost.counters watches a detached copy.api.counters = state.counters;api.timings = state.timings;api.mo = state.mo;@@ -759,22 +660,16 @@var origGet = originals.scrollHeightDesc.get;var result;- // The probe runs through the live wrapper, so it increments the real counters. Saving and- // restoring them keeps the estimate from contaminating the measurement it exists to- // correct, while still timing the wrapper exactly as the app sees it, bookkeeping included.+ // Save/restore counters so the probe does not contaminate the measurement it corrects.var saved = copyRawState();try {var probe = doc.createElement("div");probe.className = "sb-selfcost-probe";- // Detached and never inserted. An attached element would make both loops pay for real- // layout, which is the app's cost and not the instrument's, and would swamp the- // difference this function is trying to resolve. The number produced here is the cost of- // the WRAPPER, which is the only part this file is responsible for.+ // Detached and never inserted: this measures the WRAPPER, not the app's layout.var i;var sink = 0;- // Warm both paths first. The first few hundred calls run in the interpreter tier, and- // whichever loop goes first would otherwise be charged for the JIT.+ // Warm both paths so neither loop is charged for the JIT.for (i = 0; i < n; i++) {sink += probe.scrollHeight;}@@ -841,14 +736,9 @@return copyPlain(result);}- // ---------------------------------------------------------------------------------------- // Public surface- // ---------------------------------------------------------------------------------------function copyPlain(v) {- // Hand written rather than JSON round tripping: JSON turns a non finite number into null- // and silently drops undefined, and a timing that reads null because of the serialiser- // would be indistinguishable from a timing that was never taken.+ // Hand written: JSON turns non-finite into null and drops undefined.if (v === null || typeof v !== "object") {return v;}@@ -878,7 +768,7 @@installedAt: api.installedAt,resetAt: api.resetAt,snapshotAt: nowStamp(),- // Per family, so a count of zero can be told apart from a patch that never landed.+ // Per family, so zero counts differ from a patch that never landed.attempted: copyPlain(state.attempted),unavailable: copyPlain(state.unavailable),counters: copyPlain(state.counters),@@ -918,9 +808,7 @@}function uninstall() {- // Restores the exact descriptors that were captured, so a page can be handed back to a- // measurement that must not carry the instrument's overhead. Counters are left alone: the- // driver usually snapshots after uninstalling.+ // Restores the captured descriptors; counters are left alone.state.active = false;try {if (originals.scrollHeightDesc && ElementProto) {@@ -986,8 +874,7 @@W.__sbLayoutCost = api;if (W.__sbLayoutCostDisabled === true) {- // Injected but told to stand down. Everything reads as not attempted, which is the correct- // answer and not a row of zeros.+ // Injected but told to stand down: everything reads as not attempted, not a row of zeros.return;}diff --git a/tests/studio/studiobench/instruments/pagejs.py b/tests/studio/studiobench/instruments/pagejs.pyindex ee6551d81d0..e568367a20f 100644--- a/tests/studio/studiobench/instruments/pagejs.py+++ b/tests/studio/studiobench/instruments/pagejs.py@@ -57,9 +57,9 @@ class _PageInstrument(Instrument):self.unavailable = f"could not install {self.script_name}: {exc}"def start_cell(self, cell: Cell) -> None:- # Re-read the page every cell. A crashed renderer is recovered by opening a NEW page, so- # an instrument that cached it in attach() would spend the rest of the run evaluating- # against a closed one.+ # Re-read the page every cell: a crashed renderer is recovered by opening a NEW page, so an+ # instrument that cached it in attach() would spend the rest of the run evaluating against a closed+ # one.self.page = self.ctx.page if self.ctx else Nonedef _eval(@@ -117,8 +117,8 @@ class FramesInstrument(_PageInstrument):passctx.cdp.on("Page.screencastFrame", on_frame)- # Tiny frames: the point is the COUNT and its timing, and a full-size capture would- # cost the renderer real encode time inside the window being measured.+ # Tiny frames: the point is the COUNT and its timing, and a full-size capture would cost the+ # renderer real encode time inside the window being measured.ctx.cdp.send("Page.startScreencast",{@@ -213,27 +213,17 @@ class FramesInstrument(_PageInstrument):"against and frame counts rest on the page's own report alone")return result- # THE THIRD CLOCK IS NOT RESOLVED, AND THIS SAYS SO RATHER THAN INVENTING IT.- #- # The design called for a tri-clock check that excludes a window the three clocks disagree- # about. Two of the three are sound here. The 1ms timer has a real expectation -- ticks- # against elapsed/clamp -- and `timer_clock_ratio` below is a direct measure of how much- # of its budget the main thread could answer. The compositor is a liveness signal only,- # for the reason in this method's docstring.- #- # The rAF loop has NO sound expectation on a headless engine. There is no display, so the- # loop runs as fast as it can and the absolute frame rate is not a user-visible quantity- # at all. Normalising it against the best window seen in the cell was tried and is wrong:- # an early idle window runs far above 60, so every later window scores about 0.44 against- # it and `clocks_agree` came out FALSE on 34 of 34 windows of a page that was demonstrably- # steady at 60 fps with 2% blocked time. A gate that fires on every window of every run is- # worse than no gate, because it gets switched off and takes the real ones with it.- #- # So `clocks_agree` is null WITH A REASON, `timer_clock_ratio` is the load-bearing- # availability signal, and the frame columns are reported as the page's own account of- # itself. Restoring the third clock needs a vsync-locked source -- a headed run, or- # `Page.startScreencast` correlated against `LatencyInfo` in a trace, which is Layer 2's- # surface, not this one's.+ # THE THIRD CLOCK IS NOT RESOLVED, AND THIS SAYS SO RATHER THAN INVENTING IT. Two of the three are+ # sound: the 1ms timer has a real expectation and `timer_clock_ratio` measures how much of its+ # budget the main thread could answer, and the compositor is a liveness signal only. The rAF loop+ # has NO sound expectation on a headless engine, since there is no display and it runs as fast as+ # it can. Normalising against the best window in the cell was tried and is wrong: an early idle+ # window runs far above 60, so every later window scores about 0.44 and `clocks_agree` came out+ # FALSE on 34 of 34 windows of a page demonstrably steady at 60 fps with 2% blocked time.+ # So `clocks_agree` is null WITH A REASON, `timer_clock_ratio` is the load-bearing availability+ # signal, and the frame columns are the page's own account of itself. Restoring the third clock+ # needs a vsync-locked source: a headed run, or `Page.startScreencast` correlated against+ # `LatencyInfo` in a trace, which is Layer 2's surface.result["timer_clock_ratio"] = round(lag_ticks / expected_ticks, 3)result["clocks_agree"] = Noneresult["clocks_reason"] = (diff --git a/tests/studio/studiobench/instruments/selfcheck.py b/tests/studio/studiobench/instruments/selfcheck.pyindex 4a7b5aece84..f5052d68d29 100644--- a/tests/studio/studiobench/instruments/selfcheck.py+++ b/tests/studio/studiobench/instruments/selfcheck.py@@ -55,16 +55,16 @@ from ..scoring.schema import ExcludedCell, MeasureSTALL_TOLERANCE_MS = 20.0INJECTED_STALL_MS = 120.0-#: Milliseconds of main-thread time burned per SSE chunk when the streaming-cost injection is-#: armed. Sized against what it has to stand in for: preprocessLaTeX over a 96,000 character reply-#: is projected at 246 ms across a whole stream, so a per-chunk figure in the low single digits is-#: the same ORDER as the effect the metric is meant to resolve, not a boulder it cannot miss.+#: Milliseconds of main-thread time burned per SSE chunk when the streaming-cost injection is armed.+#: Sized against what it stands in for: preprocessLaTeX over a 96,000 character reply is projected+#: at 246 ms across a whole stream, so a per-chunk figure in the low single digits is the same ORDER+#: as the effect the metric must resolve.INJECTED_STREAM_COST_MS = 3.0#: The share of injected cost `stream_cost` must read back. Not 1.0: the metric measures the task#: chain from the chunk to the event loop draining, and a burn queued as a microtask lands inside-#: that chain but the chain also contains the app's own work, so recovery is bounded below by-#: attribution and above by nothing. Under-recovery is the failure that matters, because a metric-#: that recovers a quarter of a known cost will under-report an unknown one by the same factor.+#: that chain but so does the app's own work. Under-recovery is the failure that matters, because a+#: metric that recovers a quarter of a known cost will under-report an unknown one by the same+#: factor.MIN_STREAM_COST_RECOVERY = 0.70INJECTED_INPUT_DELAY_MS = 400.0MIN_INPUT_P95_SHIFT_MS = 350.0@@ -143,8 +143,9 @@ class SelfCheckReport:return {"ok": self.ok, "gates": [gate.to_json() for gate in self.gates]}-# ---------------------------------------------------------------------------------------# pure evaluators++# ---------------------------------------------------------------------------------------@@ -446,13 +447,12 @@ def evaluate_tri_clock()-# ---------------------------------------------------------------------------------------# browser-side snippets and drivers++#: Burns a known amount of main-thread time once, on the next frame. A busy wait, not a sleep: a+#: sleep is recovered through a different scheduler path than a blocked main thread.# ----------------------------------------------------------------------------------------#: Burns a known amount of main-thread time once, on the next frame. A busy wait, not a sleep:-#: a sleep is recovered through a different scheduler path than a blocked main thread, which is-#: the thing being calibrated.STALL_INJECT_JS = """(stallMs) => {return new Promise((resolve) => {@@ -465,20 +465,17 @@ STALL_INJECT_JS = """}"""-#: Burns a known amount of main-thread time PER SSE CHUNK, inside the task chain that chunk-#: starts. This is the streaming analogue of STALL_INJECT_JS: a stall injected once tests whether-#: the frame recorder is watching the right window, and this tests whether the streaming-cost-#: accumulator integrates a cost spread thinly across a whole stream, which is the shape of every-#: effect it was built for and the shape a single-action metric cannot see.-#:-#: THE BURN IS QUEUED AS A MICROTASK, not run inline in the decode wrapper, and that is what makes-#: the check independent of wrapper order. `add_init_script` runs scripts in the order they were-#: added, and the instrument's own TextDecoder wrapper is installed by `Instrument.attach` after-#: the scripts assembled in __main__. Whichever wrapper ends up outermost, a microtask queued-#: during the decode runs after the current task's synchronous code and before the MessageChannel-#: macrotask that closes the measured chain -- so the burn is inside the chain either way. Burning-#: inline would land it before the accumulator timestamps the chunk under one ordering and after-#: it under the other, and the check would silently measure nothing.+#: Burns a known amount of main-thread time PER SSE CHUNK, inside the task chain that chunk starts.+#: The streaming analogue of STALL_INJECT_JS: this tests whether the streaming-cost accumulator+#: integrates a cost spread thinly across a whole stream, which is the shape of every effect it was+#: built for.+#: THE BURN IS QUEUED AS A MICROTASK, not run inline in the decode wrapper, which makes the check+#: independent of wrapper order. `add_init_script` runs scripts in the order added and the+#: instrument's own TextDecoder wrapper is installed by `Instrument.attach` after the scripts+#: assembled in __main__; whichever ends up outermost, a microtask queued during the decode runs+#: after the current task's synchronous code and before the MessageChannel macrotask that closes+#: the chain. Burning inline would land it before the accumulator timestamps the chunk under one+#: ordering and after it under the other.STREAM_COST_INJECT_JS = """(() => {if (window.__sbStreamCostInject) { return; }@@ -582,7 +579,7 @@ INPUT_DELAY_INIT_JS = """})();"""-#: Reads `supportedEntryTypes` directly. Never a try/catch around observe().+#:Reads `supportedEntryTypes` directly. Never a try/catch around observe().LONGTASK_SUPPORT_JS = """() => {try {diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.pyindex 9ce7949c651..11f943bdf5f 100644--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_coverage.py@@ -44,9 +44,9 @@ from studiobench.scene.actions import keystroke # noqa: E402INPUT_JS = Path(__file__).resolve().parents[1] / "input.js"-#: A DOM small enough to run the instrument and explicit enough to hold a paint open. Paints-#: resolve only when this driver says so, which is what makes "still in flight at the drain" a-#: state the test can create rather than race for.+#: A DOM small enough to run the instrument and explicit enough to hold a paint open. Paints resolve+#: only when this driver says so, which makes "still in flight at the drain" a state the test can+#: create rather than race for.DRIVER = """import fs from "node:fs";let now = 0;diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.pyindex 6fb98499b99..2bc41366d1a 100644--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_input_latency.py@@ -72,12 +72,11 @@ def _typed(page, *, delay_armed: bool) -> dict:def test_an_injected_keydown_stall_moves_keystroke_p95():playwright = pytest.importorskip("playwright.sync_api", reason = "playwright is not installed")- # IMPORTORSKIP IS NOT ENOUGH IN THIS SUITE. tests/studio/- # test_heavy_thread_measurement_integrity.py puts a stub `playwright.sync_api` into- # `sys.modules` at collection time and, as its own docstring says, it stays there for the rest- # of the session. So on the CPU job the import above SUCCEEDS against the stub and every name- # it hands back raises RuntimeError when called. The stub is a bare ModuleType with no- # `__file__`, which the real package always has, so that is what distinguishes them.+ # IMPORTORSKIP IS NOT ENOUGH IN THIS SUITE. tests/studio/test_heavy_thread_measurement_integrity.py+ # puts a stub `playwright.sync_api` into `sys.modules` at collection time and it stays there for+ # the rest of the session, so on the CPU job the import above SUCCEEDS against the stub and every+ # name raises RuntimeError when called. The stub is a bare ModuleType with no `__file__`, which the+ # real package always has.if getattr(playwright, "__file__", None) is None:pytest.skip("playwright.sync_api is the CPU-job stub, so there is no browser to drive")@@ -91,9 +90,9 @@ def test_an_injected_keydown_stall_moves_keystroke_p95():context.add_init_script(input_delay_init_script())context.add_init_script(resources.read_text("instruments/input.js"))page = context.new_page()- # `goto`, never `set_content`: `document.open()` removes every listener registered on- # the window, so a page built that way silently discards the injected stall and the- # test would pass against a broken instrument.+ # `goto`, never `set_content`: `document.open()` removes every listener registered on the window, so+ # a page built that way silently discards the injected stall and the test would pass against a+ # broken instrument.page.goto(URL)quiet = _typed(page, delay_armed = False)diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.pyindex 936bd0e4aef..dbe533f6777 100644--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost.py@@ -31,7 +31,7 @@ from studiobench.instruments.streamcost import StreamCostInstrument # noqa: E40STREAMCOST_JS = Path(__file__).resolve().parents[1] / "streamcost.js"-#: `IDLE_GAP_MS` in the instrument. A stall longer than this is the case that used to vanish.+#:`IDLE_GAP_MS` in the instrument. A stall longer than this is the case that used to vanish.IDLE_GAP_MS = 1500HARNESS_JS = r"""@@ -66,8 +66,7 @@ setTimeout(() => {#: The same file, driven through the ordering that loses a burst: a window that closes while the#: chain the last chunk started has not reached its macrotask yet. `readWhilePending` decides which-#: of the two orderings the harness produces, so the pinned case and its control differ by nothing-#: else.+#: ordering the harness produces, so the pinned case and its control differ by nothing else.PENDING_HARNESS_JS = r"""const fs = require("fs");const src = fs.readFileSync(process.argv[2], "utf8");@@ -111,9 +110,9 @@ if (readWhilePending) {#: The same file again, driven through ONE `decode()` of a whole batched read. `payload` decides#: what that read carries: a burst of the pacer's own frames larger than the decoder's scan bound,-#: the same burst small enough to sit under it, or a blob of the size that bound exists to keep out.-#: The burn after the decode stands in for the SSE parse, the delta accumulation and the render ---#: the task chain `delta_task_ms` is supposed to be charging.+#: the same burst small enough to sit under it, or a blob of the size that bound exists to keep+#: out. The burn after the decode stands in for the SSE parse, the delta accumulation and the+#: render.BATCH_HARNESS_JS = r"""const fs = require("fs");const src = fs.readFileSync(process.argv[2], "utf8");@@ -238,7 +237,8 @@ def one_decoded_batch(mode: str, burn_ms: float) -> dict:#: The task chain one burst starts, in the harness above. Large enough that losing it is-#: unmistakable in the assertions and small enough that node's timers stay honest.+#: unmistakable and small enough that node's timers stay honest.+# This is what `delta_task_ms` charges.BURST_CHAIN_MS = 40.0@@ -287,8 +287,7 @@ def test_a_burst_whose_chain_has_already_closed_is_charged_once():assert out["second"]["delta_task_ms"] < BURST_CHAIN_MS * 0.1, out-#: `MAX_SSE_CHUNK_CHARS` in the instrument. A single decode above it is the case that used to be-#: discarded whole.+#:`MAX_SSE_CHUNK_CHARS` in the instrument. A single decode above it is the case that used to be discarded whole.MAX_SSE_CHUNK_CHARS = 65536@@ -405,8 +404,8 @@ class _FakeStreamCostPage:than driving node keeps the test about the DRIVER's ordering, which is where the defect lives."""- #: What one boundary scan costs. `querySelectorAll` collects its matches up front over the- #: whole document, so this is the one part of the instrument that grows with the rung.+ #: What one boundary scan costs. `querySelectorAll` collects its matches up front over the whole+ #: document, so this is the one part of the instrument that grows with the rung.SCAN_MS = 3.9def __init__(self) -> None:@@ -471,8 +470,8 @@ def test_the_close_side_drain_does_not_disturb_the_window_s_own_reading():assert out["streaming_observed"] is Trueassert out["reply_chars_delta"] == 1_000- # The window's own overhead figure is what `read()` returned; the close scan is reported- # beside it rather than folded into it.+ # The window's own overhead figure is what `read()` returned; the close scan is reported beside it+ # rather than folded into it.assert out["overhead_ms"] == pytest.approx(_FakeStreamCostPage.SCAN_MS, abs = 0.05)assert out["close_scan_overhead_ms"] == pytest.approx(_FakeStreamCostPage.SCAN_MS, abs = 0.05)diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.pyindex 5e445774807..cea83f6df43 100644--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_bias.py@@ -37,12 +37,12 @@ if str(_STUDIO_TESTS) not in sys.path:_STREAMCOST_JS = _STUDIO_TESTS / "studiobench" / "instruments" / "streamcost.js"-#: The two documents. 40,000 elements is the size the 289.6 ms figure was measured against; 4,000-#: is roughly what a window of six messages leaves standing at the same rung.+#: The two documents. 40,000 elements is the size the 289.6 ms figure was measured against; 4,000 is+#: roughly what a window of six messages leaves standing at the same rung.FULL_ELEMENTS = 40_000WINDOWED_ELEMENTS = 4_000-#: How many reads to take. The quantity is a few milliseconds, so one reading is noise.+#:How many reads to take. The quantity is a few milliseconds, so one reading is noise.READS = 40#: What counts as "no longer biased". The old reading's residual is milliseconds per call; the new@@ -125,8 +125,8 @@ def browser():def _page(browser, elements: int):page = browser.new_page(viewport = {"width": 900, "height": 600})page.set_content("<!doctype html><meta charset=utf-8><body></body>")- # BEFORE the document exists, exactly as the real harness installs it via add_init_script:- # the hook has to be on TextDecoder.prototype before any decode happens.+ # BEFORE the document exists, exactly as the real harness installs it via add_init_script: the hook+ # has to be on TextDecoder.prototype before any decode happens.page.add_script_tag(content = _STREAMCOST_JS.read_text(encoding = "utf-8"))got = page.evaluate(BUILD_JS, elements)page.evaluate(FEED_JS, 200)@@ -134,8 +134,7 @@ def _page(browser, elements: int):def _per_call_ms(page, which: str) -> float:- # Median of several passes. A single pass on a shared machine picks up whatever else is- # running, and this box is running other people's sweeps.+ # Median of several passes: a single pass on a shared machine picks up whatever else is running.return statistics.median(page.evaluate(TIME_JS, [which, READS]) for _ in range(5))@@ -165,8 +164,8 @@ def test_the_wire_denominator_is_the_same_price_on_a_windowed_document(browser,f"RESIDUAL {wire_residual:+.4f} ms/call")- # The old reading really is cheaper on the smaller document. If this ever stops being true the- # test below is proving nothing, so it is asserted rather than assumed.+ # The old reading really is cheaper on the smaller document. If this ever stops being true the test+ # below is proving nothing, so it is asserted rather than assumed.assert dom_residual > 0, ("the O(document) read was not measurably cheaper on the windowed document, so this "f"machine cannot demonstrate the bias at all (full {dom_full}, windowed {dom_win})"@@ -195,8 +194,8 @@ def test_the_wire_count_is_identical_on_both_documents_for_identical_traffic(broassert a["wire_chars"] == b["wire_chars"] > 0assert a["wire_frames"] == b["wire_frames"] == 200assert a["wire_parse_failures"] == b["wire_parse_failures"] == 0- # The DOM reading, by contrast, is a different number on the two documents -- which is exactly- # why it could not be the denominator.+ # The DOM reading, by contrast, is a different number on the two documents, which is exactly why it+ # could not be the denominator.assert a["wire_chars"] == 200 * len("token 0 ") or a["wire_chars"] > 0diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.pyindex 0db00ed4bb5..d3d5a8c0f7a 100644--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py@@ -124,9 +124,9 @@ def test_a_clean_stream_is_scoreable_and_counts_every_character(page):"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the- # id is which decoder the two numbers above are about, and the exact integer- # depends on how many decoders the page has built before this assertion.+ # Not pinned to a value: the id says which decoder the two numbers above are about, and the integer+ # depends on how many decoders the page has built.+ # Same footing as `carried_flushes` beside it."decoder_id": got["decoder_id"],}, got@@ -206,18 +206,15 @@ def test_a_clean_window_stays_scoreable(page):assert out["reply_chars_delta"] == len("all good")-# ── a frame split INSIDE the "data:" prefix ─────────────────────────-#-# The hook starts buffering when a chunk contains a complete `data:`. The socket does not respect-# that boundary: it can cut a frame anywhere, including between the "da" and the "ta:". Neither-# half then contains the marker and neither completes a buffered frame, so both were discarded ---# and the frame's characters left the denominator WITHOUT incrementing `wireParseFailures`, so the-# window-integrity check above could not see it either. Later frames make the window look-# scoreable while the count underneath it is short, which inflates every cost-per-character-# derived from it, at exactly the moment the instrument exists to measure: chunks arrive ragged-# when the renderer is jammed.+# A frame split INSIDE the "data:" prefix. The hook starts buffering when a chunk contains a+# complete `data:`, but the socket can cut a frame between the "da" and the "ta:": neither half+# contains the marker or completes a buffered frame, so both were discarded and the frame's+# characters left the denominator WITHOUT incrementing `wireParseFailures`. Later frames then make+# the window look scoreable while the count underneath is short, inflating every+# cost-per-character at exactly the moment the instrument exists to measure.+# ── a frame split INSIDE the "data:" prefix ─────────────────────────def _halves(text: str, at: int) -> tuple:return text[:at], text[at:]@@ -236,9 +233,8 @@ def test_the_counter_survives_a_split_inside_the_data_prefix(page):"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the- # id is which decoder the two numbers above are about, and the exact integer- # depends on how many decoders the page has built before this assertion.+ # Not pinned to a value: the id says which decoder the two numbers above are about.+ # Same footing as `carried_flushes` beside it."decoder_id": got["decoder_id"],}, got@@ -277,9 +273,8 @@ def test_unrelated_text_ending_in_a_marker_letter_does_not_corrupt_the_next_fram"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the- # id is which decoder the two numbers above are about, and the exact integer- # depends on how many decoders the page has built before this assertion.+ # Not pinned to a value: the id says which decoder the two numbers above are about.+ # Same footing as `carried_flushes` beside it."decoder_id": got["decoder_id"],}, got@@ -438,9 +433,8 @@ def test_an_aborted_frame_does_not_follow_the_stream_that_replaces_it(page):"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value, on the same footing as `carried_flushes` beside it: the- # id is which decoder the two numbers above are about, and the exact integer- # depends on how many decoders the page has built before this assertion.+ # Not pinned to a value: the id says which decoder the two numbers above are about.+ # Same footing as `carried_flushes` beside it."decoder_id": got["decoder_id"],}, got@@ -534,8 +528,8 @@ def test_an_abort_does_not_cost_the_next_response_its_reading_when_that_one_is_s_feed(page, tail)out = inst.close(_Window())assert out["reply_chars_delta"] == len("a clean reply"), out- # The new response DID carry a frame across a decode boundary, so the counter really moved:- # this window is accepted despite that, not because nothing was counted.+ # The new response DID carry a frame across a decode boundary, so the counter really moved: this+ # window is accepted despite that, not because nothing was counted.assert out["wire_carried_frames_counted_in_window"] == 0, outassert out["reply_chars_scoreable"] is True, out@@ -595,15 +589,14 @@ def test_a_frame_that_really_did_straddle_the_open_still_refuses_its_window(pageassert out["reply_chars_scoreable"] is False, out-# ── the tail of a split frame is stream traffic on the numerator too ─────────-#-# `noteSse` was gated on the marker while the character counter was gated on `looksSse ||-# pending`, so the two halves of one frame were attributed to two different things: the tail's-# characters went into the denominator and its render work went nowhere. The bias is downward on-# `stream_delta_cost_ms_per_kchar` and it grows with fragmentation, which is the regime the-# instrument exists to measure.+# The tail of a split frame is stream traffic on the numerator too. `noteSse` was gated on the+# marker while the character counter was gated on `looksSse || pending`, so the two halves of one+# frame went to two different things: the tail's characters into the denominator and its render+# work nowhere. The bias is downward on `stream_delta_cost_ms_per_kchar` and grows with+# fragmentation, which is the regime the instrument exists to measure.+# ── the tail of a split frame is stream traffic on the numerator too ─────────def test_the_tail_of_a_split_frame_is_counted_as_stream_traffic(page):"""THE DEFECT, at the quantity the numerator is built from.diff --git a/tests/studio/studiobench/instruments/streamcost.js b/tests/studio/studiobench/instruments/streamcost.jsindex 408db129bd5..92df11c06eb 100644--- a/tests/studio/studiobench/instruments/streamcost.js+++ b/tests/studio/studiobench/instruments/streamcost.js@@ -1,108 +1,50 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//-// The streaming-phase cost accumulator. Installed as an init script before any app code runs.-//-// WHAT IS MISSING FROM THE HARNESS THAT THIS FILLS. `time_in_jank_pct`, `jank_index` and-// `max_frame_ms` do cover the stream -- `_frame_measures` pools every window whose kind is not-// `idle`, and the streaming windows are in that pool. What they cannot do is SEPARATE it. One-// 57.3 s film collapses eighteen action windows and the streaming stretch into a single number,-// and the action windows dominate it: measured on a 100K null control, `reasoning_toggle` alone+// The streaming-phase cost accumulator, installed as an init script before any app code runs.+// WHAT THE HARNESS CANNOT DO: `time_in_jank_pct`, `jank_index` and `max_frame_ms` cover the+// stream but cannot SEPARATE it. One 57.3 s film collapses eighteen action windows and the+// streaming stretch into one number; on a 100K null control `reasoning_toggle` alone// contributes 2,865 ms of blocked time at 99.3% busy with a 1,866 ms worst frame, while the-// streaming stretch next to it runs at 3.6% busy. A change to the streaming path moves the second-// and is scored against the first.-//-// The window KIND cannot be used to make that separation, which is the trap this file exists past.-// `SceneRunner._gap_window` opens EVERY inter-slot gap as `kind = "stream"`, so on the standard-// film eighteen windows are labelled `stream:` and only the first four contain any streaming at-// all; `stream:drain` at the end of the cell is 7 ms of nothing, the stream having finished-// forty seconds earlier. The streaming phase is therefore detected here, from the SSE traffic-// itself, and not read off a label.-//-// THREE THINGS ARE ACCUMULATED, and they are different quantities on purpose:-//-// deltaTaskMs The main-thread time of the task chains that SSE chunks start. Measured from-// the first decode of a burst to the moment the event loop next reaches a-// macrotask, so it spans the SSE parse, the delta accumulation, the cumulative-// re-parse and React's own render and commit. This is the TARGETED numerator: it-// excludes the background churn -- highlighting, GC, the app's own timers -- that-// a whole-window blocked-time figure charges to the stream.-//-// blockedMs Blocked time, on exactly the discipline frames.js uses (a 1 ms timer, gap minus-// the calibrated clamp), but ACCUMULATED ONLY WHILE THE STREAM IS RUNNING. This-// is the BROAD numerator, and it is the honest one: it catches stream-driven cost-// that lands outside the delta's own task chain, which is most of the async-// highlighting work.-//-// streamingMs How much of the window the stream was actually running, so a window that is-// half stream and half idle is not read as a whole window of streaming.-//-// WHY THE END OF A TASK CHAIN IS TIMED WITH MessageChannel AND NOT setTimeout. A nested-// `setTimeout(fn, 0)` is clamped to 4 ms once the nesting level passes five, and a chunk arriving-// inside a fetch reader continuation inherits that nesting. That clamp would add a systematic-// ~4 ms to every chunk -- about 720 ms over a 13 s stream -- which is larger than most effects-// this is meant to resolve. A MessageChannel message is not clamped and runs as a macrotask-// immediately after the current task and its microtasks, which is exactly the boundary wanted.-//-// This is NOT the MessageChannel pattern frames.js bans. That one is a ping-pong LOOP that-// re-posts itself and ticks about 150,000 times a second, halving the frame rate before any app-// code runs. This posts once per SSE burst, about fourteen times a second at field cadence, and-// only while a stream is in flight.-//-// AND IT DOES NOT TOUCH requestAnimationFrame. frames.js owns the one rAF loop that counts-// frames. Nothing here schedules, wraps or counts a rAF, so the 888-fps inversion cannot recur-// through this file.+// streaming stretch next to it runs at 3.6% busy.+// The window KIND cannot separate them either: `SceneRunner._gap_window` opens every+// inter-slot gap as `kind = "stream"`, so eighteen windows are labelled `stream:` and only+// four carry streaming. The phase is detected here from the SSE traffic itself.+// THREE ACCUMULATORS, different quantities on purpose. `deltaTaskMs`: main-thread time of the+// task chains SSE chunks start, first decode of a burst to the next macrotask (the TARGETED+// numerator). `blockedMs`: frames.js blocked time, but only while the stream runs (the BROAD+// numerator, catching async highlighting off the chain). `streamingMs`: how much of the window+// the stream was actually running.+// Task-chain end is timed with MessageChannel, not setTimeout: a nested `setTimeout(fn, 0)` is+// clamped to 4 ms past nesting level five, ~720 ms over a 13 s stream. Not the ping-pong loop+// frames.js bans (one post per SSE burst, ~14/s, only in flight) and it never touches rAF, so+// the 888-fps inversion cannot recur here.(() => {if (window.__sb && window.__sb.streamcost) return;window.__sb = window.__sb || {};- // How long after the last SSE chunk the stream is still considered in flight. Field cadence is- // 24 characters every 73 ms and the pacer is deficit-scheduled, so it bursts after a jam rather- // than sleeping a gap per chunk; a jammed renderer can therefore go quiet for a while and then- // receive a burst. 1500 ms is about twenty cadence gaps, long enough not to chop a jam in half- // and short enough that the forty seconds of post-stream film are never counted as streaming.+ // How long after the last SSE chunk the stream still counts as in flight. The+ // deficit-scheduled pacer bursts after a jam rather than sleeping per chunk, so 1500 ms is+ // about twenty cadence gaps: long enough not to chop a jam, short enough that the post-stream+ // film is never counted as streaming.+ // Field cadence is 24 characters every 73 ms.const IDLE_GAP_MS = 1500;- // How much of a decoded chunk is scanned for the relay's framing. The substring search below is- // the per-event work this file declares as O(1), and an unbounded scan over a bundle, a blob or- // a paste would make it O(the page's whole TextDecoder traffic) instead.- //- // IT BOUNDS THE SCAN AND NOT THE PAYLOAD, and it used to bound the payload: the guard read- // `out.length <= MAX_SSE_CHUNK_CHARS` and dropped anything longer, on the premise that a decode- // this large is not relay traffic. That premise is false, and it is false in exactly the case- // this instrument exists for. A read does not carry one cadence gap of the stream, it carries- // everything the browser buffered since the last one, so the size of a read is the arrival rate- // times the length of the stall in front of it. Measured against real chromium reading the real- // pacer through the app's own `getReader()` loop, the largest read of a stream is 32.5- // characters per millisecond of stall at fast cadence, dead linear over 500 to 3,000 ms: a- // 2,000 ms stall lands a 65,000 character read and a 3,000 ms stall a 97,500 character one,- // which the old guard discarded whole. That payload is well-formed SSE -- 470 `data:` frames of- // the pacer's own framing -- and it is the single largest task chain of the stream, so the guard- // took the worst burst out of `sseChunks`, `lastSseAt` and the `deltaTaskMs` numerator at the- // one moment they matter most. Field cadence arrives at 2.97 characters per millisecond and- // would need a 22 s stall, so this is reachable on `--cadence fast` and not on the default.- //- // HOW MUCH IT COST, and it is not a rounding error. The size of the loss is whatever the app- // spends per streamed character, because what is dropped is one chain over a third of the reply.- // Sweeping that rate against the real pacer, with the read loop otherwise identical:- //- // 0.0 ms per 1,000 characters 15.7 vs 15.4 ms -1.3%, which is the noise floor- // 0.5 ms per 1,000 characters 55.7 vs 104.7 ms 46.8% of the numerator gone- // 2.0 ms per 1,000 characters 154.0 vs 350.5 ms 56.1%- // 5.0 ms per 1,000 characters 349.1 vs 835.5 ms 58.2%- //- // A harness that does no work per character loses nothing measurable, which is why this hid: it- // needs an app that actually renders what it streams. It also gets WORSE as the app gets slower,- // because a slower reader falls further behind and the batch it is then handed is bigger -- the- // same shape as the timer attribution above, a build getting worse reading cheaper.- //- // Scanning a bounded head rather than rejecting keeps both of the things the cap was for. The- // work stays bounded by a constant that does not grow with the payload or with the rung, and a- // bundle still has to put `data:` in its first 65,536 characters to be mistaken for a stream --- // while a real burst, which starts a frame every 279 characters, is found immediately. Measured- // in v8: 0.04 us on an SSE batch and 0.68 us on a 2 MB blob with no frame in it, against 0.70 us- // for the same scan left unbounded.+ // How much of a decoded chunk is scanned for the relay's framing: the substring search below+ // is the per-event work this file declares O(1), and an unbounded scan would make it O(the+ // page's whole TextDecoder traffic).+ // IT BOUNDS THE SCAN, NOT THE PAYLOAD. The guard used to drop any decode longer than the cap,+ // on the false premise that a large decode is not relay traffic: a read carries everything+ // buffered since the last one, and at fast cadence the largest read is 32.5 characters per ms+ // of stall, so a 3,000 ms stall lands a well-formed 97,500 character read of 470 `data:`+ // frames, which the old guard discarded whole. Swept against the real pacer that cost 46.8% to+ // 58.2% of the numerator, and nothing at 0 ms per 1,000 characters: it hid on a harness that+ // does no per-character work and got worse as the app got slower. Scanning a bounded head+ // keeps both original goals: constant-bounded work, and a bundle must still put `data:` in its+ // first 65,536 characters.+ // Field cadence arrives at 2.97 characters per millisecond.+ // Measured in v8: 0.04 us on an SSE batch and 0.68 us on a 2 MB blob with no frame in it,+ // against 0.70 us for the same scan left unbounded.const MAX_SSE_CHUNK_CHARS = 65536;const S = {@@ -112,62 +54,38 @@blockedMs: 0,streamingMs: 0,lastSseAt: 0,- // Wall time this file spent inside its own hooks, so the overhead it declares is measured- // rather than asserted. An instrument that guesses its own cost cannot be checked against- // the overhead_growth_with_length gate.+ // Wall time spent inside this file's own hooks, so its declared overhead is measured rather+ // than asserted; an instrument that guesses its own cost cannot be checked against the+ // overhead_growth_with_length gate.overheadMs: 0,decodeCalls: 0,everStreamed: false,- // CUMULATIVE characters of assistant text this page has been SENT, counted off the wire.- // Never reset by `reset()`: the denominator a window wants is the growth across it, which is- // the difference of two readings of this, and resetting it per window would make every- // reading zero.+ // CUMULATIVE characters of assistant text this page has been SENT, counted off the wire. Never+ // reset by `reset()`: a window wants the growth, which is the difference of two readings.wireChars: 0,wireFrames: 0,wireParseFailures: 0,};- // The incremental SSE buffer. A decode() call is a slice of the socket, not an SSE frame: one- // call can carry three frames and half of a fourth. Whatever is left after the last blank line- // stays here until the rest of it arrives.- //- // PER DECODER, NOT PER PAGE. A `TextDecoder` belongs to ONE response: the app builds a new one- // beside its own `buffer` for every streaming request, so no legitimate frame ever spans two of- // them. A single page-wide buffer therefore did not model reassembly, it modelled the socket,- // and `stop_generation` exists to cut a socket mid-frame. The abandoned JSON tail then had no- // `\n\n` to close it, so the next response's first chunk was glued behind it, the merged part- // failed `startsWith("data:")` and was skipped by `continue` WITHOUT counting a parse failure --- // the silently short denominator this file's own comments call the unacceptable outcome. Worse,- // the residue never cleared, so `pending_chars` stayed above zero and `reply_chars_scoreable`- // refused EVERY window from the abort onwards, and once non-empty it pulled unrelated decoder- // traffic in through the `pending.length > 0` branch below.- //- // Keyed weakly, so a finished response's buffer is collected with its decoder rather than- // retained. Within one response the same decoder is reused for every `decode(chunk, {stream:- // true})` call, so a genuine split still reassembles exactly as before.+ // The incremental SSE buffer: a decode() call is a slice of the socket, so one call can carry+ // three frames and half of a fourth. PER DECODER, NOT PER PAGE: a `TextDecoder` belongs to one+ // response, and a page-wide buffer modelled the socket instead of reassembly, so a+ // `stop_generation` cutting a socket mid-frame glued the next response's first chunk behind an+ // unclosed JSON tail, skipped without counting a parse failure while `pending_chars` stayed+ // above zero and refused every later window. Keyed weakly, so the buffer dies with its+ // decoder.+ // It failed `startsWith("data:")`.const DECODER_STATE = new WeakMap();- //: How many times a frame that was ALREADY BUFFERED when a decode call began was completed and- //: counted. Cumulative and never reset, for the same reason `wireChars` is not: a window wants- //: the growth across it. This is what turns "a buffer was pending when the window opened" into- //: "characters that arrived before this window were counted inside it", which is the thing that- //: actually makes the denominator wrong. A buffer left behind by an ABORTED response never- //: completes, so it never increments this and never costs the next response its reading.- //:- //: PER DECODER, alongside `pending`, and it was page-global on `S` until it was not. `close`- //: pairs the buffer pending AT THE OPEN with the flushes carried ACROSS the window, and those- //: were read at two scopes: the buffer belonged to one decoder, the counter to all of them.- //: `stop_generation` leaves the old decoder holding half a frame and `send_turn` follows it in- //: all three shipped schedules, so if that NEW response was itself split -- ordinary, and- //: likeliest exactly when the renderer is jammed -- its own split moved the shared counter and- //: the window was refused for a stale buffer that never flushed. Measured before this change: a- //: two-chunk reply after an abort delivered all 13 of its characters inside the window and was- //: refused, while the same traffic without the abort was accepted. The loss is biased against- //: the most expensive windows, in the flattering direction.+ //: How many times a frame already buffered when a decode call began was completed and counted.+ //: Cumulative and never reset, like `wireChars`: this turns "a buffer was pending at the open"+ //: into "characters that arrived before this window were counted inside it". An aborted+ //: response's buffer never completes, so it never increments this. PER DECODER, like `pending`:+ //: page-global, `close` paired one decoder's buffer with flushes counted across all of them, so+ //: a split after a `stop_generation` refused the window for a stale buffer that never flushed.const CARRIED_BY_ID = new Map();- //: A bounded history, on the same discipline as `MAX_PENDING_CHARS`. `close` asks about the- //: decoder pending when the window OPENED, which by then may be neither active nor alive, so- //: the count cannot live only in the `WeakMap`. Keyed by an integer and holding no reference to- //: any `TextDecoder`, so it cannot defeat the weak scoping above, and trimmed so a long film- //: cannot grow it without limit.+ //: A bounded history, on the same discipline as `MAX_PENDING_CHARS`: `close` asks about the+ //: decoder pending when the window OPENED, which may be neither active nor alive, so the count+ //: cannot live only in the `WeakMap`. Keyed by an integer holding no reference to any+ //: `TextDecoder`, and trimmed so a long film cannot grow it without limit.const MAX_DECODER_HISTORY = 64;let DECODER_SEQ = 0;const noteCarried = (st) => {@@ -177,9 +95,8 @@CARRIED_BY_ID.delete(CARRIED_BY_ID.keys().next().value);}};- //: The carried count of a NAMED decoder, or of whichever one is active when no name is given.- //: A decoder that never carried a frame is absent from the map and answers 0, which is the same- //: answer it would have given had it still been alive.+ //: The carried count of a NAMED decoder, or of whichever is active when no name is given. A+ //: decoder that never carried a frame is absent and answers 0, as it would have while alive.const carriedFor = (id) => {if (typeof id !== "number") return active.carriedFlushes;return CARRIED_BY_ID.get(id) || 0;@@ -198,23 +115,17 @@}return st;};- //: The decoder that most recently delivered a chunk, which is the stream a window is measuring.- //: `wireIntegrity` reports THIS buffer: half a frame of the response being measured is a short- //: denominator and must refuse the window, while half a frame of a response that was aborted- //: three slots ago says nothing about it.- //: Built by `newState()` rather than by hand: a sentinel missing any field a real decoder state- //: carries reads `undefined` on a window that opens before the first stream, and `undefined` is- //: not zero. Its id comes from the same sequence, so `carriedFor` answers about it like any other.+ //: The decoder that most recently delivered a chunk, i.e. the stream a window is measuring.+ //: `wireIntegrity` reports THIS buffer: half a frame of the measured response is a short+ //: denominator, while half a frame of a response aborted three slots ago says nothing. Built by+ //: `newState()` so a window opening before the first stream reads zero, not undefined.let active = newState();- //: The decoder holding a speculative marker fragment, if any, and there is only ever one worth- //: holding. A fragment is at most four characters and lives on the decoder that produced it, but- //: it has to be REPORTED whoever holds it: a decoder whose first chunk is "dat" has not been- //: identified as the stream yet, and leaving its fragment out of `wireIntegrity` would be the- //: instrument saying nothing was outstanding while a frame was. Erring the other way costs at- //: most four characters of unrelated traffic marking a window unscoreable, which is the- //: direction this file has always chosen. It is dropped as soon as some OTHER decoder is- //: identified as the stream, so an unrelated chunk that happened to end in "data" cannot leave a- //: permanent residue behind it.+ //: The decoder holding a speculative marker fragment, if any. A fragment is at most four+ //: characters and lives on the decoder that produced it, but must be reported whoever holds it:+ //: a decoder whose first chunk is "dat" is not identified as the stream yet, and omitting its+ //: fragment would claim nothing was outstanding while a frame was. Erring the other way costs+ //: at most four characters marking a window unscoreable. Dropped once another decoder is the+ //: stream.let markerHold = null;const setMarkerTail = (st, frag) => {st.markerTail = frag;@@ -224,26 +135,18 @@const heldMarkerChars = () =>active.markerTail.length +(markerHold && markerHold !== active ? markerHold.markerTail.length : 0);- // A frame is a few hundred bytes. If this ever grows past a sane bound the stream is not what- // we think it is, and dropping the buffer is better than growing it without limit inside a hook- // that runs fourteen times a second.+ // A frame is a few hundred bytes; past a sane bound the stream is not what we think it is, and+ // dropping the buffer beats growing it without limit in a hook that runs 14 times a second.const MAX_PENDING_CHARS = 262144;- // ── the frame marker, and the halves of it a split can leave ──────────────────────────────────- //- // The socket can cut a frame ANYWHERE, including inside these five characters: one decode()- // returns "da" and the next "ta: {...}\n\n". Neither chunk contains the marker, so the detector- // below saw two unrelated chunks, discarded both, and lost the frame WITHOUT counting a parse- // failure -- which left the wire character count short with nothing anywhere to say so, and- // `reply_chars_scoreable` reporting the window as sound. The denominator going quietly short- // inflates every cost-per-character above it, and it goes short exactly when the renderer is- // jammed and chunks arrive ragged, which is the moment the instrument exists to measure.+ // The socket can cut a frame anywhere, including inside these five characters: one decode()+ // returns "da" and the next "ta: {...}\n\n". Neither contains the marker, so the detector+ // discarded both and lost the frame without counting a parse failure, leaving the denominator+ // short exactly when the renderer is jammed and chunks arrive ragged.const SSE_MARKER = "data:";- // The fragment of the marker the last chunk MIGHT have ended on. At most four characters ("d",- // "da", "dat", "data"), so this cannot become the memory hazard MAX_PENDING_CHARS guards- // `pending` against: buffering whole unrelated TextDecoder chunks on the chance that one of them- // is an SSE frame would be a worse bug than the one being fixed.- // Lives in the per-decoder state above, for the same reason `pending` does.+ // The fragment of the marker the last chunk might have ended on: at most four characters, so+ // it cannot become the memory hazard MAX_PENDING_CHARS guards `pending` against. Lives in the+ // per-decoder state, for the same reason `pending` does.//: The longest tail of `s` that is a PROPER PREFIX of the marker, or "".const partialMarkerTail = (s) => {@@ -253,10 +156,9 @@return "";};- //: Does `s` CONTINUE the marker that `frag` started? This is what keeps a speculative fragment- //: harmless. Unrelated traffic ending in "d" would otherwise be glued onto the front of the next- //: chunk, and a real frame arriving there would become "ddata: {...}", which does not start with- //: the marker and would be skipped in silence -- the same defect one step to the left.+ //: Does `s` CONTINUE the marker `frag` started? Without this, unrelated traffic ending in "d"+ //: would be glued onto the next chunk and a real frame would become "ddata: {...}", skipped in+ //: silence: the same defect one step to the left.const continuesMarker = (frag, s) => {const rest = SSE_MARKER.slice(frag.length);const n = Math.min(rest.length, s.length);@@ -264,34 +166,20 @@};const now = () => performance.now();- // Takes the instant to judge, rather than reading the clock itself, so a caller that has to- // attribute a whole INTERVAL can ask about the instant the interval began. See the timer below.+ // Takes the instant to judge rather than reading the clock, so a caller attributing a whole+ // INTERVAL can ask about the instant it began. See the timer below.const streamingAt = (t) => S.lastSseAt > 0 && t - S.lastSseAt < IDLE_GAP_MS;- // ── the task-chain timer ──────────────────────────────────────────────────────────────────- //- // One pending measurement at a time. A burst of chunks delivered in one task must be charged- // once, from the first chunk to the loop draining, not once per chunk: the pacer sends the- // whole shortfall in a single burst when it has fallen behind, and charging per chunk would- // multiply one task chain by the number of chunks that started it.+ // One pending measurement at a time: a burst delivered in one task must be charged once, from+ // the first chunk to the loop draining. The pacer sends the whole shortfall in one burst when+ // behind, and charging per chunk would multiply one chain by the chunks that started it.let chainStart = null;// Close whatever chain is open and charge it to the accumulator it was opened against. Called- // from the MessageChannel callback -- the ordinary end of a chain -- and from `read()`, which- // is the case that used to lose it.- //- // WHY read() HAS TO DO THIS. `read()` arrives on its own task, from the driver, at a window- // boundary; the message closing the last burst's chain is a task too, and the two are on- // different task queues, so a window can close in the gap between a burst's decode and its- // chain's macrotask. The snapshot then takes `deltaTaskMs` without that burst in it and- // `reset()` zeroes the accumulator, so the callback charges the burst to a fresh one -- which- // the tail `read(0)` in `StreamCostInstrument.close` discards, and which the next `open()`- // resets in any case. The burst's characters are counted in the denominator and its targeted- // cost is silently gone from the numerator. Reproduced against real chromium under a synthetic- // stream of known per-burst cost: 0 to 2 of 200 bursts lost per run, one per window boundary at- // most, and always downward.- //- // The stale message is harmless: it finds `chainStart === null` and returns. A burst that- // starts after this posts a message of its own.+ // from the MessageChannel callback and from `read()`, the case that used to lose it: `read()`+ // arrives on its own task at a window boundary, on a different queue, so a window could close+ // between a burst's decode and its chain's macrotask, counting the characters and losing the+ // cost. Reproduced in chromium: 0 to 2 of 200 bursts lost per run, always downward. A stale+ // message is harmless, finding `chainStart === null`.const closeChain = () => {if (chainStart === null) return;S.deltaTaskMs += now() - chainStart;@@ -300,32 +188,16 @@const chan = new MessageChannel();chan.port1.onmessage = closeChain;- // ── the wire-side character counter ───────────────────────────────────────────────────────- //- // THIS REPLACES AN O(DOCUMENT) READ THAT BIASED THE COMPARISON, and the bias only became- // visible once an arm existed that changes the size of the document.- //- // The denominator used to be read from the DOM: `querySelectorAll('[data-role="assistant"]')`,- // last element, `textContent.length`, at both ends of every window. That is O(the whole- // document) regardless of how few elements match, and it measured 3.9 ms per call against- // 42,000 elements -- 38.8 ms per cell at 10K and 289.6 ms at 100K. The file's own note said it- // "is identical on both arms of an A/B and cancels in a paired ratio". That was true of every- // arm this project had ever run, and it is FALSE for a virtualised one: an arm whose entire- // purpose is to put a tenth of the elements in the document pays a tenth of this cost, so the- // instrument hands the treatment a saving it did not earn, in the direction that flatters- // exactly the hypothesis under test. Nothing measured on such an arm could be quoted while that- // read was in the paired path.- //- // Counting off the wire removes it rather than balancing it. Both arms are fed by the SAME- // pacer -- that is a design invariant of runtime/ab.py, not a coincidence -- so the bytes are- // identical by construction and this counter is identical by construction. It is O(the chunk),- // about fourteen chunks a second, and independent of the thread's size, the rung and the arm.- //- // It is also a BETTER denominator than the one it replaces. The DOM read measured the last- // assistant message, so a `send_turn` mid-film made the reading shrink and the window's growth- // unmeasurable ("it is a different message"). Characters delivered in the window is the- // quantity the cost per character actually wants, and it does not care how many messages they- // were spread over.+ // THIS REPLACES AN O(DOCUMENT) READ THAT BIASED THE COMPARISON. The denominator used to be a+ // `querySelectorAll('[data-role="assistant"]')` last-element textContent read at both ends of+ // every window: 38.8 ms per cell at 10K and 289.6 ms at 100K. "It cancels in a paired ratio"+ // is FALSE for a virtualised arm, which pays a tenth of the cost and is handed a saving it did+ // not earn, flattering the hypothesis under test. Counting off the wire removes it: both arms+ // are fed by the SAME pacer (a runtime/ab.py invariant), so the counter is identical by+ // construction, O(the chunk), and independent of thread size, rung and arm. It is also the+ // better denominator: the DOM read shrank on a mid-film `send_turn`.+ // About 3.9 ms per call against 42,000 elements.+ // `textContent.length` over the whole document.const countDeltaChars = (st, text, carriedMarker) => {const carried = st.pending.length > 0 || Boolean(carriedMarker);st.pending += text;@@ -334,11 +206,11 @@st.pending = "";return;}- // Frames are separated by a blank line. Anything after the last one is incomplete.+ // Frames are separated by a blank line; anything after the last one is incomplete.const parts = st.pending.split("\n\n");st.pending = parts.pop();- // Something that was in the buffer before this call just became a counted frame. A window- // whose OPEN saw a non-empty buffer is only wrong if this happens inside it.+ // Something buffered before this call just became a counted frame. A window whose OPEN saw a+ // non-empty buffer is only wrong if this happens inside it.if (carried && parts.length > 0) noteCarried(st);for (const part of parts) {const line = part.trim();@@ -350,16 +222,16 @@const choices = frame && frame.choices;if (!choices || !choices.length) continue;const delta = choices[0].delta || {};- // Both fields, and both are counted. `_gguf_chat_delta_line` emits reasoning as- // `reasoning_content` WITH `content: ""` beside it, so summing them is not double- // counting; it is the two halves of one turn.+ // Both fields, and both counted: `_gguf_chat_delta_line` emits reasoning as+ // `reasoning_content` with `content: ""` beside it, so summing them is the two halves of one+ // turn, not double counting.const content = typeof delta.content === "string" ? delta.content.length : 0;const reasoning =typeof delta.reasoning_content === "string" ? delta.reasoning_content.length : 0;S.wireChars += content + reasoning;S.wireFrames += 1;} catch (err) {- // COUNTED, NOT SWALLOWED. A parse failure means the denominator is short by an unknown+ // COUNTED, NOT SWALLOWED: a parse failure means the denominator is short by an unknown// amount, and a silently short denominator inflates every cost-per-character above it.S.wireParseFailures += 1;}@@ -377,94 +249,65 @@}};- // ── the SSE detector ──────────────────────────────────────────────────────────────────────- //// The app reads the relay's response through its own TextDecoder, so decode() is the first- // main-thread code that sees a chunk. Wrapping it costs one call per chunk and is O(1) in the- // thread's size. This hook is the flat part of the instrument; the reply-length read below is- // the part that is not, and its measured cost is recorded there rather than claimed away here.+ // main-thread code to see a chunk: one call per chunk, O(1) in thread size. This hook is the+ // flat part of the instrument; the reply-length read below is not, and records its own cost.const nativeDecode = TextDecoder.prototype.decode;TextDecoder.prototype.decode = function (input, options) {const out = nativeDecode.call(this, input, options);const t = now();S.decodeCalls += 1;if (typeof out === "string" && out.length > 0) {- // Reassembly state belongs to THIS decoder. Whether this decoder is also the stream being- // MEASURED is not known yet -- that is decided below, once the chunk has been looked at --- // and promoting it here handed `active` to any decoder in the page. An unrelated one has an- // empty buffer, so `wireIntegrity` then reported nothing outstanding while an SSE decoder- // held half a frame: the window closing there published a denominator short by that frame- // with a clean bill of health, and the window the suffix landed in was handed the whole- // frame and accepted it. A response that was aborted mid-frame keeps its half frame to- // itself.+ // Reassembly state belongs to THIS decoder. Whether it is also the stream being MEASURED is+ // decided below, once the chunk has been looked at: promoting here handed `active` to any+ // decoder in the page, so `wireIntegrity` reported nothing outstanding while an SSE decoder+ // held half a frame.const st = stateFor(this);- // The fragment the previous chunk ended on, but only if THIS chunk continues it. A split+ // The fragment the previous chunk ended on, but only if THIS chunk continues it: a split// inside the marker is repaired here rather than in the buffer, so a fragment that turns out- // to be ordinary text ending in "d" is dropped instead of corrupting the frame behind it.- // `markerTail` is only ever set when `pending` is empty, so the two can never both hold a- // half of the same frame.+ // to be ordinary text ending in "d" is dropped. `markerTail` is only set when `pending` is+ // empty, so the two can never hold halves of the same frame.const carriedMarker = Boolean(st.markerTail && continuesMarker(st.markerTail, out));const chunk = carriedMarker ? st.markerTail + out : out;setMarkerTail(st, "");- // THE BOUND IS ON THE SCAN, NOT ON THE PAYLOAD. A chunk at or under the cap is its own head,- // so nothing is allocated on the ordinary path and the ordinary path is unchanged. Over the- // cap, v8 slices a string by reference rather than by copy, so the head costs no walk of the- // payload either. Both things the cap was for survive: the search stays bounded by a constant- // that grows with neither the payload nor the rung, and a bundle, a blob or a paste still has- // to put the marker in its first MAX_SSE_CHUNK_CHARS characters to be mistaken for a stream.- // What does NOT survive is the old rejection: a read carries everything the browser buffered- // since the last one, so a stall past about two seconds at fast cadence hands the app one- // well-formed 97,500 character batch, and dropping it took the largest burst of the stream- // out of `sseChunks`, `lastSseAt` and the `deltaTaskMs` numerator at the moment a stall makes- // it largest.+ // THE BOUND IS ON THE SCAN, NOT THE PAYLOAD. A chunk at or under the cap is its own head, so+ // the ordinary path allocates nothing, and over the cap v8 slices by reference. Both original+ // goals survive: a constant-bounded search, and a bundle must still put the marker in its+ // first MAX_SSE_CHUNK_CHARS characters. What does not survive is the old rejection, which+ // dropped a well-formed 97,500 character batch at the moment a stall made it largest.const head = chunk.length <= MAX_SSE_CHUNK_CHARS ? chunk : chunk.slice(0, MAX_SSE_CHUNK_CHARS);const looksSse = head.indexOf(SSE_MARKER) >= 0;- // A CONTINUATION IS STREAM TRAFFIC, AND IT STARTS A TASK CHAIN LIKE ANY OTHER CHUNK.- // `noteSse` was gated on the marker alone while the counter below was gated on- // `looksSse || pending`, so the tail of a frame the socket cut inside its JSON body was- // COUNTED in the denominator and charged to nothing. The three quantities it skipped are- // the three the batched-read note above names: `sseChunks`, `lastSseAt` and the- // `deltaTaskMs` numerator. When the suffix lands on a later task the first half rendered- // nothing and its chain has already closed, so the render the suffix does start is charged- // to no window at all, and `stream_delta_cost_ms_per_kchar` is biased DOWNWARD exactly as- // fragmentation rises -- the mirror of the denominator defect the comment below describes,- // and in the flattering direction. A stale `lastSseAt` also lets `replyChars` decide the- // stream has gone idle and return `null` for a window still carrying it.+ // A CONTINUATION IS STREAM TRAFFIC AND STARTS A TASK CHAIN LIKE ANY OTHER CHUNK. `noteSse`+ // was gated on the marker alone while the counter was gated on `looksSse || pending`, so the+ // tail of a frame cut inside its JSON body was counted in the denominator and charged to+ // nothing, biasing `stream_delta_cost_ms_per_kchar` DOWNWARD as fragmentation rises. A stale+ // `lastSseAt` also lets `replyChars` call the stream idle for a window still carrying it.const continuesFrame = st.pending.length > 0;if (looksSse || continuesFrame) noteSse();- // `looksSse || pending` and not just `looksSse`. THE SECOND HALF OF A SPLIT FRAME CONTAINS- // NO "data:" -- it is the tail of a JSON body and a blank line -- so gating the counter on- // that marker dropped the whole frame whenever the socket cut one in two. Found by- // test_the_counter_survives_a_frame_split_across_two_decode_calls, which is precisely the- // condition the instrument exists to measure: chunks arrive ragged when the renderer is- // jammed, so the denominator would have gone quietly short exactly where the numerator went- // up, and the cost per character would have been overstated at the worst moment.- //- // Once a partial frame is held, every subsequent chunk is fed until it completes. Unrelated- // TextDecoder traffic can therefore land in the buffer; it cannot be counted, because it- // will not parse as a frame, and it is bounded by MAX_PENDING_CHARS and reported through- // wire_parse_failures rather than absorbed.- //- // AND THE CHUNK THAT IS NEITHER is kept only as far as it could be the START of a marker.- // That is the third case, the one a complete-marker test cannot see: "da" carries no marker- // and completes no buffered frame, and discarding it loses the frame that arrives next.+ // `looksSse || pending`, not just `looksSse`: THE SECOND HALF OF A SPLIT FRAME CONTAINS NO+ // "data:", so gating the counter on the marker dropped the whole frame whenever the socket cut+ // one in two. Chunks arrive ragged when the renderer is jammed, so the denominator went short+ // exactly where the numerator went up. Once a partial frame is held every chunk is fed until+ // it completes; unrelated traffic is bounded by MAX_PENDING_CHARS and reported through+ // wire_parse_failures. A chunk that is neither is kept only as far as it could START a marker:+ // "da" carries no marker and completes no frame, and discarding it loses the next frame.+ // Found by test_the_counter_survives_a_frame_split_across_two_decode_calls.// The whole chunk is fed to the counter and only the SCAN was bounded above: the denominator// is characters delivered, so counting a batched read's head would understate it by exactly// the amount a stall made it large.- // AND ONLY NOW IS THIS DECODER THE ONE A WINDOW IS MEASURING: it either carries the relay's- // framing or is completing a frame of its own. A decoder that is neither cannot take- // `active` away from one that is.+ // ONLY NOW IS THIS DECODER THE ONE A WINDOW IS MEASURING: it either carries the relay's+ // framing or is completing a frame of its own, and a decoder that is neither cannot take+ // `active` from one that is.if (looksSse || continuesFrame) {- // A fragment held by a DIFFERENT decoder cannot be part of this stream, and this one is- // the stream. Dropped rather than carried, so unrelated traffic that ended in "data"- // cannot report an outstanding frame for the rest of the cell.+ // A fragment held by a DIFFERENT decoder cannot be part of this stream. Dropped rather than+ // carried, so unrelated traffic ending in "data" cannot report an outstanding frame.if (markerHold && markerHold !== st) setMarkerTail(markerHold, "");active = st;countDeltaChars(st, chunk, carriedMarker);} else {- // A speculative fragment is kept on the decoder that produced it, and it is reported- // through `wireIntegrity` only if that decoder is the active stream. Unrelated traffic- // ending in "d" is therefore held without ever claiming to be an outstanding frame.+ // A speculative fragment stays on the decoder that produced it and is reported through+ // `wireIntegrity` only if that decoder is the active stream, so unrelated traffic ending in+ // "d" never claims an outstanding frame.setMarkerTail(st, partialMarkerTail(chunk));}}@@ -472,26 +315,18 @@return out;};- // ── blocked time while streaming ──────────────────────────────────────────────────────────- //// The same 1 ms timer discipline as frames.js, and deliberately the same calibrated clamp:- // blocked time is a SUBTRACTION against an idle floor, and two instruments subtracting two- // different floors would report two different amounts of block for one page. The clamp is read- // from frames.js rather than calibrated again here, so if frames.js could not establish one- // this reports null with frames.js's own reason.+ // blocked time is a subtraction against an idle floor, and two instruments subtracting+ // different floors would report two amounts of block for one page. The clamp is read from+ // frames.js, so if none could be established this reports null with frames.js's reason.let lastTick = now();const tick = () => {const t = now();const gap = t - lastTick;- // Attributed by the state at the START of the interval, never at its end. A timer callback- // cannot run while the main thread is blocked -- it is queued when the timer expires and- // waits for the stack to empty -- so a stall is only ever observed once it is already over.- // Reading `streaming()` here would therefore ask whether the stream is in flight NOW, after- // the stall, and a stall longer than IDLE_GAP_MS answers no: the whole interval, which IS- // the stall, would be dropped from both accumulators. That discards precisely the worst- // stream-induced stalls, and it discards them at exactly the point where a regression grows- // past 1.5 s, so a build getting worse would read cheaper. A negative difference (a chunk- // that arrived DURING the interval) is below the threshold too, which is the overlap case.+ // Attributed by the state at the START of the interval, never at its end: a timer callback+ // cannot run while the main thread is blocked, so a stall is only observed once over, and+ // reading `streaming()` here would drop the whole interval for any stall longer than+ // IDLE_GAP_MS, discarding the worst stream-induced stalls so a worsening build reads cheaper.const wasStreaming = streamingAt(lastTick);lastTick = t;if (wasStreaming) {@@ -504,25 +339,13 @@};setTimeout(tick, 1);- // Characters of the reply currently being streamed. The LAST assistant message only, not the- // whole thread: at the 100K rung the thread holds about 190,000 assistant characters and- // reading all of them costs real time on every window boundary, while the quantity wanted is- // the growth of the one reply the pacer is feeding. Reading the last message alone is O(reply)- // rather than O(thread), so this stays flat as the rung climbs.- //- // It is also the only reading that survives the end of the film. `thread_reopen` and- // `delete_message` rebuild and then cut the thread, so a whole-thread character count jumps and- // then falls for reasons that have nothing to do with streaming.- // MEASURED, and it is not free. `querySelectorAll` is O(the whole DOM), not O(the reply), so- // this is the one part of the instrument whose cost grows with the rung. Called at both ends of- // every window it totalled 38.8 ms per cell at 10K and 289.6 ms at 100K -- about 3.9 ms per- // call against 42,000 elements.- //- // Most of that was spent on windows with no stream in them. The standard film opens about- // thirty-seven windows and only eight carry streaming, so the read is skipped once the stream- // has been finished for longer than the idle gap. `null` is returned rather than a stale count,- // and a window that then turns out to have carried traffic reports its growth as unmeasurable- // with a reason instead of inventing a delta from a reading that was never taken.+ // Characters of the reply currently streaming: the LAST assistant message only, since at 100K+ // the thread holds about 190,000 assistant characters while the quantity wanted is the growth+ // of the one reply the pacer is feeding. O(reply) rather than O(thread), and the only reading+ // that survives `thread_reopen` and `delete_message`. Not free: `querySelectorAll` is O(the+ // whole DOM), 289.6 ms per cell at 100K, and most of it was spent on windows with no stream,+ // so the read is skipped once the stream has been idle past the gap; `null` is returned rather+ // than a stale count, and a window that did carry traffic reports unmeasurable with a reason.const replyChars = (force) => {if (!force && S.lastSseAt > 0 && now() - S.lastSseAt >= IDLE_GAP_MS) return null;const all = document.querySelectorAll('[data-role="assistant"]');@@ -533,12 +356,12 @@window.__sb.streamcost = {// Drain the window. `elapsedMs` is the DRIVER's measure, passed in for the same reason- // frames.js takes it: the page cannot be trusted to read its own clock promptly in the very- // windows this is measuring.+ // frames.js takes it: the page cannot be trusted to read its own clock promptly in exactly+ // the windows this is measuring.read(elapsedMs) {const t = now();- // BEFORE the snapshot, because a chain still in flight belongs to the window that started- // it and `reset()` below is about to throw it away. See `closeChain`.+ // BEFORE the snapshot, because a chain still in flight belongs to the window that started it+ // and `reset()` is about to throw it away. See `closeChain`.closeChain();const f = window.__sb.frames;const clampInfo = f && f.clamp ? f.clamp() : { clampMs: null, reason: "frames.js absent" };@@ -547,8 +370,8 @@sse_chunks_attempted: true,sse_bursts: S.sseBursts,decode_calls: S.decodeCalls,- // Never a bare zero: a window with no streaming in it says so with a flag, and the- // scoring layer skips it rather than folding a zero cost into the numerator.+ // Never a bare zero: a window with no streaming says so with a flag, and the scoring layer+ // skips it rather than folding a zero cost into the numerator.streaming_observed: S.sseChunks > 0,streaming_ms: Math.round(S.streamingMs * 10) / 10,streaming_ms_attempted: true,@@ -558,9 +381,8 @@clamp_ms: clampInfo.clampMs === null ? null : clampInfo.clampMs,};if (clampInfo.clampMs === null || clampInfo.clampMs === undefined) {- // Blocked time is a subtraction against the idle floor. Without a floor there is no- // subtraction to make, and reporting the raw lag instead would be a different quantity- // wearing this one's name.+ // Blocked time is a subtraction against the idle floor; without a floor there is no+ // subtraction to make, and the raw lag would be a different quantity wearing this one's name.out.stream_blocked_ms = null;out.stream_blocked_ms_reason ="no timer clamp was established, so there is no idle floor to subtract: " +@@ -576,34 +398,25 @@return out;},- // THE DENOMINATOR, read off the wire. O(1): it returns a counter the decode hook maintains.- // Called at window open and window close, so the window's growth is the difference.- //- // Cumulative since page load and monotonic, so unlike the DOM reading it replaces there is no- // "the reply shrank, so it is a different message" case to handle: characters delivered only- // ever go up.- // The two things that can make `wireChars` short by an unknown amount, read at the same O(1)- // cost as the counter itself so a window boundary can capture both ends.- // `forId` names the decoder to answer about, which `close` passes back from what `open`- // sampled. WHOSE BUFFER AND WHOSE FLUSH HAVE TO BE THE SAME DECODER, and reading the active- // one at both ends does not make them so: the decoder pending at the open can complete its- // carried frame inside the window and be replaced as active before the close, and comparing- // the two ends' ids would call that a mismatch and discard the very carry it looks for.- // Naming the decoder answers regardless of who is active now, so a window that really did- // count characters delivered before it opened is still refused.+ // THE DENOMINATOR, read off the wire. O(1), called at window open and close so the growth is+ // the difference. Cumulative since page load and monotonic, so unlike the DOM reading it+ // replaces there is no "the reply shrank, so it is a different message" case.+ // The two things that can make `wireChars` short by an unknown amount, at the counter's O(1)+ // cost. `forId` names the decoder to answer about, because the buffer and the flush must+ // belong to the SAME decoder: the one pending at the open can complete its carried frame+ // inside the window and be replaced as active before the close, so comparing the two ends'+ // ids would discard the very carry it looks for.wireIntegrity(forId) {// The marker fragment counts as buffered, because it is: the frame it begins has not been- // counted yet, so a window closing on it has a denominator that is short by that frame. The- // cost of being honest here is that a stray one to four characters of unrelated traffic can- // mark a window unscoreable, which is the direction to err in -- an unscoreable window is- // "we could not tell", and a silently short denominator is "it was fine".+ // counted, so a window closing on it is short by that frame. The cost is that one to four+ // characters of unrelated traffic can mark a window unscoreable, the right way to err.return {failures: S.wireParseFailures,pending_chars: active.pending.length + heldMarkerChars(),// WHICH decoder the two numbers above are about, so the close can ask about the same one.decoder_id: active.id,- // Read as a DELTA across the window by `StreamCostInstrument.close`, so a buffer that was- // pending at the open refuses the window only when ITS OWN frame was completed inside it.+ // Read as a DELTA across the window by `StreamCostInstrument.close`, so a buffer pending at+ // the open refuses the window only when its own frame was completed inside it.carried_flushes: carriedFor(forId),};},@@ -611,14 +424,10 @@return S.wireChars;},- // The OLD reading, kept as a cross-check and NEVER called inside a measured window. See- // `end_cell` in streamcost.py: it runs once per cell, after the film, where its cost is- // charged to nothing.- //- // Worth keeping rather than deleting, because the two numbers answer different questions and- // a disagreement between them is a finding: the wire count is what the app was SENT and the- // DOM count is what it RENDERED. On a windowed arm they are expected to disagree, by exactly- // the messages that are not mounted.+ // The OLD reading, kept as a cross-check and never called inside a measured window (see+ // `end_cell` in streamcost.py, once per cell after the film). Worth keeping because the two+ // answer different questions, what the app was SENT versus what it RENDERED, and on a windowed+ // arm they should disagree by exactly the unmounted messages.replyCharsDom(force) {const t = now();const n = replyChars(Boolean(force));@@ -646,7 +455,7 @@},// For the selftest: force the detector into the streaming state without a real stream, so a- // synthetic injection can be measured on exactly the accumulators a real stream uses.+ // synthetic injection is measured on the accumulators a real stream uses.__markStreaming() {noteSse();},diff --git a/tests/studio/studiobench/instruments/streamcost.py b/tests/studio/studiobench/instruments/streamcost.pyindex 09518632a65..c2db6423920 100644--- a/tests/studio/studiobench/instruments/streamcost.py+++ b/tests/studio/studiobench/instruments/streamcost.py@@ -66,34 +66,30 @@ class StreamCostInstrument(_PageInstrument):self._overhead_ms = 0.0def start_cell(self, cell: Cell) -> None:- # PER CELL, like every other instrument that declares overhead (heap.py, tracing.py,- # coverage.py all zero theirs here). One instrument instance serves the whole session, so- # an accumulator carried across cells reports cell k's overhead as the sum of cells 1..k.- # The rung ladder is run in ascending order, so that sum climbs with the rung and- # `overhead_growth_with_length` -- the gate whose entire job is to catch an instrument- # whose cost tracks the treatment -- would read manufactured growth off a flat instrument.+ # PER CELL, like every other instrument that declares overhead. One instrument instance serves the+ # whole session, so an accumulator carried across cells reports cell k's overhead as the sum of+ # cells 1..k; the rung ladder runs ascending, so `overhead_growth_with_length`, the gate whose job+ # is to catch an instrument whose cost tracks the treatment, would read manufactured growth off a+ # flat instrument.super().start_cell(cell)self._overhead_ms = 0.0self._chars_open = Nonedef open(self, window: Window) -> None:- # Drain first, so the window starts from zero even if the previous close did not run- # (an instrument that raised is disabled for the rest of the cell, and the accumulator- # would otherwise carry that cell's remaining traffic into this window).+ # Drain first, so the window starts from zero even if the previous close did not run (an+ # instrument that raised is disabled for the rest of the cell, and the accumulator would carry+ # that cell's remaining traffic into this window).self._eval("() => window.__sb.streamcost && window.__sb.streamcost.reset()")- # O(1), off the wire. This used to be a `querySelectorAll` over the whole document at- # both ends of every window; see the long note in streamcost.js for why that had to go.+ # O(1), off the wire. This used to be a `querySelectorAll` over the whole document at both ends of+ # every window; see the note in streamcost.js.self._chars_open = self._eval("() => window.__sb.streamcost && window.__sb.streamcost.replyChars()")- # THE DENOMINATOR'S INTEGRITY, SAMPLED AT BOTH ENDS. A frame that cannot be parsed --- # unrelated `TextDecoder` traffic appended while `pending` holds a split SSE frame, say --- # increments a diagnostic and leaves `wireChars` short by an amount nobody can recover.- # Counting the failures was not enough on its own: nothing consulted the count, so the- # affected `reply_chars_delta` was still accepted as a denominator and every cost-per-- # character derived from it came out inflated by an unknown factor. A delta spanning a new- # failure, or ending with an unterminated frame still buffered, is now marked unscoreable- # at the window that contains it.+ # THE DENOMINATOR'S INTEGRITY, SAMPLED AT BOTH ENDS. A frame that cannot be parsed increments a+ # diagnostic and leaves `wireChars` short by an unrecoverable amount. Counting failures was not+ # enough: nothing consulted the count, so the affected `reply_chars_delta` was still accepted and+ # every cost-per-character came out inflated. A delta spanning a new failure, or ending with an+ # unterminated frame still buffered, is now marked unscoreable.self._integrity_open = (self._eval("() => window.__sb.streamcost && window.__sb.streamcost.wireIntegrity()")or {}@@ -109,26 +105,22 @@ class StreamCostInstrument(_PageInstrument):"unavailable": self.unavailable or "the page did not answer","stream_cost_attempted": False,}- # No `force` argument any more. The old DOM read was skipped on windows long past the- # stream because it was expensive; this one is a counter read, so it is taken on every- # window unconditionally and the "the open-read was skipped, so the pair is half-taken"- # case cannot arise.+ # No `force` argument any more. The old DOM read was skipped on windows long past the stream+ # because it was expensive; this is a counter read, taken unconditionally, so the half-taken pair+ # cannot arise.chars_close = self._eval("() => window.__sb.streamcost.replyChars()")out["stream_cost_attempted"] = Trueout["reply_chars_open"] = self._chars_openout["reply_chars_close"] = chars_close- # WHERE THE DENOMINATOR CAME FROM, in the payload rather than in this file. A reader- # comparing a run recorded before this change with one recorded after is comparing two- # different quantities -- the growth of the last mounted message against the characters- # delivered to the page -- and nothing else in the row would tell them so.+ # WHERE THE DENOMINATOR CAME FROM, in the payload rather than in this file: a reader comparing+ # runs recorded before and after this change is comparing the growth of the last mounted message+ # against the characters delivered to the page.out["reply_chars_source"] = "sse_wire"- # ABOUT THE DECODER THAT WAS PENDING AT THE OPEN, named rather than assumed. The refusal- # below pairs a buffer with a flush, and the two were read at different scopes: the buffer- # belonged to one decoder, the flush counter was page-wide. Since `send_turn` follows- # `stop_generation` in every shipped schedule, an abort's orphaned half frame was routinely- # paired with a carried flush produced by the NEW response's own split, refusing a window- # that had delivered every one of its characters. Naming the open's decoder makes the pair- # a statement about one response.+ # ABOUT THE DECODER THAT WAS PENDING AT THE OPEN, named rather than assumed. The refusal below+ # pairs a buffer with a flush, and the two were read at different scopes: the buffer belonged to+ # one decoder, the flush counter was page-wide. Since `send_turn` follows `stop_generation` in+ # every shipped schedule, an abort's orphaned half frame was routinely paired with a carried+ # flush from the NEW response's split, refusing a window that had delivered every character.integrity = (self._eval("(id) => window.__sb.streamcost.wireIntegrity(id)",@@ -138,25 +130,19 @@ class StreamCostInstrument(_PageInstrument):)failures = (integrity.get("failures") or 0) - (self._integrity_open.get("failures") or 0)residual = integrity.get("pending_chars") or 0- # AND THE OTHER END OF THE SAME SPLIT. `pending` is not cleared by `reset()` -- it cannot- # be, it holds half a frame -- so a frame the socket cut across a window boundary is still- # buffered when the NEXT window opens. Its suffix arrives inside that window, the parser- # adds the WHOLE frame's characters there and empties the buffer, and the close reading- # then sees no failures and no residual and calls the window scoreable. Part of its- # denominator was delivered before it opened. The window that closed on the split is- # already refused by `residual`; this refuses its partner, and the two together are what- # make `reply_chars_delta` mean "characters delivered IN this window" rather than- # "characters counted in this window".- #- # AND IT IS THE FLUSH, NOT THE BUFFER, THAT MAKES THE DELTA WRONG. `open()` samples the- # integrity before the action has created the response it is about to measure, so the- # buffer it sees can belong to a response that is already over: `stop_generation` exists to- # cut a socket mid-frame and `send_turn` follows it in every shipped schedule. That half- # frame is never completed and never counted anywhere, so it takes nothing out of this- # window's delta -- refusing on its presence alone threw away the one stream-cost reading- # the send_turn window has. `carried_flushes` counts the completions of frames that were- # already buffered when the decode began, so the pair "a buffer was pending at the open"- # and "a carried frame was counted inside the window" is what the refusal now needs.+ # AND THE OTHER END OF THE SAME SPLIT. `pending` is not cleared by `reset()` (it holds half a+ # frame) so a frame cut across a window boundary is still buffered when the NEXT window opens;+ # its suffix arrives inside that window, the parser adds the WHOLE frame's characters there, and+ # the close sees no failures and calls the window scoreable although part of its denominator was+ # delivered before it opened. The window that closed on the split is already refused by+ # `residual`; this refuses its partner, and together they make `reply_chars_delta` mean+ # 'delivered IN this window'.+ # AND IT IS THE FLUSH, NOT THE BUFFER, THAT MAKES THE DELTA WRONG. `open()` samples integrity+ # before the action has created the response, so the buffer it sees can belong to a response+ # that is already over: `stop_generation` cuts a socket mid-frame and `send_turn` follows it.+ # That half frame is never completed or counted, so refusing on its presence alone threw away+ # the one stream-cost reading the send_turn window has. `carried_flushes` counts completions of+ # frames already buffered when the decode began, so the refusal needs both halves of the pair.pending_at_open = self._integrity_open.get("pending_chars") or 0carried = (integrity.get("carried_flushes") or 0) - (self._integrity_open.get("carried_flushes") or 0@@ -192,13 +178,10 @@ class StreamCostInstrument(_PageInstrument):"than the idle gap when the window opened")elif chars_close < self._chars_open:- # NOW UNREACHABLE, AND KEPT ANYWAY. The wire counter is cumulative and monotonic, so- # it cannot go backwards; if it ever does, the instrument has been reset underneath- # itself and the delta is meaningless. Under the old DOM reading this branch fired- # legitimately and often: `send_turn` starts a new assistant message and- # `thread_reopen` rebuilds the thread, so the LAST message was routinely shorter at- # the close of a window than at its open, and those windows lost their denominator- # for a reason that had nothing to do with streaming.+ # NOW UNREACHABLE, AND KEPT ANYWAY: the wire counter is cumulative and monotonic, so going+ # backwards means the instrument was reset underneath itself and the delta is meaningless. Under+ # the old DOM reading this fired legitimately and often, because `send_turn` starts a new+ # assistant message and `thread_reopen` rebuilds the thread.out["reply_chars_delta"] = Noneout["reply_chars_delta_reason"] = (f"the wire character counter went backwards, from {self._chars_open} to "@@ -211,18 +194,13 @@ class StreamCostInstrument(_PageInstrument):self._overhead_ms += float(out.get("overhead_ms") or 0.0)- # THE CLOSE-SIDE SCAN IS NOT FREE AND WAS NOT COUNTED. `read()` snapshots `overhead_ms`- # into its result and then calls `reset()`, so the `replyChars()` above -- which at close- # is FORCED whenever the window carried traffic, and is the `querySelectorAll` that is- # O(the whole DOM) rather than O(the reply) -- accumulated into a page-side total that- # nothing ever read: the next `open()` begins by resetting it. Exactly half of every- # window's boundary scans were therefore missing from the one number whose job is to make- # the level 0 claim checkable from the payload rather than from a docstring, and the half- # that was missing is the rung-dependent half. Drained here, after the scan, rather than- # by reordering the pair: the close read needs `force`, and `force` is- # `streaming_observed`, which only `read()` can answer. Nothing else is lost by the second- # drain -- every other accumulator it clears was already snapshotted above, and `open()`- # clears them again before the next window.+ # THE CLOSE-SIDE SCAN IS NOT FREE AND WAS NOT COUNTED. `read()` snapshots `overhead_ms` and then+ # calls `reset()`, so the `replyChars()` above (forced at close whenever the window carried+ # traffic, and O(the whole DOM)) accumulated into a page-side total nothing ever read, since the+ # next `open()` resets it. Half of every window's boundary scans were missing from the one number+ # that makes the level 0 claim checkable, and the missing half is the rung-dependent one. Drained+ # here rather than by reordering the pair, because the close read needs `force`, which only+ # `read()` can answer.tail = self._eval("() => window.__sb.streamcost.read(0)")close_scan_ms = float(tail.get("overhead_ms") or 0.0) if isinstance(tail, dict) else 0.0self._overhead_ms += close_scan_ms@@ -233,16 +211,12 @@ class StreamCostInstrument(_PageInstrument):return outdef end_cell(self, cell: Cell) -> Optional[dict]:- # Declared even though level 0 is not obliged to: the whole argument for calling this- # level 0 is that its cost does not grow with the rung, and a claim like that should be- # checkable from the payload rather than from this docstring.- #- # THE ONE DOM READ, and it happens HERE, which is after the last window of the cell has- # closed and before the next cell's first one opens. Its cost is charged to no window and- # therefore to no arm. It exists because the wire count and the DOM count answer different- # questions -- what the app was sent, and what it rendered -- and their disagreement is- # the cheapest available check that a windowed arm is dropping text rather than merely- # not mounting it.+ # Declared even though level 0 is not obliged to: the whole argument for calling this level 0 is+ # that its cost does not grow with the rung, and that should be checkable from the payload.+ # THE ONE DOM READ, after the cell's last window closed and before the next cell's first opens, so+ # its cost is charged to no window and no arm. It exists because the wire count and the DOM count+ # answer different questions, what the app was sent and what it rendered, and their disagreement+ # is the cheapest check that a windowed arm is dropping text rather than merely not mounting it.wire = self._eval("() => window.__sb.streamcost.wireStats()") or {}dom_chars = self._eval("() => window.__sb.streamcost.replyCharsDom(true)")return {diff --git a/tests/studio/studiobench/instruments/tracing.py b/tests/studio/studiobench/instruments/tracing.pyindex 6c7b5a3e99a..23717693e7d 100644--- a/tests/studio/studiobench/instruments/tracing.py+++ b/tests/studio/studiobench/instruments/tracing.py@@ -42,19 +42,18 @@ from typing import Any, Callable, Sequencefrom ..analysis import CellFailure++# L0 is the only level headline numbers may come from: nothing is attached beyond the renderer's own metrics counters.# --------------------------------------------------------------------- ladder-# L0 is the only level headline numbers may come from. Nothing is attached to-# the renderer beyond the metrics counters it already maintains for itself.L0 = "L0"-# L1 adds the timeline trace: task boundaries, frames, layout, user timing. No-# CPU profiler, so no stacks, so no naming, but the cheapest level that gives a-# real task tree.+# L1 adds the timeline trace (task boundaries, frames, layout, user timing). No CPU profiler,+# so no stacks and no naming, but the cheapest level giving a real task tree.L1 = "L1"# L2 adds the V8 CPU profiler. This is the level that NAMES A FRAME.L2 = "L2"-# L3 adds precise coverage and heap sampling. Every timing from L3 is discarded-# by construction; only integers cross the boundary.+# L3 adds precise coverage and heap sampling. Every timing from L3 is discarded by+# construction; only integers cross the boundary.L3 = "L3"LEVELS = (L0, L1, L2, L3)@@ -64,32 +63,29 @@ _TIMELINE_CATEGORIES: tuple[str, ...] = ("disabled-by-default-devtools.timeline","disabled-by-default-devtools.timeline.frame","blink.user_timing",- # `toplevel` carries `ThreadControllerImpl::RunTask` with `src_file` /- # `src_func`, which is the C++ that POSTED each task. `RunTask` itself has- # empty args and says nothing about origin.+ # `toplevel` carries `ThreadControllerImpl::RunTask` with `src_file` / `src_func`, the C++ that+ # POSTED each task; `RunTask` itself says nothing about origin."toplevel","toplevel.flow",- # `scheduler` is the one that matters most and it is easy to miss. It does- # not add events; it adds TYPED ARGS to the `toplevel` slice:- # `args.renderer_main_thread_task_execution.task_type` and- # `args.sequence_manager_task.queue_name`. Those turn task origin from an- # inference into a read value: a React scheduler callback arrives labelled- # `TASK_TYPE_*POSTED_MESSAGE` on `FRAME_PAUSABLE_TQ`, and a timer arrives- # labelled `TASK_TYPE_JAVASCRIPT_TIMER_*`. Without this category both are- # just "a mojo pipe task" and have to be told apart by guesswork.+ # `scheduler` matters most and is easy to miss: it adds no events, only TYPED ARGS to the+ # `toplevel` slice (task_type, queue_name), which turn task origin from an inference into a+ # read value. A React scheduler callback arrives labelled `TASK_TYPE_*POSTED_MESSAGE` on+ # `FRAME_PAUSABLE_TQ`, a timer as `TASK_TYPE_JAVASCRIPT_TIMER_*`.+ # Spelled `args.renderer_main_thread_task_execution.task_type` and+ # `args.sequence_manager_task.queue_name`."scheduler",- # `sequence_manager` adds the DoWork / SelectNextTask / DoIdleWork scoping- # slices. It does NOT carry queue names, contrary to a natural reading.+ # `sequence_manager` adds the DoWork / SelectNextTask / DoIdleWork scoping slices. It does NOT+ # carry queue names, contrary to a natural reading."sequence_manager","latencyInfo","benchmark","input",)-# `disabled-by-default-devtools.timeline.stack` attaches call-site information-# to timeline events. `disabled-by-default-v8.cpu_profiler` is the one that-# produces `Profile` / `ProfileChunk`, and without it every microsecond stays a-# bucket with no stack.+# `...timeline.stack` attaches call-site information to timeline events;+# `disabled-by-default-v8.cpu_profiler` is what produces `Profile` / `ProfileChunk`, and+# without it every microsecond stays a bucket with no stack.+# In full, `disabled-by-default-devtools.timeline.stack`._STACK_CATEGORIES: tuple[str, ...] = ("disabled-by-default-devtools.timeline.stack","disabled-by-default-v8.cpu_profiler",@@ -97,19 +93,15 @@ _STACK_CATEGORIES: tuple[str, ...] = ("disabled-by-default-v8.gc",)-# There is NO knob for the tracing CPU profiler's sampling interval.-# `v8/src/profiler/tracing-cpu-profiler.cc` hard-codes 100 us when the-# `disabled-by-default-v8.cpu_profiler` category is enabled, and the-# `disabled-by-default-v8.cpu_profiler.hires` category that used to lower it is-# no longer registered in V8's category list at all: DevTools still sends it,-# and it does nothing. Effective spacing measured on a real capture is ~150 us-# once sampler overhead and idle are included.-#-# The consequence is structural and cannot be tuned away: a set of 40 us-# scheduler tasks yields at most one sample each, so leaf rankings over short-# task windows are underpowered. That is handled honestly by-# `analysis.cpuprofile.self_time_in_windows`, which reports `underpowered`-# rather than by a category flag that does not work.+# There is NO knob for the tracing CPU profiler's sampling interval:+# `v8/src/profiler/tracing-cpu-profiler.cc` hard-codes 100 us, and the `...cpu_profiler.hires`+# category that used to lower it is no longer registered in V8's category list, so DevTools+# still sends it and it does nothing. Measured spacing on a real capture is ~150 us.+# That category is `disabled-by-default-v8.cpu_profiler.hires`.+# The consequence is structural: a set of 40 us scheduler tasks yields at most one sample each,+# so leaf rankings over short task windows are underpowered.+# `analysis.cpuprofile.self_time_in_windows` reports `underpowered` rather than relying on a+# category flag that does not work.TRACING_PROFILER_INTERVAL_US = 100CATEGORIES_BY_LEVEL: dict[str, tuple[str, ...]] = {@@ -119,19 +111,18 @@ CATEGORIES_BY_LEVEL: dict[str, tuple[str, ...]] = {L3: _TIMELINE_CATEGORIES + _STACK_CATEGORIES,}-# The default perfetto buffer is 200 MB. A long rung at L2 produces a lot of-# ProfileChunks, so the buffer is set explicitly and large: an overflow costs a-# whole cell, and memory is cheaper than a re-run.+# The default perfetto buffer is 200 MB. A long rung at L2 produces many ProfileChunks, so the+# buffer is set explicitly and large: an overflow costs a whole cell, and memory is cheaper+# than a re-run.DEFAULT_BUFFER_KB = 640 * 1024-# `bufferUsageReportingInterval` is clamped to a 250 ms floor in-# `tracing_handler.cc` (`kMinimumReportingInterval`), so asking for less is-# silently ignored. 500 ms is what the DevTools frontend itself uses.+# `bufferUsageReportingInterval` is clamped to a 250 ms floor in `tracing_handler.cc`, so asking+# for less is silently ignored. 500 ms is what the DevTools frontend uses.+# `kMinimumReportingInterval`.BUFFER_POLL_MS = 500-# A trace this close to full did not lose data but was about to, and the next-# run at the next rung will. Surfaced as a warning so a rung ladder does not-# walk off a cliff between two cells.+# A trace this close to full did not lose data but was about to, and the next rung will.+# Surfaced as a warning so a rung ladder does not walk off a cliff.BUFFER_WARN_FRACTION = 0.80@@ -237,8 +228,7 @@ class TraceCapture:self.buffer_kb = int(buffer_kb)self.record_mode = record_modecats = list(CATEGORIES_BY_LEVEL[level]) + [c for c in extra_categories if c]- # Order-stable de-duplication so the recorded category list in the- # report is reproducible between runs.+ # Order-stable de-duplication so the recorded category list is reproducible between runs.self.categories: tuple[str, ...] = tuple(dict.fromkeys(cats))self._wait = wait or (lambda ms: time.sleep(ms / 1000.0))self._usage: list[float] = []@@ -252,8 +242,8 @@ class TraceCapture:def _subscribe(self) -> None:if self._subscribed:return- # Playwright delivers the event's `params` object as the single- # positional argument, and it is `None` for a param-less event.+ # Playwright delivers the event's `params` object as the single positional argument, and it is+ # `None` for a param-less event.self.cdp.on("Tracing.bufferUsage", self._on_buffer_usage)self.cdp.on("Tracing.tracingComplete", self._on_complete)self._subscribed = True@@ -268,9 +258,8 @@ class TraceCapture:def start(self) -> None:if self.level == L0:- # L0 attaches nothing. Making this a no-op rather than an error is- # what lets one code path run every level, including the level whose- # entire definition is "do not instrument".+ # L0 attaches nothing. A no-op rather than an error is what lets one code path run every level,+ # including the one whose definition is 'do not instrument'.self._running = Trueself._t_start = time.perf_counter()return@@ -340,8 +329,8 @@ class TraceCapture:f"Tracing.tracingComplete did not arrive within {timeout_s:.0f}s",)self._wait(50)- # `dataLossOccurred` can arrive first on a very small trace; give the- # stream handle a moment before deciding there is not one.+ # `dataLossOccurred` can arrive first on a very small trace; give the stream handle a moment+ # before deciding there is not one.grace = time.time() + 5.0while "stream" not in self._complete and time.time() < grace:self._wait(50)@@ -405,8 +394,8 @@ class TraceCapture:try:self.cdp.send("IO.close", {"handle": handle})except Exception:- # The stream is temp storage in the browser process; failing to- # close it leaks a file, it does not invalidate the trace we hold.+ # The stream is temp storage in the browser process; failing to close it leaks a file, it does+ # not invalidate the trace we hold.passblob = "".join(parts)raw = base64.b64decode(blob) if binary else blob.encode("utf-8")@@ -415,9 +404,6 @@ class TraceCapture:return raw.decode("utf-8", errors = "strict"), chunks-# ------------------------------------------------------------------- overheads--@dataclassclass OverheadLedger:"""Per-rung measured cost of instrumentation, and the disqualification gate.@@ -438,11 +424,10 @@ class OverheadLedger:run. Same rule, same tolerance, two entry points."""- # rung label -> level -> observed cost of the identical cell (ms, or any- # single consistent scalar such as median frame time)+ # rung label -> level -> observed cost of the identical cell (ms, or any single consistent+ # scalar such as median frame time)cells: dict[str, dict[str, float]] = field(default_factory = dict)- # A level whose overhead ratio rises by more than this across the ladder is- # disqualified from exponent claims.+ # A level whose overhead ratio rises by more than this across the ladder is disqualified from exponent claims.growth_tolerance: float = 0.15def record(self, rung: str, level: str, cost: float) -> None:@@ -612,29 +597,17 @@ def load_trace_json(result: TraceResult) -> dict[str, Any]:return json.loads(result.text)-# ═══════════════════════════════════════════════════════════════════════════# Harness adapter (INTERFACES.md section 3)-# ═══════════════════════════════════════════════════════════════════════════-#-# ONE TRACE PER WINDOW, not one per cell. The alternative was a cell-long trace-# plus `performance.mark` bracketing to locate each window inside it, and it was-# rejected for three reasons, in increasing order of importance:-#-# 1. A mark is a `Runtime.evaluate` round trip, which is renderer-visible work-# inside the measured interval. Small, but correlated with nothing and-# still avoidable.-# 2. A cell-long trace at L2 accumulates ProfileChunks for the whole cell, so-# the buffer-overflow risk grows with cell length. Per-window traces keep-# each buffer bounded by one window.-# 3. Per-window numbers have to exist at `close()` time, because the harness-# emits `window.row()` right after. A trace drained at `end_cell` can only-# backfill the cell row, so every window row would carry nulls.-#-# `Tracing.end` and the drain happen inside `close()`, which the harness calls-# AFTER it has stamped `t_close_ms`. The window duration is therefore already-# fixed and the drain cannot inflate it. The `Tracing.start` in `open()` does-# land inside the window, so it is timed and reported as `overhead_ms` rather-# than waved away.+# ONE TRACE PER WINDOW, not one per cell. The alternative, a cell-long trace plus+# `performance.mark` bracketing, was rejected because a mark is a `Runtime.evaluate` round trip+# inside the measured interval; because a cell-long L2 trace accumulates ProfileChunks so+# overflow risk grows with cell length; and above all because per-window numbers must exist at+# `close()` time, since the harness emits `window.row()` right after and a trace drained at+# `end_cell` would leave every window row null.+# `Tracing.end` and the drain happen inside `close()`, which the harness calls AFTER stamping+# `t_close_ms`, so the window duration is already fixed and the drain cannot inflate it. The+# `Tracing.start` in `open()` does land inside the window, so it is timed and reported as+# `overhead_ms`.from ..analysis import assert_no_bare_zero, measured, merge, unmeasured # noqa: E402from . import register_instrument # noqa: E402@@ -664,26 +637,23 @@ class TracingInstrument:self.ctx = ctxdef start_cell(self, cell: Any) -> None:- # `ctx.page` and `ctx.cdp` may be REPLACED between cells when a crashed- # renderer is recovered, so they are re-read here and never cached in- # `attach`. INTERFACES.md section 7 is explicit about this.+ # `ctx.page` and `ctx.cdp` may be REPLACED between cells when a crashed renderer is recovered,+ # so they are re-read here and never cached in `attach`. INTERFACES.md section 7.self.cell = cellself.cdp = getattr(self.ctx, "cdp", None)self._overhead_ms = 0.0self._windows = 0self._failed_windows = []lvl = int(getattr(cell, "instrument_level", 1) or 1)- # The instrument ladder is expressed HERE. L0 never reaches this method- # because the registry filters on `level`, and L3 uses the same category- # set as L2: L3 adds coverage and heap sampling, which are other- # instruments, not more categories.+ # The instrument ladder is expressed HERE. L0 never reaches this method because the registry+ # filters on `level`, and L3 uses the same category set as L2 (its coverage and heap sampling+ # are other instruments, not more categories).self.trace_level = L1 if lvl <= 1 else (L2 if lvl == 2 else L3)if self.cdp is not None:try:enable_metrics(self.cdp)except Exception:- # Metrics are a cross-check, not the measurement. Losing them- # costs the cross-check and nothing else.+ # Metrics are a cross-check, not the measurement. Losing them costs the cross-check and nothing else.passdef open(self, window: Any) -> None:@@ -702,18 +672,14 @@ class TracingInstrument:self.capture = Noneself.metrics = Noneelse:- # THE METRICS PROBE MAY NOT TAKE THE TRACE WITH IT. `start_cell` says what losing the- # probe is meant to cost -- "the cross-check and nothing else" -- and dropping a- # STARTED capture here cost the whole run instead. Tracing is per-browser: with- # `self.capture` cleared, `close()` returns early and never sends `Tracing.end`,- # `detach()` has nothing to stop, and the next window's `Tracing.start` fails with- # "Tracing has already been started (possibly in another tab)" -- so every remaining- # window reports `tracing did not start`, while the abandoned session keeps recording- # into the browser underneath the numbers this tool is taking. One failed- # `Performance.getMetrics` (the domain is enabled under a bare `except` in- # `start_cell`, so it can be off) was enough. `close()` and `_analyse` already guard- # every use of `self.metrics` with `is not None`, so the cross-check is the only- # thing that goes.+ # THE METRICS PROBE MAY NOT TAKE THE TRACE WITH IT. Dropping a STARTED capture here cost the+ # whole run: tracing is per-browser, so with `self.capture` cleared `close()` returns early and+ # never sends `Tracing.end`, `detach()` has nothing to stop, and the next window's+ # `Tracing.start` fails with 'Tracing has already been started', so every remaining window+ # reports `tracing did not start` while the abandoned session keeps recording underneath. One+ # failed `Performance.getMetrics` was enough. `close()` and `_analyse` already guard every use+ # of `self.metrics`, so only the cross-check goes.+ # `start_cell` says what losing the metrics probe costs, and it can be off.try:self.metrics = MetricsWindow(self.cdp)self.metrics.open()@@ -755,10 +721,9 @@ class TracingInstrument:return payloaddef end_cell(self, cell: Any) -> dict | None:- # `overhead_ms` is required from every instrument at level >= 1- # (INTERFACES.md section 3) and it is what the report layer's- # `overhead_growth_with_length` gate consumes. It is a MEASURED wall- # cost of this instrument's own calls, not an estimate from a table.+ # `overhead_ms` is required from every instrument at level >= 1 (INTERFACES.md section 3) and+ # feeds the report layer's `overhead_growth_with_length` gate. It is a MEASURED wall cost of+ # this instrument's own calls, not an estimate from a table.out = merge(measured("overhead_ms", round(self._overhead_ms, 3)),measured("windows_traced", self._windows),@@ -823,9 +788,8 @@ class TracingInstrument:},)- # Cross-check against the renderer's own accounting. A disagreement means- # the trace is missing tasks, so it is reported rather than silently- # tolerated, but it does not void the window on its own.+ # Cross-check against the renderer's own accounting. A disagreement means the trace is missing+ # tasks, so it is reported rather than silently tolerated, but it does not void the window.try:if self.metrics is not None:payload.update(@@ -845,9 +809,8 @@ class TracingInstrument:unmeasured("task_duration_crosscheck_drift", f"{type(exc).__name__}: {exc}"))- # Named frames need the V8 profiler, which only L2+ turns on. At L1 this- # is an honest null with a reason, never an empty list that reads as- # "nothing was hot".+ # Named frames need the V8 profiler, which only L2+ turns on. At L1 this is an honest null with+ # a reason, never an empty list that reads as 'nothing was hot'.if self.trace_level == L1:payload.update(unmeasured(diff --git a/tests/studio/studiobench/pacer.py b/tests/studio/studiobench/pacer.pyindex f56cb2b0365..c6ba7b43f2a 100644--- a/tests/studio/studiobench/pacer.py+++ b/tests/studio/studiobench/pacer.py@@ -47,12 +47,11 @@ from dataclasses import dataclass, fieldfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom typing import Optional-# The field's own arrival rate: 90,262 characters in 276 seconds is 327 characters a second,-# which at 24 characters a chunk is one chunk every 73ms. It is part of the fixture and getting it-# wrong is how the earlier harness failed to reproduce anything: at a 2ms gap the renderer is the-# bottleneck from the first chunk, the run opens at 45 fps instead of 59, and there is no healthy-# baseline left to degrade FROM. The trace's whole shape is "idle between chunks at the start, not-# idle at the end".+# The field's own arrival rate: 90,262 characters in 276 seconds is 327 a second, which at 24+# characters a chunk is one chunk every 73ms. Getting it wrong is how the earlier harness+# failed to reproduce anything: at a 2ms gap the renderer is the bottleneck from the first+# chunk, the run opens at 45 fps instead of 59, and there is no healthy baseline to degrade+# FROM. The trace's shape is 'idle between chunks at the start, not idle at the end'.CAD_FIELD = (24, 73)# For the small rungs, where the field cadence would spend four minutes streaming a reply nobody# is measuring the cadence of.@@ -72,12 +71,12 @@ class Script:chunk_chars: int = CAD_FIELD[0]gap_ms: int = CAD_FIELD[1]model: str = "studiobench-pacer"- # Emitted before anything else, so a driver can prove the stream it is watching is the one it- # asked for rather than a leftover from the previous cell.+ # Emitted first, so a driver can prove the stream it is watching is the one it asked for rather+ # than a leftover from the previous cell.tag: str = ""- # Set by the driver to make the pacer stop mid-reply, for the stop-generation action. The- # pacer keeps the socket OPEN and idle rather than closing it, because a close is a different- # event from a cancel and the app distinguishes them.+ # Set by the driver to stop the pacer mid-reply for the stop-generation action. The socket is+ # kept OPEN and idle rather than closed, because a close is a different event from a cancel and+ # the app distinguishes them.hold_after_chars: Optional[int] = None@@ -101,8 +100,8 @@ class StreamStats:reasoning_chars_sent: int = 0content_chars_sent: int = 0# How often the deficit scheduler had to send more than one chunk to catch up, and the worst- # shortfall it saw. A run where these stay at zero had no backpressure at all; a run where- # max_deficit is 40 had the socket blocked for three seconds.+ # shortfall. Zero here means no backpressure at all; a max_deficit of 40 means the socket was+ # blocked for three seconds.bursts: int = 0max_deficit: int = 0# Time spent inside a blocking write. This IS the backpressure, measured rather than inferred.@@ -113,9 +112,8 @@ class StreamStats:held: bool = Falseerror: Optional[str] = Noneduration_ms: Optional[float] = None- # The cadence achieved, against the one asked for. A gap that came out at 91ms when 73 was- # requested is a machine that could not keep up, and it belongs in the report next to the- # numbers it distorted.+ # The cadence achieved against the one asked for: a gap of 91ms when 73 was requested is a+ # machine that could not keep up, and it belongs next to the numbers it distorted.requested_gap_ms: int = 0achieved_gap_ms: Optional[float] = None@@ -143,8 +141,8 @@ class PacerState:self.stats: list[StreamStats] = []self.requests_seen = 0self.model_ids: list[str] = ["studiobench-pacer"]- # Flipped by the driver to release a held stream (the stop-generation action asserts the- # app's cancel, so the pacer must be able to sit still while it happens).+ # Flipped by the driver to release a held stream (the stop-generation action asserts the app's+ # cancel, so the pacer must sit still while it happens).self.release = threading.Event()def set_script(self, script: Script) -> None:@@ -196,7 +194,7 @@ class _Handler(BaseHTTPRequestHandler):protocol_version = "HTTP/1.1"server_version = "studiobench-pacer"- # Quiet. A per-request line on stderr at 24 characters a chunk is 4,000 lines a reply.+ # Quiet: a per-request line on stderr at 24 characters a chunk is 4,000 lines a reply.def log_message(self, fmt, *args) -> None: # noqa: A003pass@@ -305,8 +303,8 @@ class _Handler(BaseHTTPRequestHandler):)if not body.get("stream", True):- # A non-streaming request is a bug in the driver, not a mode: every mechanism this- # tool measures is on the streaming path. Say so rather than serve it.+ # A non-streaming request is a bug in the driver, not a mode: every mechanism this tool measures+ # is on the streaming path.self._json(400,{@@ -323,19 +321,18 @@ class _Handler(BaseHTTPRequestHandler):self.send_header("Content-Type", "text/event-stream")self.send_header("Cache-Control", "no-cache, no-transform")self.send_header("Connection", "keep-alive")- # CHUNKED, explicitly. On HTTP/1.1 a response with neither Content-Length nor- # Transfer-Encoding is a keep-alive response of unknown length, and the reader blocks- # forever waiting for a body that already arrived -- measured here as a client that hung- # past a 60s timeout having received every byte. Uvicorn, which is what Unsloth's own- # backend streams through, sends chunked for a StreamingResponse, so this is also the- # framing the relay and the browser see in production.+ # CHUNKED, explicitly. On HTTP/1.1 a response with neither Content-Length nor Transfer-Encoding+ # is a keep-alive response of unknown length, and the reader blocks forever waiting for a body+ # that already arrived, measured here as a client hanging past a 60s timeout having received+ # every byte. Uvicorn, which Unsloth's backend streams through, sends chunked for a+ # StreamingResponse, so this is also the production framing.self.send_header("Transfer-Encoding", "chunked")self.send_header("X-Accel-Buffering", "no")self.end_headers()# The role chunk first, as every OpenAI-compatible server sends it. The app tolerates its- # absence, but its presence is what makes the first delta an APPEND rather than a create,- # which is the path the cumulative buffer takes for every chunk after it.+ # absence, but its presence makes the first delta an APPEND rather than a create, which is the+ # path every chunk after it takes.chunks: list[tuple[str, str]] = []for piece in _split(script.reasoning, script.chunk_chars):chunks.append(("reasoning", piece))@@ -350,8 +347,8 @@ class _Handler(BaseHTTPRequestHandler):last_activity = time.monotonic()while sent < len(chunks):now = time.monotonic()- # DEFICIT SCHEDULING. How many chunks SHOULD have gone out by now, from wall- # clock alone; send the shortfall in one burst. Never sleep(gap) per chunk.+ # DEFICIT SCHEDULING: how many chunks SHOULD have gone out by now, from wall clock alone; send+ # the shortfall in one burst. Never sleep(gap) per chunk.should_have_sent = int((now - t0) / gap_s)deficit = min(should_have_sent, len(chunks)) - sentif deficit <= 0:@@ -359,9 +356,8 @@ class _Handler(BaseHTTPRequestHandler):self._write(b": keep-alive\n\n", stats)stats.keepalives += 1last_activity = now- # Sleep to the next boundary, not a fixed tick: waking late is the SIGNAL, and- # the burst on the next pass is what a real backend does when a socket that- # was backed up clears.+ # Sleep to the next boundary, not a fixed tick: waking late is the SIGNAL, and the burst on the+ # next pass is what a real backend does when a backed-up socket clears.target = t0 + (sent + 1) * gap_stime.sleep(max(0.0, min(target - now, KEEPALIVE_S)))continue@@ -372,9 +368,8 @@ class _Handler(BaseHTTPRequestHandler):for _ in range(deficit):kind, piece = chunks[sent]if kind == "reasoning":- # `content: ""` alongside, which is exactly what the backend's own- # `_gguf_chat_delta_line` emits. Without it the app takes a different- # branch in delta accumulation and the cumulative <think> buffer is+ # `content: ""` alongside, exactly as the backend's own `_gguf_chat_delta_line` emits. Without+ # it the app takes a different delta-accumulation branch and the cumulative <think> buffer is# assembled by a path the shipping build never uses.delta = {"reasoning_content": piece, "content": ""}stats.reasoning_chars_sent += len(piece)@@ -399,20 +394,19 @@ class _Handler(BaseHTTPRequestHandler):script.hold_after_chars is not Noneand stats.chars_sent >= script.hold_after_chars):- # HOLD, do not close. The stop-generation action needs the app to be visibly- # mid-stream while it presses stop; closing the socket instead would end the- # stream on its own and the action would measure a stream that had already- # finished. The keep-alives make the hold indistinguishable from a model- # thinking, which is what it is standing in for.+ # HOLD, do not close: the stop-generation action needs the app visibly mid-stream while it+ # presses stop, and closing would end the stream on its own so the action would measure a+ # stream that had already finished. The keep-alives make the hold indistinguishable from a+ # model thinking.stats.held = Truewhile not self.state.release.wait(KEEPALIVE_S):self._write(b": keep-alive\n\n", stats)stats.keepalives += 1break- # BOTH terminal signals. `streamChatCompletions` throws StreamInterruptedError at EOF- # unless it saw a finish_reason or a [DONE]; sending one and not the other measures the- # app's error path and calls it a benchmark.+ # BOTH terminal signals: `streamChatCompletions` throws StreamInterruptedError at EOF unless it+ # saw a finish_reason or a [DONE], so sending one and not the other measures the app's error+ # path and calls it a benchmark.self._write(_sse(_chunk_frame(request_id, model, {}, finish_reason = "stop")), stats)prompt_tokens = max(1, len(script.reasoning) // 4)completion_tokens = max(1, stats.chars_sent // 4)@@ -643,8 +637,8 @@ def check_planned_streams(streams: list[dict], planned: list[dict]) -> dict:"checked": bool(planned),"planned_turns": len(planned),"turns": turns,- # The throwaway turns and any retry, kept so a reader can see WHAT else the fixture served- # without them being mistaken for a planned turn that failed.+ # The throwaway turns and any retry, kept so a reader can see what else the fixture served+ # without mistaking it for a planned turn that failed."extra": remaining,"reason": None if ok else "; ".join(t["reason"] for t in failures),}@@ -712,8 +706,8 @@ def _selftest() -> int:stats = pacer.last_stats()assert stats and stats["completed"], f"stream did not complete: {stats}"expected = pacer.expected_duration_ms(reasoning, content, "fast")- # The deficit scheduler's whole promise: duration is set by wall clock, not by how fast- # the reader is. Generous bound because a loaded CI box is exactly the case it must hold in.+ # The deficit scheduler's whole promise: duration is set by wall clock, not by how fast the+ # reader is. Generous bound because a loaded CI box is the case it must hold in.assert (elapsed_ms >= expected * 0.75), f"stream finished in {elapsed_ms:.0f}ms, faster than the {expected:.0f}ms cadence"@@ -732,22 +726,20 @@ def _selftest() -> int:f"{stats['bursts']} bursts, max deficit {stats['max_deficit']}")- # PHASE 2: the deficit claim itself. A reader that stalls must NOT slow the stream down;- # it must come back to a burst and the run must still end on wall clock. This is the- # difference between measuring a renderer and measuring a server that waits for one.+ # PHASE 2: the deficit claim itself. A stalled reader must NOT slow the stream down; it must+ # come back to a burst and the run must still end on wall clock, the difference between+ # measuring a renderer and measuring a server that waits for one.pacer.reset()- # Big enough to exhaust the loopback send buffer, which autotunes to `wmem_max` (4MB on- # this kernel). A 400KB reply does not block a loopback write no matter how long the- # reader sleeps, so a smaller body tests nothing.+ # Big enough to exhaust the loopback send buffer, which autotunes to `wmem_max` (4MB on this+ # kernel). A 400KB reply does not block a loopback write however long the reader sleeps.body = "D" * 12_000_000cad = {"chunk_chars": 12_000, "gap_ms": 1}pacer.load("", body, tag = "backpressure", **cad)conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)- # A small receive buffer is what makes this a test. With the default buffer the kernel- # absorbs a 17KB reply whole, the pacer's writes never block, it is never late, and there- # is correctly no deficit to schedule -- which is what the first version of this check- # measured and wrongly called a failure. Backpressure needs a reader that cannot keep up- # AND a pipe that fills, which is exactly the case a jammed renderer produces.+ # A small receive buffer is what makes this a test: with the default buffer the kernel absorbs a+ # 17KB reply whole, the pacer's writes never block and there is correctly no deficit, which the+ # first version of this check measured and wrongly called a failure. Backpressure needs a reader+ # that cannot keep up AND a pipe that fills.conn.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 65536)conn.settimeout(60)conn.connect((pacer.host, pacer.port))@@ -767,8 +759,8 @@ def _selftest() -> int:+ payload)started = time.monotonic()- # Read nothing at all for a stretch several gaps long, then drain. The socket buffer- # absorbs it, so the pacer keeps its own clock and owes a shortfall when it next looks.+ # Read nothing for a stretch several gaps long, then drain: the socket buffer absorbs it, so the+ # pacer keeps its own clock and owes a shortfall when it next looks.time.sleep(0.4)conn.setblocking(True)seen = b""@@ -787,12 +779,10 @@ def _selftest() -> int:f"rather than scheduling against wall clock: {s2}")# The honest form of the machine-independence claim. No scheduler can make a stream finish- # faster than the reader consumes it, so total duration is NOT bounded by the cadence when- # the reader is the throughput limit. What the deficit scheduler promises is narrower and- # is exactly what is checked here: the pacer itself adds no delay beyond the cadence, so- # every millisecond of overrun is accounted for by time blocked in a write. If it were- # sleeping a gap per chunk instead, the overrun would exceed the blocked time by the- # stall, since each late chunk would then push the NEXT one late as well.+ # faster than the reader consumes it, so total duration is NOT bounded by the cadence when the+ # reader is the throughput limit. What is checked is narrower: the pacer adds no delay beyond+ # the cadence, so every millisecond of overrun is accounted for by time blocked in a write.+ # Sleeping a gap per chunk instead, the overrun would exceed the blocked time by the stall.unblocked_ms = stalled_ms - s2["write_block_ms"]assert unblocked_ms <= expected2 * 2.5 + 1000, (f"the stream spent {unblocked_ms:.0f}ms not blocked on a write, against a "diff --git a/tests/studio/studiobench/report/build.py b/tests/studio/studiobench/report/build.pyindex a457dd55f13..1aea467d5ca 100644--- a/tests/studio/studiobench/report/build.py+++ b/tests/studio/studiobench/report/build.py@@ -40,8 +40,8 @@ def _records(path: str | Path) -> list[dict[str, Any]]:try:out.append(json.loads(line))except ValueError:- # A truncated final line is expected when a run was killed mid-write; the- # records before it are still good and are still reported.+ # A truncated final line is expected when a run was killed mid-write; the records before it are+ # still good and are still reported.continuereturn out@@ -63,24 +63,19 @@ def _completion_by_rung(records: Sequence[Mapping[str, Any]]) -> dict[int, tupleif not completed:reason = f"{failure.get('kind') or 'unknown'}: {failure.get('message') or 'no message'}"# NOT COMPLETE, RATHER THAN NOT PRESENT. A cell whose thread lost messages, or that stopped- # following the reply it was measuring, reaches here `completed=True` with a full set of- # timings, because both gates are advisory where they are emitted. Those timings are- # CHEAPER than a correct cell's -- fewer rows to render -- and the ladder is ABSOLUTE, with- # no second arm to contradict them, so the rung was scored against fixed anchors and came- # out green and fast on a cell whose own self-check had already recorded the loss.- #- # It is marked incomplete rather than dropped, which is the distinction this module is- # built on: `score_rung` gives an incomplete rung 0 and KEEPS ITS WEIGHT, and aggregating- # over only the rungs that survived is the crash-beats-limp bug wearing a different hat. A- # build whose thread loses its middle at 100K is not usable at 100K, and lowering `onset`- # is exactly the honest way to say so. Dropping the cell would delete the most important- # thing the run has to say.+ # following the reply, reaches here `completed=True` with a full set of timings, because both+ # gates are advisory where they are emitted. Those timings are CHEAPER than a correct cell's and+ # the ladder is ABSOLUTE with no second arm to contradict them, so the rung came out green and+ # fast on a cell whose own self-check had recorded the loss.+ # It is marked incomplete rather than dropped: `score_rung` gives an incomplete rung 0 and KEEPS+ # ITS WEIGHT, and aggregating over only the rungs that survived is the crash-beats-limp bug in a+ # different hat. A build whose thread loses its middle at 100K is not usable at 100K, and lowering+ # `onset` is the honest way to say so.elif str(r.get("cell_id")) in gate_failures:completed = Falsereason = gate_failures[str(r.get("cell_id"))]- # If a rung was repeated and any rep failed, the rung is not clean. Recording the failure- # rather than the success is deliberate: the interesting fact about a bimodal rung is- # that it can fail, not that it can pass.+ # If a rung was repeated and any rep failed, the rung is not clean. Recording the failure rather+ # than the success is deliberate: the interesting fact about a bimodal rung is that it can fail.prev = out.get(int(tokens))if prev is None or (prev[0] and not completed):out[int(tokens)] = (completed, reason)@@ -91,17 +86,15 @@ def score_payload(path: str | Path, declared_rungs: Sequence[int] | None = None)"""Score one run. `declared_rungs` is the ladder the tier promised, in tokens."""# BEFORE anything is scored, and against the RAW rows. `--report` reads the same file the run- # wrote, so a probe run that was allowed to print its own A/B table would be scorable a second- # time here, hours later, by somebody who was not the one who set the variable. It reads the- # unfiltered rows deliberately: a probed attempt that was later superseded still means this- # payload was recorded with the camera in the shot.+ # wrote, so a probe run allowed to print its own A/B table would be scorable a second time here by+ # somebody who did not set the variable. It reads the unfiltered rows deliberately: a probed+ # attempt later superseded still means the payload was recorded with the camera in the shot.raw = _records(path)refuse_if_probed(raw, str(path))- # A CELL THAT WAS RE-RUN IS SCORED ON THE RUN THAT FINISHED IT. `--resume` re-runs the cells- # that died, under the same `cell_id`, into the same file; scoring both attempts as one cell- # kept the crash forever, so a rung that had since been re-run successfully still came out- # INCOMPLETE and zero. The dead attempt is still in the payload and still in EXCLUDED CELLS,- # which is where a superseded crash belongs -- it is only kept out of the score.+ # A CELL THAT WAS RE-RUN IS SCORED ON THE RUN THAT FINISHED IT. `--resume` re-runs dead cells under+ # the same `cell_id` into the same file, and scoring both attempts as one cell kept the crash+ # forever, so a rung since re-run successfully still came out INCOMPLETE and zero. The dead+ # attempt is still in the payload and in EXCLUDED CELLS; it is only kept out of the score.records = latest_attempt_rows(raw)measures = measures_from_records(records)completion = _completion_by_rung(records)@@ -133,10 +126,10 @@ def build_report() -> tuple[str, LadderScore, dict[str, Any]]:"""Return (rendered summary, ladder, assembled payload)."""- # FIRST, before the payload is assembled. `score_payload` refuses too, but it runs second- # here, and `assemble_rows` validates the schema on the way past: a probed payload that also- # trips some unrelated schema complaint would report THAT instead, and the refusal would- # never be reached. A refusal that any other failure can pre-empt is not a refusal.+ # FIRST, before the payload is assembled. `score_payload` refuses too but runs second here, and+ # `assemble_rows` validates the schema on the way past, so a probed payload that also tripped an+ # unrelated schema complaint would report THAT instead. A refusal that any other failure can+ # pre-empt is not a refusal.refuse_if_probed(_records(path), str(path))payload = assemble_rows(path)ladder = score_payload(path, declared_rungs)diff --git a/tests/studio/studiobench/report/overhead.py b/tests/studio/studiobench/report/overhead.pyindex 9ef4ba773a1..edec25b22fa 100644--- a/tests/studio/studiobench/report/overhead.py+++ b/tests/studio/studiobench/report/overhead.py@@ -26,13 +26,13 @@ from typing import Any, Mappingfrom ..scoring.schema import Measure-#: Overhead may vary this much across the ladder before the level is disqualified. Generous:-#: some growth is unavoidable (more DOM means a longer snapshot), and a tight bound would-#: disqualify every level on every machine, which is the same as having no gate.+#: Overhead may vary this much across the ladder before the level is disqualified. Generous: some+#: growth is unavoidable (more DOM means a longer snapshot), and a tight bound would disqualify+#: every level on every machine.MAX_OVERHEAD_GROWTH_RATIO = 1.5-#: Growth below this many milliseconds is not worth acting on however large the ratio looks.-#: A level whose overhead goes from 0.01 ms to 0.05 ms has a ratio of 5 and does not matter.+#: Growth below this many milliseconds is not worth acting on however large the ratio looks: a level+#: whose overhead goes from 0.01 ms to 0.05 ms has a ratio of 5 and does not matter.MIN_ABSOLUTE_GROWTH_MS = 1.0diff --git a/tests/studio/studiobench/report/payload.py b/tests/studio/studiobench/report/payload.pyindex 2120689033f..4053d0d942e 100644--- a/tests/studio/studiobench/report/payload.py+++ b/tests/studio/studiobench/report/payload.py@@ -205,10 +205,10 @@ def assemble(path: str | Path, *, validate: bool = True) -> dict[str, Any]:return payload-#: How the harness layer's `row_type` values map onto the sections of an assembled payload.-#: Layer 1 writes rows through its `Recorder`; this layer reads them. Keeping the mapping in one-#: table rather than spread through the renderer means a new row type is one line here and a-#: visible `unknown_rows` entry until somebody decides where it belongs.+#: How the harness layer's `row_type` values map onto the sections of an assembled payload. Layer 1+#: writes rows through its `Recorder` and this layer reads them; keeping the mapping in one table+#: means a new row type is one line here and a visible `unknown_rows` entry until somebody decides+#: where it belongs.ROW_TYPE_SECTIONS: Mapping[str, str] = {"run_meta": "header","gate": "selfcheck",@@ -217,28 +217,24 @@ ROW_TYPE_SECTIONS: Mapping[str, str] = {"action": "actions","sample": "samples","failure": "crashes",- # Bookkeeping about HOW the A/B was run, not a measurement of the app. It belongs beside the- # identity fields so a reader can see whether the order was balanced without digging.+ # Bookkeeping about HOW the A/B was run, not a measurement of the app, so it belongs beside the+ # identity fields where a reader can see whether the order was balanced."ab_plan": "header",- # The optional surface sweep. Its own section: a surface row is a coverage fact about the UI,- # not a timing, and folding it into `actions` would put it in front of the scorer.+ # The optional surface sweep. Its own section: a surface row is a coverage fact about the UI, not a+ # timing, and folding it into `actions` would put it in front of the scorer."surface": "surfaces",- # The comparability key. Its own section rather than `header`, for two independent reasons.- #- # First, `header` is collapsed to its FIRST row when the payload is assembled, so a second row- # filed there is dropped without a word. Second, the row's `fields` block is identity- # bookkeeping and not a measurement: `instrument_level: 0` is a true statement about how the- # run was instrumented, exactly like the `identity` and `config` subtrees, so the section is- # exempted from the bare-zero ban rather than made to fake a Measure. Left unmapped the row- # fell into `unknown_rows`, which nothing exempts, and the walker killed every real-path+ # The comparability key. Its own section rather than `header` for two reasons: `header` is+ # collapsed to its FIRST row when the payload is assembled, so a second row filed there is dropped+ # without a word; and the row's `fields` block is identity bookkeeping, not a measurement, so the+ # section is exempted from the bare-zero ban rather than made to fake a Measure. Left unmapped the+ # row fell into `unknown_rows`, which nothing exempts, and the walker killed every real-path# session on `$.unknown_rows[0].fields.instrument_level = 0`."comparability": "comparability",- # The terminal marker for a cell that did not finish. NOT `cells`, which is what the scorer- # reads, and NOT an exclusion source: the `cell` row it follows is emitted with- # `completed: false` immediately before it on the same path, and `excluded_from_rows` already- # turns that into a `rung_incomplete` exclusion. Filing this row as a second exclusion would- # count one abort twice. It exists so a reader scanning FORWARD can discard the cell's window- # rows, and that is all it is kept for here.+ # The terminal marker for a cell that did not finish. NOT `cells`, which is what the scorer reads,+ # and NOT an exclusion source: the `cell` row it follows is emitted with `completed: false`+ # immediately before it and `excluded_from_rows` already turns that into a `rung_incomplete`+ # exclusion, so filing this as a second exclusion would count one abort twice. It exists so a+ # reader scanning FORWARD can discard the cell's window rows."cell_aborted": "aborted_cells",}diff --git a/tests/studio/studiobench/report/render.py b/tests/studio/studiobench/report/render.pyindex fa8da31048a..e5e4263934d 100644--- a/tests/studio/studiobench/report/render.py+++ b/tests/studio/studiobench/report/render.py@@ -34,7 +34,7 @@ from ..scoring.schema import Measurefrom ..scoring.score import LadderScore, RungScorefrom .payload import excluded_totals-#: The three frame numbers that must appear together or not at all.+#:The three frame numbers that must appear together or not at all.HEADLINE_FRAME_METRICS = ("time_in_jank_pct", "jank_index", "max_frame_ms")@@ -141,8 +141,8 @@ def render_rung_table(ladder: LadderScore) -> str:def render_rung_metrics(rung: RungScore, *, indent: str = " ") -> str:lines = [f"{indent}{rung.tokens:,} tokens -- score {rung.score:.1f}"]if not rung.complete:- # Printing six "not attempted" lines for a rung that never ran buries the one fact that- # matters, which is why it did not run.+ # Printing six "not attempted" lines for a rung that never ran buries the one fact that matters,+ # which is why it did not run.lines.append(f"{indent} INCOMPLETE: {rung.incomplete_reason}")lines.append(f"{indent} scores 0 and keeps its weight; no metric was measured at this rung"@@ -217,9 +217,9 @@ def render_ab_table(result: AbResult) -> str:lines.append("")lines.append("VOID. No numbers are quotable from this comparison.")lines.append(f" {result.void_reason}")- # The paragraph under the reason explains the NULL CONTROL, and printing it under an- # incomplete plan named the wrong cause for the void. A void has more than one cause; the- # reason line carries which one, and only a null control gets the null-control paragraph.+ # The paragraph under the reason explains the NULL CONTROL, and printing it under an incomplete+ # plan named the wrong cause for the void. A void has more than one cause; the reason line carries+ # which one.if result.is_null_control:lines.append(" The null-treatment control measures whether this machine can currently tell "@@ -252,10 +252,9 @@ def render_ab_table(result: AbResult) -> str:if metric.ci_low is not Noneelse "too few pairs")- # A BOUND IS NOT A MEASUREMENT. An arm under its instrument floor contributes the floor,- # so the ratio understates the true magnitude and must not be quoted as a point estimate.- # Marked in the ratio cell rather than footnoted, for the reason the void path gives: the- # table gets screenshotted and the note does not.+ # A BOUND IS NOT A MEASUREMENT. An arm under its instrument floor contributes the floor, so the+ # ratio understates the true magnitude and must not be quoted as a point estimate. Marked in the+ # ratio cell rather than footnoted, because the table gets screenshotted and the note does not.ratio_cell = (f">={metric.ratio_geomean:.3f}"if metric.bounded and metric.ratio_geomean >= 1.0diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py b/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.pyindex eae5980188c..3e55a5e8fd4 100644--- a/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py+++ b/tests/studio/studiobench/report/selftest/test_studiobench_ab_report_resume.py@@ -72,7 +72,7 @@ def test_a_fully_resumed_ab_keeps_the_table_the_measured_run_wrote(tmp_path):_render_ab(paths, SIDES, MEASURED, "c0ffee")measured = (paths.out / "ab.md").read_text(encoding = "utf-8")# A real reading, as opposed to an empty table. The fixture is one pair, so the verdict is- # INCONCLUSIVE rather than a direction; what matters here is that something was measured.+ # INCONCLUSIVE rather than a direction; what matters is that something was measured.assert "NO READING" not in measuredassert "keystroke_p95_ms" in measureddiff --git a/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py b/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.pyindex 204a34481cb..5830449ebe0 100644--- a/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py+++ b/tests/studio/studiobench/report/selftest/test_studiobench_cli_contracts.py@@ -95,20 +95,17 @@ def test_one_attached_studio_driven_twice_is_a_null_control():assert is_null_control(sides) is True-# ── the stream-cost injection needs two origins ─────────────────────────────────────────────-#-# The injection is a CONTEXT init script gated on `window.location.origin`, and both arms are-# driven by one browser context and one page, so the origin is the only thing that can tell them-# apart. Two arms on one origin therefore BOTH burn the injected cost, the difference between them-# is zero, and `evaluate_stream_cost_recovery_gate` -- which reads back-# `(injected_rate - base_rate) * chars` -- reports a recovery of nothing and blames the-# accumulator. The one flag whose job is to separate "the change did nothing" from "the metric is-# not watching" would answer the second when the truth is neither.-#+# The stream-cost injection needs two origins. The injection is a CONTEXT init script gated on+# `window.location.origin`, and both arms are driven by one browser context and one page, so the+# origin is the only thing that can tell them apart. Two arms on one origin BOTH burn the injected+# cost, the difference is zero, and `evaluate_stream_cost_recovery_gate` reports a recovery of+# nothing and blames the accumulator, so the one flag whose job is to separate 'the change did+# nothing' from 'the metric is not watching' answers the second when the truth is neither.# One attached Unsloth driven twice is not a mistake in general: it is a null control this tool-# detects on purpose, pinned by the test directly above. It is only fatal with the injection on.+# detects on purpose, pinned by the test above. It is only fatal with the injection on.+# ── the stream-cost injection needs two origins ─────────────────────────────────────────────def _inject_args(attach, attach_b, *extra):return parse_args(["--attach", attach, "--ab", "main", "--attach-b", attach_b, *extra])diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py b/tests/studio/studiobench/report/selftest/test_studiobench_liveness.pyindex fd3ba853ed2..91c05f7dfe5 100644--- a/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py+++ b/tests/studio/studiobench/report/selftest/test_studiobench_liveness.py@@ -62,18 +62,18 @@ def test_an_action_that_did_not_run_fails(tmp_path):def test_a_missed_slot_fails_even_though_the_action_ran(tmp_path):- # `ran` and `slot_missed` are different failures: the second means the film moved on without- # it, so its timing describes a different session than every other arm's. Default slack is 0,- # which is the right default for the quiet machine a measurement is taken on.+ # `ran` and `slot_missed` are different failures: the second means the film moved on without it,+ # so its timing describes a different session than every other arm's. Default slack is 0, right+ # for the quiet machine a measurement is taken on.path = write_payload(tmp_path, [cell([ran("settings", slot_missed = True)])])assert main(["--assert-liveness", str(path)]) == 1def test_slack_excuses_a_missed_slot_and_only_up_to_the_number_given(tmp_path):- # A missed slot is a fact about the MACHINE. The scene is a fixed-duration film on the wall- # clock and is designed to roll on through one, precisely so a slow machine does not take a- # different path through a different-length session. On a two-core shared runner, failing on- # that makes the gate a speed test of the runner rather than a check that the harness works.+ # A missed slot is a fact about the MACHINE. The scene is a fixed-duration film on the wall clock+ # designed to roll on through one, precisely so a slow machine does not take a different path+ # through a different-length session. On a two-core shared runner, failing on that makes the gate+ # a speed test of the runner.one = write_payload(tmp_path / "a", [cell([ran("settings", slot_missed = True)])])assert main(["--assert-liveness", str(one), "--allow-slot-misses", "1"]) == 0two = write_payload(@@ -84,8 +84,8 @@ def test_slack_excuses_a_missed_slot_and_only_up_to_the_number_given(tmp_path):def test_slack_never_excuses_a_scene_problem(tmp_path):- # The distinction the whole split rests on. An action that was never planned, or whose button- # was not there, is the harness lying, and no amount of machine slack makes that acceptable.+ # The distinction the whole split rests on: an action that was never planned, or whose button was+ # not there, is the harness lying, and no amount of machine slack makes that acceptable.path = write_payload(tmp_path,[cell([{"action": "message_menu", "ran": False, "reason": "no More button in the DOM"}])],@@ -94,9 +94,8 @@ def test_slack_never_excuses_a_scene_problem(tmp_path):def test_an_action_that_missed_its_slot_is_not_also_counted_as_a_scene_problem(tmp_path):- # A missed slot is recorded as `ran: False` WITH `slot_missed: True`, so an implementation- # that checks `ran` first would file every missed slot under the category that slack cannot- # excuse, and the flag would do nothing at all.+ # A missed slot is recorded as `ran: False` WITH `slot_missed: True`, so an implementation that+ # checks `ran` first would file every missed slot under the category slack cannot excuse.path = write_payload(tmp_path,[@@ -118,8 +117,8 @@ def test_an_action_that_missed_its_slot_is_not_also_counted_as_a_scene_problem(tdef test_a_tolerated_miss_still_says_the_run_is_not_quotable(tmp_path, capsys):- # Exit 0 here claims only that the harness was not the cause. The payload still has a hole in- # it, and saying so is the difference between tolerating a miss and hiding one.+ # Exit 0 here claims only that the harness was not the cause. The payload still has a hole in it,+ # and saying so is the difference between tolerating a miss and hiding one.path = write_payload(tmp_path, [cell([ran("settings", slot_missed = True)])])assert main(["--assert-liveness", str(path), "--allow-slot-misses", "1"]) == 0assert "Do not quote a number from this payload" in capsys.readouterr().out@@ -131,11 +130,11 @@ def test_negative_slack_is_treated_as_none(tmp_path):def test_a_not_run_allowance_does_not_swallow_a_missed_slot(tmp_path):- # Where the two rules above meet. `--allow-not-run` excuses an action the fixture cannot mount- # at all, and a missed slot is a fact about the machine, so the slot is classified FIRST and- # the allowance never reaches it. Checked the other way round, a listed name would vanish from- # both buckets: not a scene problem, and not a missed slot either, so `--allow-slot-misses`- # would silently stop counting exactly the actions most likely to overrun.+ # Where the two rules above meet. `--allow-not-run` excuses an action the fixture cannot mount at+ # all, and a missed slot is a fact about the machine, so the slot is classified FIRST and the+ # allowance never reaches it. Checked the other way round, a listed name would vanish from both+ # buckets and `--allow-slot-misses` would silently stop counting exactly the actions most likely+ # to overrun.path = write_payload(tmp_path,[cell([{"action": "image_upload", "ran": False, "slot_missed": True, "reason": "late"}])],@@ -188,15 +187,15 @@ def test_an_action_whose_own_assertion_failed_fails(tmp_path):def test_an_action_that_passed_its_assertion_still_passes(tmp_path):- # The pair for the test above: `expect_ok = True` is the healthy recording, and a gate that- # failed on it would be unusable rather than strict.+ # The pair for the test above: `expect_ok = True` is the healthy recording, and a gate that failed+ # on it would be unusable rather than strict.path = write_payload(tmp_path, [cell([ran("keystroke", expect_ok = True)])])assert main(["--assert-liveness", str(path)]) == 0def test_an_unattempted_action_reports_not_run_rather_than_its_assertion(tmp_path):- # `ActionResult.__post_init__` forces `expect_ok = None` when `ran` is False, so the two- # failures never blur: the reader is told the action never happened, not that it misbehaved.+ # `ActionResult.__post_init__` forces `expect_ok = None` when `ran` is False, so the reader is+ # told the action never happened, not that it misbehaved.path = write_payload(tmp_path,[cell([{"action": "message_menu", "ran": False, "expect_ok": None, "reason": "no slot"}])],@@ -205,9 +204,8 @@ def test_an_unattempted_action_reports_not_run_rather_than_its_assertion(tmp_patdef test_a_failed_assertion_is_a_scene_problem_that_slack_cannot_excuse(tmp_path):- # The third bucket meeting the first. An assertion that failed says the surface is broken, not- # that the machine was slow, so it belongs with the scene problems and no amount of- # `--allow-slot-misses` may buy it a pass.+ # The third bucket meeting the first: an assertion that failed says the surface is broken, not+ # that the machine was slow, so it belongs with the scene problems.path = write_payload(tmp_path,[cell([ran("message_menu", expect_ok = False, reason = "the menu opened with no items")])],@@ -250,8 +248,8 @@ def test_non_cell_rows_are_ignored_but_do_not_count_as_cells(tmp_path):cell([ran("keystroke")]),],)- # The stray top-level action row must not fail the run: actions are read from inside their- # cell, so counting them twice would make the gate depend on payload layout.+ # The stray top-level action row must not fail the run: actions are read from inside their cell,+ # so counting them twice would make the gate depend on payload layout.assert main(["--assert-liveness", str(path)]) == 0@@ -264,7 +262,6 @@ def test_blank_lines_are_skipped(tmp_path, blank):# ── a cell that was re-run is judged on the run that finished it ─────-NOW = "s-now"OLD = "s-before"@@ -322,8 +319,8 @@ def test_the_superseded_attempt_does_not_count_as_a_second_cell(tmp_path):assert main(["--assert-liveness", str(path)]) == 0finally:m._log = real- # The summary names scene problems and missed slots apart on this branch, so the count is- # asserted on its own rather than against one spelling of the rest of the line.+ # The summary names scene problems and missed slots apart on this branch, so the count is asserted+ # on its own rather than against one spelling of the rest of the line.assert any("1 cell(s)" in line and "0 scene problem(s)" in line for line in logged), loggeddiff --git a/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py b/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.pyindex 863572fb2e3..84e56a9f809 100644--- a/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py+++ b/tests/studio/studiobench/report/selftest/test_studiobench_overhead_layoutcost.py@@ -35,8 +35,9 @@ from studiobench.report.overhead import ( # noqa: E402from studiobench.scoring import Measure # noqa: E402-# ---------------------------------------------------------------------------------------# the layout-cost adapter++# ---------------------------------------------------------------------------------------@@ -110,8 +111,9 @@ def test_layoutcost_js_parses():assert result.returncode == 0, result.stderr-# ---------------------------------------------------------------------------------------# the overhead gate++# ---------------------------------------------------------------------------------------diff --git a/tests/studio/studiobench/report/selftest/test_studiobench_report.py b/tests/studio/studiobench/report/selftest/test_studiobench_report.pyindex 337b0b2ab64..6041594a0d6 100644--- a/tests/studio/studiobench/report/selftest/test_studiobench_report.py+++ b/tests/studio/studiobench/report/selftest/test_studiobench_report.py@@ -54,8 +54,9 @@ def _metrics(keystroke_ms: float = 25.0) -> dict[str, Measure]:}-# ---------------------------------------------------------------------------------------# incremental payload++# ---------------------------------------------------------------------------------------@@ -151,8 +152,9 @@ def test_an_empty_excluded_block_is_still_printed_as_a_claim():render_excluded({"excluded_cells": None})-# ---------------------------------------------------------------------------------------# editorial policy++# ---------------------------------------------------------------------------------------diff --git a/tests/studio/studiobench/runtime/ab.py b/tests/studio/studiobench/runtime/ab.pyindex b107ad55f27..fbfad2a5abc 100644--- a/tests/studio/studiobench/runtime/ab.py+++ b/tests/studio/studiobench/runtime/ab.py@@ -197,27 +197,23 @@ def order_is_balanced(plan: list[tuple[Target, Cell, RungPlan]]) -> bool:continueseen.add(key)first_counts[target.label] += 1- # Every label is seeded at zero first. Counting only the labels that DID run first reports a- # single-rep plan -- where one side always goes first and nothing cancels -- as balanced,- # which is the one answer this function exists to prevent.+ # Every label is seeded at zero first. Counting only the labels that DID run reports a single-rep+ # plan, where one side always goes first and nothing cancels, as balanced, which is the one+ # answer this function exists to prevent.return len(labels) > 1 and len(set(first_counts.values())) == 1#: The per-cell gates whose failure means the cell's TIMINGS ARE NOT A READING OF THE BUILD, and#: therefore the only ones that may take the whole cell out of the ratios.-#:-#: NAMED RATHER THAN "ANY FAILED GATE", because a per-cell gate is not automatically fatal and the-#: one that is not says so itself. `timer_clamp` fails whenever idle calibration cannot establish a-#: floor -- an overloaded machine, or simply the frames instrument not being loaded -- and-#: `session.py` is explicit that this is "NOT fatal, and NOT silently zero": blocked time is a-#: subtraction against that floor, so `busy_pct` is null with the reason attached AND EVERY OTHER-#: COLUMN STANDS. Excluding the cell for it would delete keystroke latency, frame and census-#: readings that were measured correctly, and would do it most often on exactly the machines least-#: able to spare a repetition.-#:+#: NAMED RATHER THAN 'ANY FAILED GATE', because a per-cell gate is not automatically fatal.+#: `timer_clamp` fails whenever idle calibration cannot establish a floor, and `session.py` is+#: explicit that this is 'NOT fatal, and NOT silently zero': `busy_pct` is null with the reason+#: attached AND EVERY OTHER COLUMN STANDS. Excluding the cell would delete keystroke latency,+#: frame and census readings that were measured correctly, most often on the machines least able+#: to spare a repetition.#: The two below are different in kind: both say the FILM ITSELF was wrong. A thread that lost-#: messages and a reply that stopped being rendered do not produce a suspect column, they produce a-#: cheaper cell, and there is no metric in it that can be trusted afterwards.+#: messages and a reply that stopped being rendered produce a cheaper cell, not a suspect column,+#: and no metric in it can be trusted afterwards.INVALIDATING_CELL_GATES: frozenset[str] = frozenset({"thread_complete", "follows_the_stream"})@@ -304,10 +300,9 @@ def failed_invalidating_gates(records: Sequence[Mapping[str, Any]]) -> dict[str,if keep is not None and row.get("session_id") not in (None, keep):continuedetail = row.get("detail") if isinstance(row.get("detail"), dict) else {}- # NOT MEASURED IS NOT FAILED. See `gate_detail_is_unmeasured`, which is shared with- # `sweep/ui_parity.py` so the two admission lists cannot drift apart on it. Readiness now- # refuses a cell with no thread viewport outright, so that narrowing is the second of two- # doors on the same hole.+ # NOT MEASURED IS NOT FAILED. See `gate_detail_is_unmeasured`, shared with `sweep/ui_parity.py` so+ # the two admission lists cannot drift. Readiness now refuses a cell with no thread viewport+ # outright, so that narrowing is the second of two doors on the same hole.if gate_detail_is_unmeasured(detail):continuewhy = detail.get("reason") or detail.get("coverage_reason") or "the cell's own self-check"@@ -339,14 +334,13 @@ def unmeasured_planned_cells("""from ..scoring.from_payload import latest_attempt_rows- # THE SAME TWO FILTERS `readings_by_arm` APPLIES, because this function exists to notice the- # holes that one punches. It drops a cell for `completed is not True` AND for a failed- # invalidating gate; reading only the first left the second kind of hole invisible. A cell that- # completed but lost its thread's middle, or whose reply stopped being rendered, is removed- # from the ratios here and takes its healthy partner with it through the arm intersection in- # `compare_arms`, while this said the plan was whole -- so `ab.md` published a verdict over- # the rungs that survived instead of the VOID that is the point of the guard. That is the- # partial-plan selection bias, arriving by the gate road instead of the crash road.+ # THE SAME TWO FILTERS `readings_by_arm` APPLIES, because this function exists to notice the holes+ # that one punches: it drops a cell for `completed is not True` AND for a failed invalidating+ # gate, and reading only the first left the second kind invisible. A cell that completed but lost+ # its thread's middle is removed from the ratios and takes its healthy partner with it through+ # the arm intersection, while this said the plan was whole, so `ab.md` published a verdict over+ # the surviving rungs instead of the VOID that is the point of the guard.+ # The intersection is `compare_arms`.failed = failed_invalidating_gates(records)complete: set = set()for row in latest_attempt_rows(records):@@ -400,8 +394,8 @@ def readings_by_arm("""from ..scoring.from_payload import latest_attempt_rows, measures_by_cell- # The session filter below scopes the CELL rows, but `action` and `window` rows are collected- # by `cell_id` alone, and a resumed retry reuses the cell id of the attempt that died. Without+ # The session filter below scopes the CELL rows, but `action` and `window` rows are collected by+ # `cell_id` alone and a resumed retry reuses the cell id of the attempt that died, so without# this the completed-cell filter admitted the dead attempt's windows into the retry's reading.records = list(latest_attempt_rows(records))failed_gates = failed_invalidating_gates(records)@@ -460,10 +454,9 @@ def compare_arms(weights_id = weights_id() if callable(weights_id) else str(weights_id),session_id = session_id,)- # Paired PER REPETITION, matching (rung, rep) on both sides. Repetition r of the base and- # repetition r of the treatment ran adjacent in time, so pairing them is what makes the- # comparison paired at all; pooling reps into one reading per rung throws away every- # observation but the first and leaves the bootstrap with nothing to resample.+ # Paired PER REPETITION, matching (rung, rep) on both sides: repetition r of each arm ran adjacent+ # in time, which is what makes the comparison paired at all. Pooling reps into one reading per+ # rung throws away every observation but the first and leaves the bootstrap nothing to resample.pairs = []for key in sorted(set(base) & set(treatment)):rung, _rep = keydiff --git a/tests/studio/studiobench/runtime/browser.py b/tests/studio/studiobench/runtime/browser.pyindex 7b327a6f493..4a2d223329d 100644--- a/tests/studio/studiobench/runtime/browser.py+++ b/tests/studio/studiobench/runtime/browser.py@@ -23,12 +23,11 @@ from pathlib import Pathfrom typing import Any, Callable, Optional-# The engine matching the tester's desktop webview family. Unsloth ships as a Tauri app, so the-# thing a user actually looks at is a system webview, not the browser they happen to have.-# Windows -> WebView2, which is Chromium: `channel="msedge"` is the closest available.-# macOS -> WKWebView, which is WebKit.-# Linux -> WebKitGTK. Playwright's `webkit` is NOT WebKitGTK; it is a different embedding of-# the same engine, so it is labelled A PROXY and never as the real thing.+# The engine matching the tester's desktop webview family. Unsloth ships as a Tauri app, so what a+# user looks at is a system webview, not the browser they happen to have: Windows -> WebView2,+# which is Chromium, so `channel="msedge"` is the closest available; macOS -> WKWebView, which is+# WebKit; Linux -> WebKitGTK, which Playwright's `webkit` is NOT (a different embedding of the same+# engine), so it is labelled A PROXY.def default_engine() -> tuple[str, dict, str]:system = platform.system()if system == "Windows":@@ -118,9 +117,9 @@ def launch(browser = factory.launch(headless = headless)context = browser.new_context(viewport = {"width": viewport[0], "height": viewport[1]},- # Clipboard read is what proves the Copy action actually copied. Without it the action- # still runs and the assertion reports that it could not be proved, which is the honest- # outcome rather than a pass.+ # Clipboard read is what proves the Copy action actually copied. Without it the action still runs+ # and the assertion reports that it could not be proved, which is the honest outcome rather than a+ # pass.permissions = ["clipboard-read", "clipboard-write"] if chosen == "chromium" else None,)if robust is not None:@@ -131,11 +130,11 @@ def launch(for script in init_scripts or []:context.add_init_script(script)page = context.new_page()- # A GLOBAL CEILING on Playwright's actionability waits. The default is 30 seconds, which is- # longer than most of this suite's slot budgets: one action waiting out the default on an- # element that is present but not visible overran a 9-second slot by more than three times,- # and the scheduler cannot intervene because the action still holds its own window. Actions- # that legitimately need longer pass an explicit timeout.+ # A GLOBAL CEILING on Playwright's actionability waits. The default is 30 seconds, longer than most+ # of this suite's slot budgets: one action waiting out the default on an element that is present+ # but not visible overran a 9-second slot by more than three times, and the scheduler cannot+ # intervene because the action still holds its own window. Actions that legitimately need longer+ # pass an explicit timeout.page.set_default_timeout(8000)cdp = Noneif chosen == "chromium":diff --git a/tests/studio/studiobench/runtime/bundle_guard.py b/tests/studio/studiobench/runtime/bundle_guard.pyindex 2125f724020..31ccf54b7f1 100644--- a/tests/studio/studiobench/runtime/bundle_guard.py+++ b/tests/studio/studiobench/runtime/bundle_guard.py@@ -39,10 +39,10 @@ from typing import Optional_WINDOW = 4000# BACKTICKS as well as quotes. Unsloth's production bundle is minified with a pass that rewrites-# short string literals as template literals, so the marker on the shipped build reads-# ``rendererPackageName:`react-dom` `` and a quote-only pattern misses it on every asset -- which-# reads as "build mode could not be established" on a perfectly good production build, i.e. the-# gate fails exactly where it is supposed to pass. Verified against the shipped bundle.+# short string literals as template literals, so the marker reads with backticks and a quote-only+# pattern misses it on every asset, which reads as "build mode could not be established" on a+# perfectly good production build. Verified against the shipped bundle.+# The marker reads ``rendererPackageName:`react-dom` ``._RENDERER_RE = re.compile(r'rendererPackageName\s*:\s*["\'`]react-dom["\'`]')_BUNDLETYPE_RE = re.compile(r"bundleType\s*:\s*([01])")@@ -66,8 +66,8 @@ class BundleVerdict:"entry_url": self.entry_url,"entry_bytes": self.entry_bytes,"checked": self.checked,- # Attempted flags, because "bundle_type is None" and "bundle_type is 0" are opposite- # findings and a bare null cannot say which.+ # Attempted flags, because "bundle_type is None" and "bundle_type is 0" are opposite findings and a+ # bare null cannot say which."bundle_type_attempted": self.entry_url is not None,"vite_probe_attempted": self.vite_client_status is not None,}@@ -115,8 +115,8 @@ def _entry_urls(base_url: str, html: str) -> list[str]:for src in re.findall(r'<script[^>]+src=["\']([^"\']+)["\']', html):url = src if src.startswith("http") else base_url.rstrip("/") + "/" + src.lstrip("/")out.append(url)- # A modulepreload is how Vite names the real entry chunk when the script tag points at a- # loader shim, so it is worth following when the script tags carry nothing.+ # A modulepreload is how Vite names the real entry chunk when the script tag points at a loader+ # shim, so it is worth following when the script tags carry nothing.for href in re.findall(r'<link[^>]+rel=["\']modulepreload["\'][^>]+href=["\']([^"\']+)["\']', html):@@ -182,9 +182,8 @@ def check_bundle(base_url: str) -> BundleVerdict:if match is None:checked.append(f"{url}: no rendererPackageName marker, {len(raw)} bytes")continue- # The SAME chunk, and within a small window of the marker. A bundle this size carries- # several `bundleType` occurrences and picking the wrong one is picking a different- # package's answer to a question about react-dom.+ # The SAME chunk, and within a small window of the marker: a bundle this size carries several+ # `bundleType` occurrences and picking the wrong one is picking a different package's answer.lo = max(0, match.start() - _WINDOW)hi = min(len(text), match.end() + _WINDOW)found = _BUNDLETYPE_RE.search(text, lo, hi)diff --git a/tests/studio/studiobench/runtime/lifecycle.py b/tests/studio/studiobench/runtime/lifecycle.pyindex f0917111049..d690c304f1d 100644--- a/tests/studio/studiobench/runtime/lifecycle.py+++ b/tests/studio/studiobench/runtime/lifecycle.py@@ -44,9 +44,8 @@ class StudioInstall:bootstrap_password: Optional[str] = Noneport: Optional[int] = Nonepid: Optional[int] = None- #: The commit `branch` RESOLVED TO, which is the build that was installed. `branch` is what- #: the caller typed, and a branch or a movable tag is not a build. See- #: `__main__.commit_problems`.+ #: The commit `branch` RESOLVED TO, which is the build that was installed. `branch` is what the+ #: caller typed, and a branch or a movable tag is not a build. See `__main__.commit_problems`.commit: Optional[str] = None@property@@ -54,14 +53,14 @@ class StudioInstall:return f"http://127.0.0.1:{self.port}"-#: What the backend gives an access token: `auth/authentication.py`, ACCESS_TOKEN_EXPIRE_MINUTES.-#: Only a FALLBACK. The expiry actually enforced is the `exp` claim in the token this run was-#: handed, which is read from the token itself; this number is what to assume when the server-#: hands back something that is not a readable JWT.+#: What the backend gives an access token (`auth/authentication.py`,+#: ACCESS_TOKEN_EXPIRE_MINUTES). Only a FALLBACK: the expiry enforced is the `exp` claim read+#: from the token itself, and this is what to assume when the server hands back something that+#: is not a readable JWT.ACCESS_TOKEN_TTL_S = 60 * 60#: How far ahead of `exp` a token is replaced. Seeding a 1M-token thread is ONE request with a-#: 900 second timeout, so a token that is merely valid at the moment the request is written is not-#: good enough: it has to still be valid when the server finishes reading the body.+#: 900 second timeout, so a token merely valid when the request is written is not good enough:+#: it must still be valid when the server finishes reading the body.TOKEN_REFRESH_MARGIN_S = 15 * 60@@ -115,7 +114,7 @@ class StudioAuth:base_url: strusername: strpassword: str- #: Unix seconds, from the token's own `exp`. None means "unknown", treated as `ACCESS_TOKEN_TTL_S`+ #: Unix seconds, from the token's own `exp`. None means unknown, treated as `ACCESS_TOKEN_TTL_S`#: from the moment this object was built.expires_at: Optional[float] = None#: Called with `self` after the token is replaced. The browser context seeds its localStorage@@ -123,9 +122,9 @@ class StudioAuth:on_rotate: Optional[Callable[["StudioAuth"], None]] = Nonerotations: int = field(default = 0, init = False)#: Turned off the first time a FRESH token still reads as expiring, which means the `exp` this- #: process reads and the server's own clock do not agree. See `rotate`.+ #: process reads and the server's clock do not agree. See `rotate`.proactive: bool = field(default = True, init = False)- #: The last `on_rotate` failure, kept rather than raised. See `rotate`.+ #:The last `on_rotate` failure, kept rather than raised. See `rotate`.hook_error: Optional[str] = field(default = None, init = False)def __post_init__(self) -> None:@@ -281,9 +280,9 @@ def register_provider(base_url: str, auth: StudioAuth, provider: ProviderSeed) -"""existing = auth_request_json(auth, f"{base_url.rstrip('/')}/api/providers/") or []for row in existing:- # Idempotent across runs. Every run binds a NEW ephemeral pacer port, so a stale entry- # from a previous run points at a port nothing is listening on, and leaving it there gives- # the picker two identically named models of which one is dead.+ # Idempotent across runs. Every run binds a NEW ephemeral pacer port, so a stale entry points at+ # a port nothing is listening on and gives the picker two identically named models of which one+ # is dead.if row.get("display_name") == provider.name:try:auth_request_json(@@ -410,8 +409,8 @@ def checkout_ref(repo: Path, ref: str) -> str:"""fetched = _run(["git", "fetch", "--tags", "origin", ref], cwd = repo, check = False)if fetched.returncode != 0:- # A ref the remote will not serve by name (an old server, or `ref^1`, which is a local- # expression rather than something to ask for). Fetch everything and resolve it here.+ # A ref the remote will not serve by name (an old server, or `ref^1`, a local expression). Fetch+ # everything and resolve it here._run(["git", "fetch", "--tags", "origin"], cwd = repo, check = False)candidates = [] if fetched.returncode != 0 else ["FETCH_HEAD"]candidates += [f"origin/{ref}", ref]@@ -432,10 +431,9 @@ def checkout_ref(repo: Path, ref: str) -> str:)-#: What `install.sh` is allowed. A multi-gigabyte download and build, documented in the README as-#: "budgeted at up to 45 minutes" and explicitly NOT part of a tier's wall clock. Named rather-#: than inlined because the run's watchdog has to add it to a deadline it would otherwise fire in-#: the middle of: see `watchdog_deadline_s`.+#: What `install.sh` is allowed. A multi-gigabyte download and build, documented as 'budgeted at+#: up to 45 minutes' and explicitly NOT part of a tier's wall clock. Named rather than inlined+#: because the run's watchdog has to add it to a deadline it would otherwise fire inside.INSTALL_TIMEOUT_S = 60 * 45@@ -452,11 +450,11 @@ def install_studio(if not (reuse_clone and (repo / ".git").exists()):if repo.exists():shutil.rmtree(repo)- # Cloned WITHOUT `--branch`, then moved onto the ref locally, so one code path serves a- # branch, a tag and a commit sha instead of two that disagree about what a ref is.+ # Cloned WITHOUT `--branch`, then moved onto the ref locally, so one code path serves a branch,+ # a tag and a commit sha instead of two that disagree about what a ref is._run(["git", "clone", remote, str(repo)])- # KEPT, not discarded. `checkout_ref` is the only place that knows which commit a ref names,- # and the ref alone cannot tell a resumed run that `main` moved underneath it.+ # KEPT, not discarded: `checkout_ref` is the only place that knows which commit a ref names, and+ # the ref alone cannot tell a resumed run that `main` moved underneath it.commit = checkout_ref(repo, branch)install_sh = repo / "install.sh"if not install_sh.exists():@@ -507,8 +505,8 @@ def _read_bootstrap_password(home: Path, log_path: Path, deadline: float) -> Opt#: How long to keep looking for the launched server's pid. It appears once the server has forked-#: and exec'd, which is after `setsid -f` has already returned, so this is a poll rather than a-#: read. Named so a test can set it to zero instead of sleeping through it.+#: and exec'd, after `setsid -f` has returned, so this is a poll rather than a read. Named so a+#: test can set it to zero.PID_DISCOVERY_TIMEOUT_S = 15.0@@ -566,20 +564,18 @@ def launch_studio(healthz_timeout_s: int = 240,password_timeout_s: int = 30,) -> StudioInstall:- # AN OCCUPIED PORT IS REFUSED BEFORE ANYTHING IS LAUNCHED, and this is the half of the- # abandoned-server failure that no cleanup can reach. `--keep-studio` asks for an Unsloth to be- # LEFT RUNNING on this port, so the next run's `unsloth studio -p <port>` finds one of our own- # servers there and aborts rather than binding (`studio/backend/run.py`, `_resolve_port` with- # `avoid_own_studio`); when the pid record is not readable it falls back to the NEXT port- # instead. Either way nothing this run launched is on `port`.- #- # Everything downstream then agrees that the launch worked. `_discover_pid` pgreps for- # `unsloth studio.*-p <port>` and finds the OLD process; `wait_for_healthz` takes its 200 from- # it; and `authenticate` retries with `BENCH_PASSWORD`, which a previous studiobench run has- # already rotated that Unsloth to, so the login succeeds as well. The run then measures the- # build the PREVIOUS invocation installed while `run_meta` records the ref this one asked for,- # and `stop_studio` kills the server the caller asked to keep. There is no reading anywhere- # that says which build answered, so this is refused rather than reported.+ # AN OCCUPIED PORT IS REFUSED BEFORE ANYTHING IS LAUNCHED, the half of the abandoned-server+ # failure no cleanup can reach. `--keep-studio` leaves an Unsloth running on this port, so the+ # next run's `unsloth studio -p <port>` finds one of our own servers and aborts rather than+ # binding, or falls back to the NEXT port when the pid record is unreadable. Either way nothing+ # this run launched is on `port`.+ # `studio/backend/run.py::_resolve_port`, `avoid_own_studio`.+ # Everything downstream then agrees the launch worked: `_discover_pid` pgreps and finds the OLD+ # process, `wait_for_healthz` takes its 200 from it, and `authenticate` retries with+ # `BENCH_PASSWORD`, which a previous run has already rotated that Unsloth to. The run measures+ # the build the PREVIOUS invocation installed while `run_meta` records the ref this one asked+ # for, and `stop_studio` kills the server the caller asked to keep. No reading anywhere says+ # which build answered.if port_is_busy(port):holder = _discover_pid(port, 0.0)raise RuntimeError(@@ -595,7 +591,7 @@ def launch_studio(bin_path = _find_unsloth_bin(install)env = {"UNSLOTH_STUDIO_HOME": str(install.home), **(extra_env or {})}# NOT `UNSLOTH_STUDIO_BLOCK_PRIVATE_PROVIDER_URLS=1`: that opt-in SSRF guard rejects any- # non-global address, which is exactly what the pacer's 127.0.0.1 base URL is.+ # non-global address, which is exactly the pacer's 127.0.0.1 base URL.env.pop("UNSLOTH_STUDIO_BLOCK_PRIVATE_PROVIDER_URLS", None)cmd = ["setsid",@@ -615,19 +611,17 @@ def launch_studio(install.bootstrap_password = _read_bootstrap_password(install.home, log_path, time.time() + password_timeout_s)- # BEFORE the health check, not after it. An Unsloth that starts and stays unhealthy used to raise- # here with `install.pid` still None, and `stop_studio` has nothing to kill without it -- so the- # detached server was left running on the requested port while the CLI unwound. It is not idle- # there: the next attempt's `unsloth studio -p <port>` finds one of our own servers on the port- # and aborts rather than binding (`run.py:_resolve_port`, `avoid_own_studio`), while- # `wait_for_healthz` gets its 200 from the STALE process -- which by then may have finished- # starting. That run measures the build the previous attempt installed and records the ref this- # one asked for, which is the one failure this harness may never produce quietly.+ # BEFORE the health check, not after. An Unsloth that starts and stays unhealthy used to raise+ # here with `install.pid` still None, so `stop_studio` had nothing to kill and the detached+ # server was left on the requested port. The next attempt then aborts rather than binding while+ # `wait_for_healthz` gets its 200 from the STALE process, measuring the build the previous+ # attempt installed while recording the ref this one asked for: the one failure this harness may+ # never produce quietly.install.pid = _discover_pid(port)healthy = wait_for_healthz(install.base_url, healthz_timeout_s)if install.pid is None:- # One more look: a server slow enough to miss the window above is exactly the one whose- # health check just timed out, and it is the one that most needs to be terminated.+ # One more look: a server slow enough to miss the window above is exactly the one whose health+ # check just timed out, and the one that most needs terminating.install.pid = _discover_pid(port, 0.0)if not healthy:stop_studio(install)@@ -678,11 +672,10 @@ def authenticate(the run simply fails to seed a thread and reports an empty one. So the gate is read from`/api/auth/status` up front and cleared through the one endpoint that accepts the gated token."""- # Try the supplied password, then the one a PREVIOUS studiobench run rotated to. Unsloth mints- # a bootstrap password and demands it be changed; this function is what changes it, so the- # second run against the same home is handed a bootstrap password that no longer exists and- # gets a 401 whose message sends you to `reset-password`. Measured on the second run against- # a home the first run had already set up. Trying both makes reruns and `--resume` work.+ # Try the supplied password, then the one a PREVIOUS studiobench run rotated to. Unsloth mints a+ # bootstrap password and demands it be changed, and this function is what changes it, so the+ # second run against the same home is handed a password that no longer exists and gets a 401+ # pointing at `reset-password`. Trying both makes reruns and `--resume` work.attempts = [password, new_password] if password != new_password else [password]auth = Nonelast: Optional[Exception] = Nonediff --git a/tests/studio/studiobench/runtime/readiness.py b/tests/studio/studiobench/runtime/readiness.pyindex 7d37162444a..f298849bfac 100644--- a/tests/studio/studiobench/runtime/readiness.py+++ b/tests/studio/studiobench/runtime/readiness.py@@ -114,21 +114,21 @@ from dataclasses import dataclass, fieldfrom typing import Any, Callable, Optional#: The two gate modes. `full` is the historical behaviour plus the settle and end-present-#: conditions; it is STRICTLY STRONGER than what shipped and is what every normal arm runs.-#: `windowed` is the one an arm that deliberately mounts fewer nodes may ask for.+#: conditions, STRICTLY STRONGER than what shipped, and is what every normal arm runs.+#: `windowed` is for an arm that deliberately mounts fewer nodes.MODE_FULL = "full"MODE_WINDOWED = "windowed"MODES = (MODE_FULL, MODE_WINDOWED)-#: How far apart the two agreeing samples must be. 600ms rather than the old 500ms poll, because-#: the thing being ruled out is a mount loop that pauses for a frame, and a gap shorter than a-#: couple of React commits can be spanned by one.+#: How far apart the two agreeing samples must be. 600ms rather than the old 500ms poll,+#: because what is being ruled out is a mount loop that pauses for a frame, and a shorter gap+#: can be spanned by one.STABLE_GAP_MS = 600-#: How many consecutive agreeing samples are needed. Two, i.e. one confirmed repeat.+#:How many consecutive agreeing samples are needed. Two, i.e. one confirmed repeat.STABLE_SAMPLES = 2-#: Bottom tolerance, in CSS pixels. Not zero: a virtualised list settles on an estimated total-#: height and corrects it as real rows are measured, so it lands a few pixels off the exact bottom-#: and stays there. 24px is under one line of text, so it cannot hide a missing message.+#: Bottom tolerance in CSS pixels. Not zero: a virtualised list settles on an estimated total+#: height and lands a few pixels off. 24px is under one line of text, so it cannot hide a+#: missing message.BOTTOM_TOLERANCE_PX = 24DEFAULT_TIMEOUT_S = 180@@ -146,8 +146,8 @@ class ThreadNotReady(TimeoutError):self.detail = detail-#: One reading of the page. Cheap enough to run every poll: the expensive part is-#: `getElementsByTagName('*').length`, which is a live HTMLCollection length and not a walk.+#: One reading of the page. Cheap enough to run every poll:+#: `getElementsByTagName('*').length` is a live HTMLCollection length, not a walk.PROBE_JS = """(args) => {const [marker, tailChars] = args;@@ -241,10 +241,9 @@ PROBE_JS = """}"""-#: The keys that must AGREE between two samples for the page to count as settled. Deliberately-#: three quantities that move for three different reasons: the message list growing, any subtree-#: growing (a highlighter still colouring a fence moves `elements` while `mounted` sits still), and-#: the laid-out height changing (a virtualised list correcting its estimated row heights).+#: The keys that must AGREE between two samples for the page to count as settled. Three+#: quantities that move for three different reasons: the message list growing, any subtree+#: growing (a highlighter still colouring a fence), and the laid-out height changing.SETTLE_KEYS = ("mounted", "elements", "scroll_height")@@ -296,27 +295,20 @@ def evaluate(probe: dict, previous: Optional[dict], expected_messages: int, mode"composer_present": bool(probe.get("composer")),"any_message_mounted": (probe.get("mounted") or 0) > 0,"settled": bool(settled),- # The END of the thread is on screen. Both halves matter: the marker for the last user turn- # is mounted, AND it is near the end of the mounted run rather than in the middle of it.- # `2` because the thread ends user, assistant -- the marker's own row plus the reply.+ # The END of the thread is on screen: the last user turn's marker is mounted AND near the end+ # of the mounted run rather than in the middle. `2` because the thread ends user, assistant."end_present": bool(probe.get("marker_found"))and (probe.get("marker_from_end") is not None and probe["marker_from_end"] <= 2),}if mode == MODE_WINDOWED:- # THE VIRTUALIZER'S OWN CLAIM about the thread's length, and it has to be the truth the- # seeder wrote. An arm that mounts a window without publishing aria-setsize has not given- # anyone -- this harness or a screen reader -- any way to know how long the thread is, and- # it is refused here rather than scored on the assumption that it is fine.- #- # UNLESS THE WHOLE THREAD IS MOUNTED, in which case there is nothing left to declare: the- # DOM is the total. This matters at the small rungs, where the thread is shorter than the- # window and a virtualised build mounts every message exactly like the shipped one. The- # requirement has to be true of a fully-mounted thread as well as of a correctly windowed- # one, or `windowed` becomes a mode that only some rungs of the same arm can pass.- #- # It cannot be used to slip past the gate: this branch needs `mounted >= expected`, which- # is the full-mount condition itself, so a thread that has mounted 9 of 18 gets none of it.+ # THE VIRTUALIZER'S OWN CLAIM about the thread's length, which has to be the truth the seeder+ # wrote. An arm that windows without publishing aria-setsize has given nobody, this harness or+ # a screen reader, a way to know how long the thread is.+ # UNLESS THE WHOLE THREAD IS MOUNTED, where the DOM is the total. This matters at small rungs,+ # where a virtualised build mounts every message like the shipped one, or `windowed` becomes a+ # mode only some rungs of an arm can pass. It cannot be used to slip past the gate: this+ # branch needs `mounted >= expected`, the full-mount condition itself.fully_mounted = (probe.get("mounted") or 0) >= expected_messagesout["total_declared"] = fully_mounted or probe.get("setsize") is not Noneout["total_matches_seeded"] = fully_mounted or probe.get("setsize") == expected_messages@@ -325,22 +317,16 @@ def evaluate(probe: dict, previous: Optional[dict], expected_messages: int, modeprobe.get("posinset_count") == probe.get("mounted") and (probe.get("mounted") or 0) > 0)# ORDINALS THAT ARE ACTUALLY POSITIONS. The condition above only asks whether a number was- # published on every row, and three malformed shapes satisfy it while carrying no- # information at all: every row publishing `0`, every row publishing the same ordinal, and- # a window numbered from 1 rather than from where it sits in the thread. Each is a real- # virtualizer bug, each leaves the mounted set unlocatable, and each used to pass here.- #- # So the numbers have to behave like positions in a set: at least 1, no two rows claiming- # the same one, and none pointing past the end those same rows declare in `aria-setsize`.- # `setsize` is the arm's own claim and is already required to equal the seeded total by- # `total_matches_seeded`; the seeded total stands in when no setsize was published, which- # is the fully-mounted case below.- #- # THE WAIVER, and it is the same one `total_declared` carries: a thread short enough to be- # mounted whole publishes no ordinals -- the shipped build publishes none anywhere -- and- # there is nothing to validate. It needs `mounted >= expected`, the full-mount condition- # itself, AND no ordinals at all, so an arm that publishes malformed ones cannot buy its- # way out of the check by also mounting everything.+ # published, and three malformed shapes satisfy it carrying no information: every row+ # publishing `0`, every row publishing the same ordinal, and a window numbered from 1 rather+ # than from where it sits. Each is a real virtualizer bug and each used to pass here.+ # So the numbers must behave like positions in a set: at least 1, no duplicates, and none past+ # the end those rows declare in `aria-setsize` (already required to equal the seeded total by+ # `total_matches_seeded`; the seeded total stands in when none was published).+ # THE WAIVER, the same one `total_declared` carries: a thread mounted whole publishes no+ # ordinals, and the shipped build publishes none anywhere, so there is nothing to validate. It+ # needs `mounted >= expected` AND no ordinals at all, so an arm publishing malformed ones+ # cannot buy its way out by also mounting everything.declared = probe.get("setsize")if declared is None:declared = expected_messages@@ -353,49 +339,41 @@ def evaluate(probe: dict, previous: Optional[dict], expected_messages: int, modeand probe["max_posinset"] <= declared)# AND THE WINDOW IS AT THE END OF THE THREAD BY ITS OWN NUMBERING. `end_present` proves the- # last message's TEXT is mounted; this proves the ordinals agree with it. Without it a- # bottom-anchored window numbered 1..6 out of 18 -- a virtualizer publishing the index- # within the window instead of the position in the thread -- passes every other condition.+ # last message's TEXT is mounted; this proves the ordinals agree. Without it a bottom-anchored+ # window numbered 1..6 out of 18 passes every other condition.out["posinset_reaches_end"] = (fully_mounted and published == 0) or (probe.get("max_posinset") == expected_messages)# THE APP'S ANSWER FIRST, the arithmetic only when the app has not given one. A virtualised- # list settles on an estimated total height and corrects it as real rows are measured, so- # its scrollTop arithmetic can read tens of pixels off the bottom while the app -- which- # knows it is pinned -- is perfectly happy. Trusting the arithmetic over the app would fail- # a correct arm for a rounding error in a height estimate.+ # list's scrollTop arithmetic can read tens of pixels off the bottom while the app, which+ # knows it is pinned, is happy; trusting the arithmetic would fail a correct arm for a+ # rounding error.app_bottom = probe.get("app_says_at_bottom")near_bottom = (probe.get("from_bottom") is not None and probe["from_bottom"] <= BOTTOM_TOLERANCE_PX)out["anchored_at_end"] = bool(app_bottom) if app_bottom is not None else near_bottom- # THE VIEWPORT ITSELF, ASSERTED RATHER THAN INFERRED. Every windowed condition above is- # about what is inside the scroller, and each one degrades to a pass when the scroller is- # not there: `from_bottom` is null so the arithmetic is skipped, and the app's own answer- # is read off `.aui-thread-scroll-to-bottom`, which is a DESCENDANT of the viewport but is- # looked up at DOCUMENT scope. So renaming the viewport class while leaving that control- # mounted leaves `app_says_at_bottom` true, `anchored_at_end` true, and the cell admitted- # with no viewport at all.- #- # What follows a cell admitted that way is silent in every direction: the completeness- # probe returns `probe_attempted: false`, the scroll actions return `not_run` and a not-run- # action blanks only its own timings, and the census viewport fields go null. Nothing- # refuses, so the film is scored without the surface it was measuring.- #- # The signal is already probed and costs nothing to read. It is asserted only here because- # a fully mounted arm that scrolls the window instead of a div is a shape this harness- # supports, and `MODE_FULL` must not start requiring a scroller it never needed.+ # THE VIEWPORT ITSELF, ASSERTED RATHER THAN INFERRED. Every windowed condition above degrades+ # to a pass without the scroller: `from_bottom` is null so the arithmetic is skipped, and the+ # app's own answer is read off `.aui-thread-scroll-to-bottom`, a DESCENDANT of the viewport+ # looked up at DOCUMENT scope. So renaming the viewport class admits a cell with no viewport.+ # What follows is silent in every direction: the completeness probe returns+ # `probe_attempted: false`, scroll actions return `not_run` and blank only their own timings,+ # and the census viewport fields go null. Nothing refuses, so the film is scored without the+ # surface it was measuring.+ # `app_says_at_bottom` and `anchored_at_end` both read true on such a cell.+ # Asserted only here because a fully mounted arm that scrolls the window instead of a div is a+ # shape this harness supports, and `MODE_FULL` must not start requiring a scroller.out["viewport_present"] = bool(probe.get("viewport_present"))- # And the pin must have FINISHED. `--aui-scroll-stabilizer` is on the viewport while- # use-intent-aware-autoscroll is still pinning and comes off when it settles, so a page- # carrying it is mid-scroll no matter what its scrollTop says.+ # And the pin must have FINISHED: `--aui-scroll-stabilizer` is on the viewport while+ # use-intent-aware-autoscroll is still pinning, so a page carrying it is mid-scroll whatever+ # its scrollTop says.out["pin_settled"] = not probe.get("pinning")- # Not a gate, a sanity check with teeth: a "windowed" arm that mounted MORE rows than the- # thread has is not windowed, it is broken, and the reading would be nonsense.+ # Not a gate, a sanity check with teeth: a 'windowed' arm that mounted MORE rows than the+ # thread has is broken, and the reading would be nonsense.out["not_over_mounted"] = (probe.get("mounted") or 0) <= expected_messageselse:out["all_messages_mounted"] = (probe.get("mounted") or 0) >= expected_messages- # Recorded, never gated, in full mode. See the module docstring.out["total_declared"] = Noneout["total_matches_seeded"] = Noneout["posinset_on_every_row"] = None@@ -412,9 +390,8 @@ def _describe(conditions: dict, probe: dict, expected: int, mode: str) -> str:bits = [f"mounted {probe.get('mounted')} of {expected}",f"aria-setsize {probe.get('setsize')}",- # The ordinals THEMSELVES, not just how many rows carried one: "posinset 0..0 on 6 rows,- # 1 distinct" is the whole diagnosis of a malformed contract, and reading it off the raw- # probe afterwards is what this message exists to save.+ # The ordinals THEMSELVES, not just how many rows carried one: 'posinset 0..0 on 6 rows, 1+ # distinct' is the whole diagnosis of a malformed contract.f"aria-posinset {probe.get('min_posinset')}..{probe.get('max_posinset')} on "f"{probe.get('posinset_count')} of {probe.get('mounted')} rows, "f"{probe.get('posinset_distinct')} distinct",@@ -605,15 +582,14 @@ async ([toTop, steps, stepPx]) => {}"""-#: How the traversal is stepped. 400 steps of 2,000px covers 800,000px of thread, which clears the-#: tallest rung this benchmark runs, and a step that lands at either end breaks out early.+#: How the traversal is stepped. 400 steps of 2,000px covers 800,000px, clearing the tallest+#: rung, and a step landing at either end breaks out early.TRAVERSE_STEPS = 400TRAVERSE_STEP_PX = 2000-#: The ordinals mounted right now, read exactly the way PROBE_JS reads them. Run once at the top of-#: the thread, after the wait for the head marker: the last stop of the gesture reads its rows the-#: instant the paint lands, and an arm that materialises a row a beat later would otherwise have-#: that row counted as one the traversal never saw.+#: The ordinals mounted right now, read exactly as PROBE_JS reads them. Run once at the top+#: after the wait for the head marker: the gesture's last stop reads its rows the instant the+#: paint lands, so an arm materialising a row a beat later would be counted as never seen.COLLECT_ORDINALS_JS = """() => {const out = [];@@ -627,30 +603,26 @@ COLLECT_ORDINALS_JS = """}"""-#: How many missing ordinals a verdict names. The COUNT is always exact; the list is capped so a-#: thread that lost half of itself does not write a thousand integers into every payload.+#: How many missing ordinals a verdict names. The COUNT is always exact; the list is capped so+#: a thread that lost half of itself does not write a thousand integers into every payload.MISSING_ORDINALS_LISTED = 40-#: WHY `ordinal_coverage` GAVE THE ANSWER IT GAVE, which the three-valued verdict on its own cannot-#: say and which the `thread_complete` gate has to know.-#:-#: `not_applicable` and `unmeasured` are BOTH `None` and they are opposites. The first is a question-#: that does not arise on this arm: a fully mounted thread publishes no `aria-posinset` anywhere ---#: the shipped build publishes none -- so there are no ordinals to cover and none missing. The-#: second is a question that does arise and was not answered: the gesture stopped short of the top,-#: or its consecutive stops did not overlap, so rows nobody scrolled past are rows nobody looked at.-#:-#: A gate that passes both reads "we could not tell" as "it was fine", which is the store that kept-#: only its first and last page staying scoreable. A gate that fails both fails the shipped build on-#: every cell for publishing no ordinals. So the two are recorded apart, and only the second is-#: withheld from a pass.+#: WHY `ordinal_coverage` GAVE THE ANSWER IT GAVE, which the three-valued verdict cannot say+#: and the `thread_complete` gate has to know.+#: `not_applicable` and `unmeasured` are BOTH `None` and are opposites. The first is a question+#: that does not arise: a fully mounted thread publishes no `aria-posinset`, so nothing is+#: missing. The second is a question that arose and was not answered: the gesture stopped short+#: of the top, or its stops did not overlap.+#: A gate passing both reads 'we could not tell' as 'it was fine', which is the store that kept+#: only its first and last page staying scoreable; a gate failing both fails the shipped build+#: on every cell. So the two are recorded apart and only the second is withheld from a pass.COVERAGE_COMPLETE = "complete"COVERAGE_INCOMPLETE = "incomplete"COVERAGE_NOT_APPLICABLE = "not_applicable"COVERAGE_UNMEASURED = "unmeasured"#: The coverage states a cell may still be scored on. `unmeasured` is deliberately absent, and-#: `incomplete` is excluded by the verdict itself rather than by this tuple.+#: `incomplete` is excluded by the verdict itself.COVERAGE_STATES_SCOREABLE = (COVERAGE_COMPLETE, COVERAGE_NOT_APPLICABLE)@@ -703,8 +675,8 @@ def ordinal_coverage(seen.update(int(n) for n in (extra_seen or ()))expected = set(range(1, expected_messages + 1)) if expected_messages > 0 else set()missing = sorted(expected - seen)- # Intersected with "never seen anywhere", so a row that was briefly absent while its window- # was still materialising and mounted at the next stop is not reported as a lost message.+ # Intersected with 'never seen anywhere', so a row briefly absent while its window materialised+ # is not reported as a lost message.holes = sorted({int(n) for n in (traverse.get("ordinals_in_window_holes") or [])} - seen)out: dict[str, Any] = {"ordinals_seen_count": len(seen),@@ -718,9 +690,9 @@ def ordinal_coverage("traversal_stops": traverse.get("traversal_stops"),"coverage_reason": None,}- # NOT APPLICABLE BEFORE UNMEASURED. An arm publishing no ordinals anywhere has nothing to cover- # whatever the gesture did, and asking about the gesture first would report the shipped build's- # own shape as a measurement this probe failed to take -- which the gate now refuses to score.+ # NOT APPLICABLE BEFORE UNMEASURED. An arm publishing no ordinals has nothing to cover whatever+ # the gesture did, and asking about the gesture first would report the shipped build's own+ # shape as a failed measurement.if not seen:out["ordinal_coverage_complete"] = Noneout["ordinal_coverage_state"] = COVERAGE_NOT_APPLICABLE@@ -827,25 +799,24 @@ def probe_thread_completeness("mounted_at_top": seen.get("mounted"),"setsize_at_top": seen.get("setsize"),"scroll_height_at_top": top.get("scroll_height"),- # DID THE GESTURE ACTUALLY REACH THE TOP? Without this, "the head never mounted" and- # "the viewport never left the bottom" are the same reading, and the second one is a- # defect in this probe being reported as data loss in the app.+ # DID THE GESTURE ACTUALLY REACH THE TOP? Without this, 'the head never mounted' and 'the+ # viewport never left the bottom' read the same, and the second is a defect in this probe+ # reported as data loss in the app."reached_top": top.get("reached_target"),"scroll_top_after_gesture": top.get("scroll_top"),"traverse_step_px": step_px,})out.update(coverage)- # `min_posinset_seen` before there were any ordinals to see: the head marker mounting means- # position 1 was on the page, whether or not the arm numbers its rows. Kept so an arm that- # publishes nothing still fills the field the way it always did.+ # `min_posinset_seen` before there were ordinals to see: the head marker mounting means+ # position 1 was on the page, whether or not the arm numbers its rows.if out.get("min_posinset_seen") is None and found:out["min_posinset_seen"] = 1# And back to the end, so the cell resumes from the state the readiness gate described.page.evaluate(TRAVERSE_JS, [False, steps, step_px])if not found and not top.get("reached_target"):- # NOT A VERDICT ABOUT THE ARM. The gesture did not get there, so nothing was learned about- # what the arm holds, and saying otherwise would blame the app for the probe.+ # NOT A VERDICT ABOUT THE ARM: the gesture did not get there, so nothing was learned, and+ # saying otherwise would blame the app for the probe.out["head_reached"] = Noneout["reason"] = (f"the scroll gesture never reached the top of the thread (stopped at "@@ -860,14 +831,14 @@ def probe_thread_completeness()log(f" COMPLETENESS FAILED: {out['reason']}")elif out.get("ordinal_coverage_complete") is False:- # THE HEAD ARRIVED AND THE THREAD IS STILL INCOMPLETE. This is the case the marker check- # alone reported as a pass: first page kept, last page kept, middle gone.+ # THE HEAD ARRIVED AND THE THREAD IS STILL INCOMPLETE: the case the marker check alone reported+ # as a pass, first page kept, last page kept, middle gone.out["reason"] = f"the head of the thread mounted, but {out['coverage_reason']}"log(f" COMPLETENESS FAILED: {out['reason']}")elif out.get("ordinal_coverage_state") == COVERAGE_UNMEASURED:- # THE HEAD ARRIVED AND THE MIDDLE WAS NEVER INSPECTED. Not a finding about the arm, and- # not a pass either: this is the same first-page-and-last-page store re-entering through- # the unknown state, so the cell carries a reason and the gate declines to score it.+ # THE HEAD ARRIVED AND THE MIDDLE WAS NEVER INSPECTED. Not a finding about the arm and not a+ # pass: the same store re-entering through the unknown state, so the cell carries a reason and+ # the gate declines to score it.out["reason"] = (f"the head of the thread mounted, but coverage of the middle was NOT ESTABLISHED: "f"{out['coverage_reason']}"@@ -881,7 +852,6 @@ def probe_thread_completeness(f"({out.get('ordinals_seen_count')} of {expected_messages} ordinals seen)")if out.get("ordinal_coverage_complete") is None:- # Only the not-applicable kind reaches here; the unmeasured kind is caught above and- # carries a reason of its own.+ # Only the not-applicable kind reaches here; the unmeasured kind is caught above with its own reason.log(f" coverage DOES NOT APPLY: {out.get('coverage_reason')}")return outdiff --git a/tests/studio/studiobench/runtime/seeder.py b/tests/studio/studiobench/runtime/seeder.pyindex 90c877d1aeb..3b88916d8eb 100644--- a/tests/studio/studiobench/runtime/seeder.py+++ b/tests/studio/studiobench/runtime/seeder.py@@ -35,8 +35,8 @@ from ..fixture.corpus import RungPlan, Unitfrom .lifecycle import StudioAuth, auth_request_json# How close the two paths must land to count as equivalent. Not zero: a streamed reply carries a-# usage record and a duration the seeded one does not, and the composer state differs, so a-# handful of elements always differ. 2% on the quantities that scale with content.+# usage record and a duration the seeded one does not, and the composer state differs. 2% on the+# quantities that scale with content.EQUIVALENCE_TOLERANCE = 0.02@@ -55,11 +55,10 @@ def _assistant_content(unit: Unit) -> list[dict]:parts: list[dict] = []if unit.reasoning:parts.append({"type": "reasoning", "text": unit.reasoning})- # Tool calls sit BETWEEN the reasoning and the answer, which is where a real turn puts them:- # the model thinks, calls tools, then answers. reasoning.tsx groups adjacent tool-call parts- # with the reasoning above them, so the order decides whether a tool group renders inside the- # collapsible pane or as its own block, and those are different components with different- # costs.+ # Tool calls sit BETWEEN the reasoning and the answer, where a real turn puts them. reasoning.tsx+ # groups adjacent tool-call parts with the reasoning above them, so the order decides whether a+ # tool group renders inside the collapsible pane or as its own block, and those are different+ # components with different costs.for call in unit.tool_calls:parts.append(dict(call))if unit.content:@@ -85,10 +84,10 @@ class SeededThread:seeded_chars: intseconds: floatturns: int- # The markers on the FIRST and LAST user turns. The readiness gate uses `last_marker` to prove- # the end of the thread is mounted, and the completeness probe uses `first_marker` to prove a- # windowed arm still holds the head of the conversation. Plain text written by this harness,- # so neither is a guess about what a markdown renderer will do to the corpus.+ # The markers on the FIRST and LAST user turns: the readiness gate uses `last_marker` to prove the+ # end of the thread is mounted, and the completeness probe uses `first_marker` to prove a windowed+ # arm still holds the head. Plain text written by this harness, so neither is a guess about what a+ # markdown renderer will do.first_marker: Optional[str] = Nonelast_marker: Optional[str] = None@@ -109,10 +108,8 @@ class Seeder:def create_thread(self, title: str = "studiobench") -> str:thread_id = str(uuid.uuid4())- # `auth_request_json`, not `request_json`, and that is not a spelling preference: the- # seeder is asked for a thread once per cell for as long as the run lasts, and an access- # token is good for 60 minutes. See `StudioAuth`: the token is re-minted before it expires- # and once more if a 401 arrives anyway.+ # `auth_request_json`, not `request_json`: the seeder is asked for a thread once per cell for as+ # long as the run lasts and an access token is good for 60 minutes. See `StudioAuth`.auth_request_json(self.auth,self._url("/api/chat/threads"),@@ -173,8 +170,8 @@ class Seeder:parent = assistant_idstarted = time.monotonic()if messages:- # pruneMissing so this REPLACES the thread rather than merging into whatever a- # previous cell left behind. A merge would make every rung after the first cumulative.+ # pruneMissing so this REPLACES the thread rather than merging into whatever a previous cell left+ # behind; a merge would make every rung after the first cumulative.auth_request_json(self.auth,self._url(f"/api/chat/threads/{thread_id}/messages"),@@ -228,19 +225,14 @@ def compare_signatures(elements legitimately differ and gating on exact equality would fail every time for a reasonthat has nothing to do with fidelity."""- # GATED ON CONTENT, REPORTED ON REASONING.- #- # A collapsed reasoning pane in a SEEDED thread does not mount its children, while a streamed- # one does: it was open while the text arrived, and collapsing afterwards leaves the subtree- # in place. Measured directly, the same text carried 1,485 reasoning spans one way and 0 the- # other, and open-then-close on the seeded side unmounted them again, so this is not something- # seeding can be made to reproduce -- it is a property of how the app builds a thread.- #+ # GATED ON CONTENT, REPORTED ON REASONING. A collapsed reasoning pane in a SEEDED thread does not+ # mount its children while a streamed one does, because it was open while the text arrived.+ # Measured, the same text carried 1,485 reasoning spans one way and 0 the other, so this is a+ # property of how the app builds a thread rather than something seeding can reproduce.# Gating on total `highlight_spans` therefore asked a question seeding can never pass, and the- # answer moved with whatever pane state the film happened to leave behind: two runs of the- # same rung reported 2.1% and 36.4% drift. The question worth asking is whether the same text- # renders the same CONTENT, which is what these keys are. The reasoning difference is measured- # and reported below rather than swept into a pass or a fail.+ # answer moved with whatever pane state the film left behind: two runs of the same rung reported+ # 2.1% and 36.4% drift. The question worth asking is whether the same text renders the same+ # CONTENT, and the reasoning difference is measured and reported below.keys = ("assistant_messages", "content_code_blocks", "content_spans", "reasoning_panes")fields: dict = {}equivalent = True@@ -354,9 +346,9 @@ def measure_chars_per_token(}except Exception: # noqa: BLE001pass- # Last resort, and LABELLED. Counting whitespace-delimited words plus punctuation is a rough- # stand-in for a BPE tokeniser and is off by tens of percent on dense code, which is most of- # this corpus.+ # Last resort, and LABELLED: counting whitespace-delimited words plus punctuation is a rough+ # stand-in for a BPE tokeniser and is off by tens of percent on dense code, which is most of this+ # corpus.words = len(sample.split())punct = sum(1 for c in sample if not c.isalnum() and not c.isspace())est = max(1, words + punct // 2)diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.pyindex cf9cb49e337..e21fd6d33f9 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab.py@@ -86,13 +86,12 @@ def test_origin_gate_names_the_exact_origin_and_strips_the_slash():assert "return" in script-# ── the gate compares against an ORIGIN, so it has to be given one ──────────────────────────-#-# `window.location.origin` is the URL standard's canonical origin, not the URL the caller typed.-# Every expectation below was read out of chromium, by navigating a real document to the spelling-# on the left and asking the page for `window.location.origin`.+# The gate compares against an ORIGIN, so it has to be given one. `window.location.origin` is the+# URL standard's canonical origin, not the URL the caller typed, and every expectation below was+# read out of chromium by navigating a real document to the spelling on the left.+# ── the gate compares against an ORIGIN, so it has to be given one ──────────────────────────@pytest.mark.parametrize(("spelled", "origin"),[diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.pyindex c309f40137c..f6a087c3bc5 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_failed_cell_verdict.py@@ -32,7 +32,7 @@ SIDES = [{"label": "treatment", "ref": "fix", "base_url": "http://127.0.0.1:5400", "owns": True},]-#: (rung, tokens, base p95, treatment p95). 10K regresses by 100%, 100K improves by 20%.+#:(rung, tokens, base p95, treatment p95). 10K regresses by 100%, 100K improves by 20%.LADDER = (("10K", 10_000, 100.0, 200.0), ("100K", 100_000, 100.0, 80.0))diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.pyindex 8d20e329a53..0bc99473146 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ab_pair_resume.py@@ -38,8 +38,8 @@ def _target(label):return Target(label = label, ref = label, base_url = f"http://x/{label}", seeder = None, runner = None)-#: Each rung needs its OWN token count: `measures_by_cell` keys on `(rung_tokens, rep)`, so two-#: rungs sharing one number collapse into one pair and a two-rung table renders as a one-rung one.+#: Each rung needs its OWN token count: `measures_by_cell` keys on `(rung_tokens, rep)`, so two rungs+#: sharing one number collapse into one pair and a two-rung table renders as a one-rung one.RUNG_TOKENS = {"1K": 1_000, "10K": 10_000, "100K": 100_000, "500K": 500_000, "1M": 1_000_000}@@ -188,9 +188,9 @@ def _resumed_table(tmp_path, *, pair_granular: bool) -> str:def test_the_resumed_session_measures_both_arms_and_gets_a_table(tmp_path):table = _resumed_table(tmp_path, pair_granular = True)- # Both arms measured, so the pair exists and the table has a reading. One pair carries no- # bootstrap CI, so the verdict is INCONCLUSIVE; the contrast with the test below, where the- # arms never pair at all and the table says NO READING, is the thing under test.+ # Both arms measured, so the pair exists and the table has a reading. One pair carries no bootstrap+ # CI, so the verdict is INCONCLUSIVE; the contrast with the test below, where the arms never pair+ # and the table says NO READING, is the thing under test.assert "VERDICT: INCONCLUSIVE" in tableassert "NO READING" not in table@@ -227,8 +227,8 @@ def _two_rung_resumed_table(tmp_path, *, whole_table: bool) -> str:if whole_table:done = skippable_cells(work, recorded)else:- # THE PRE-FIX RULE, inline so the contrast is the change itself: skip any pair whose every- # arm is already recorded, and let the rest of the table be whatever is left.+ # THE PRE-FIX RULE, inline so the contrast is the change itself: skip any pair whose every arm is+ # already recorded, and let the rest of the table be whatever is left.by_pair: dict = {}for _t, cell, _p in work:by_pair.setdefault((cell.rung, cell.rep), []).append(cell.cell_id)@@ -265,9 +265,9 @@ def test_skipping_the_recorded_pair_publishes_a_verdict_over_the_remainder(tmp_ptable = _two_rung_resumed_table(tmp_path, whole_table = False)assert "keystroke_p95_ms 1" in table, table- # The remainder is a single pair, so it can no longer be published as a direction; the bug- # this documents is unchanged and is the two assertions around this one: the 30% regression- # is gone from the table and nothing names the rung it came from.+ # The remainder is a single pair, so it can no longer be published as a direction; the bug this+ # documents is the two assertions around it: the 30% regression is gone from the table and nothing+ # names the rung it came from.assert "VERDICT: INCONCLUSIVE" in table, tableassert "10K" not in tablediff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.pyindex ac40ad770f8..615319bea58 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_completeness_gate_rows.py@@ -45,7 +45,7 @@ from studiobench.runtime.readiness import ( # noqa: E402from studiobench.runtime.session import record_completeness_gate # noqa: E402from studiobench.runtime.types import Cell, Recorder, make_cell_id # noqa: E402-#: The arm the whole windowed readiness gate exists for, at a rung big enough to be windowed.+#:The arm the whole windowed readiness gate exists for, at a rung big enough to be windowed.CELL = Cell(cell_id = make_cell_id("100K", "B1", 0),rung = "100K",@@ -271,9 +271,8 @@ def test_every_per_cell_gate_names_its_cell():src = inspect.getsource(S)def _calls(text: str) -> list:- # A PAREN COUNTER, not a regex. `rec.gate("follows_the_stream", bool(...), ...)` contains a- # nested call, and a non-greedy regex stops at the inner closing paren -- reporting the- # outer call as missing an argument that is two lines further down.+ # A PAREN COUNTER, not a regex: `rec.gate("follows_the_stream", bool(...), ...)` contains a nested+ # call, and a non-greedy regex stops at the inner closing paren.out = []for marker in ("rec.gate(", "recorder.gate("):start = 0@@ -290,9 +289,8 @@ def test_every_per_cell_gate_names_its_cell():depth -= 1j += 1args = text[i + len(marker) : j - 1]- # Skip prose. `session.py` carries a comment reading "WHY THIS IS NOT- # `recorder.gate(...)`", and a scanner that counts it reports a defect in a- # sentence.+ # Skip prose: `session.py` carries a comment reading "WHY THIS IS NOT `recorder.gate(...)`", and a+ # scanner that counts it reports a defect in a sentence.line_start = text.rfind("\n", 0, i) + 1is_comment = text[line_start:i].lstrip().startswith("#")if args.strip() != "..." and not is_comment:diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.pyindex 6ecb6e8b617..2b47bf02402 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_composer_click.py@@ -87,8 +87,7 @@ def test_composer_click_ms_excludes_the_instrument_hooks():runner = _run()got = runner._composer_click_msassert got is not None- # The click is 100 ms and the hooks are 800 ms between them. Timed around the window this- # came back near 900.+ # The click is 100 ms and the hooks are 800 ms between them. Timed around the window this came back near 900.assert CLICK_S * 1000 <= got < CLICK_S * 1000 + TEARDOWN_S * 1000diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.pyindex 2352260d149..a61e40fa9f4 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_copy_from_store_live.py@@ -61,15 +61,14 @@ def context():b = p.chromium.launch(args = ["--no-sandbox"])except Exception as exc: # noqa: BLE001pytest.skip(f"chromium could not be launched: {exc}")- # The real harness's browser factory requests the same two. Without them the clipboard- # read-back throws and the reading is "could not be measured", which is exactly the- # NOT COMPARABLE outcome the scoring layer now produces rather than a pass.+ # The real harness's browser factory requests the same two. Without them the clipboard read-back+ # throws and the reading is "could not be measured", which is the NOT COMPARABLE outcome the+ # scoring layer produces rather than a pass.ctx = b.new_context(permissions = ["clipboard-read", "clipboard-write"])- # `navigator.clipboard` DOES NOT EXIST outside a secure context, and `set_content` leaves- # the page on about:blank, which is not one. The reading came back as- # "Cannot read properties of undefined" rather than as an empty clipboard, which would have- # been easy to misread as "the copy did not work". Fulfilled from a route so no server is- # needed and no network is touched.+ # `navigator.clipboard` DOES NOT EXIST outside a secure context, and `set_content` leaves the page+ # on about:blank, which is not one. The reading came back as "Cannot read properties of undefined"+ # rather than as an empty clipboard, which would have been easy to misread. Fulfilled from a route+ # so no server is needed.ctx.grant_permissions(["clipboard-read", "clipboard-write"], origin = ORIGIN)yield ctxctx.close()@@ -105,8 +104,7 @@ def _page(context, mode: str, copy_from_store: bool):#: Exactly what scene/actions.select_all_copy does: focus the viewport, select its contents, then a-#: REAL Control+C so the app's own copy handler runs. Anything else would be testing a different-#: code path from the one the benchmark drives.+#: REAL Control+C so the app's own copy handler runs.SELECT_JS = """async () => {const v = window.__sb.dom.viewport();@@ -180,8 +178,8 @@ def test_the_handler_puts_the_whole_conversation_on_the_clipboard(context):page.close()assert got["mounted"] == WINDOW < got["total"] == MESSAGESassert _markers_present(got["clipboard"]) == TURNS, got["clipboard"][:400]- # The selection is still short, and that is correct: it can only cover mounted nodes. This is- # the reading the old alarm was wired to, and it is why the alarm had to be moved.+ # The selection is still short, and that is correct: it can only cover mounted nodes. This is the+ # reading the old alarm was wired to, and why the alarm had to be moved.assert got["selected_chars"] < got["clipboard_chars"]@@ -226,6 +224,6 @@ def test_a_partial_selection_is_not_replaced_by_the_whole_conversation(context):assert clip != "SENTINEL"assert len(clip) < 1000, clip[:300]else:- # Chromium gave no selection for an off-screen row, so no copy was performed at all. Said- # out loud rather than passed silently: this run did not exercise the substitution guard.+ # Chromium gave no selection for an off-screen row, so no copy was performed at all. Said out loud+ # rather than passed silently: this run did not exercise the substitution guard.assert clip == "SENTINEL"diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.pyindex 327610089e0..e8fb50a6295 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_follow_coverage_carveout.py@@ -43,7 +43,7 @@ from studiobench.runtime.session import ( # noqa: E402)from studiobench.sweep.ui_parity import incomplete_cells # noqa: E402-#: The coverage actually observed on the run this carve-out was derived from, not a round number.+#:The coverage actually observed on the run this carve-out was derived from, not a round number.OBSERVED_COVERAGE = 0.481@@ -167,11 +167,10 @@ def test_low_coverage_does_not_launder_a_real_pinning_failure(tmp_path) -> None:assert refused-#: Every shape a failed `follows_the_stream` row can take, and whether it must still be fatal.-#: The two admission lists have to agree on all of them: `INVALIDATING_CELL_GATES` was centralised-#: so the scorers could not disagree about what invalidates a cell, and a predicate copied into-#: both consumers reintroduces exactly that drift one level down, where it is harder to see-#: because each copy reads correctly on its own.+#: Every shape a failed `follows_the_stream` row can take, and whether it must still be fatal. The+#: two admission lists have to agree on all of them: `INVALIDATING_CELL_GATES` was centralised so+#: the scorers could not disagree about what invalidates a cell, and a predicate copied into both+#: consumers reintroduces that drift one level down, where each copy reads correctly on its own._AGREEMENT_CASES: list[tuple[str, dict, bool]] = [("coverage-only shortfall", _coverage_short(OBSERVED_COVERAGE), False),("sliver", _coverage_short(0.13), False),@@ -236,9 +235,9 @@ _AGREEMENT_CASES: list[tuple[str, dict, bool]] = [True,),("absent instrument", {"follow_attempted": False, "reason": "sampler is not installed"}, False),- # The narrowing that must survive the refactor: `probe_attempted: False` has two producers and- # only one is an absent instrument. A missing thread viewport is the ARM missing the surface- # under test, and waiving it once let a real failure ride the instrument allowance.+ # The narrowing that must survive the refactor: `probe_attempted: False` has two producers and only+ # one is an absent instrument. A missing thread viewport is the ARM missing the surface under+ # test, and waiving it once let a real failure ride the instrument allowance.("no thread viewport", {"probe_attempted": False, "reason": "no thread viewport"}, True),]@@ -258,11 +257,12 @@ def test_both_admission_lists_agree_on_every_shape(tmp_path) -> None:assert not wrong, "wrong verdict: " + "; ".join(wrong)-# ----------------------------------------------------------------------------------------# the WRITER. Everything above drives the two consumers with a hand-built gate detail, which-# cannot see a defect in the thing that PRODUCES that detail: a `follow_verdict` that set+# the WRITER. Everything above drives the two consumers with a hand-built gate detail, which cannot+# see a defect in the thing that PRODUCES that detail: a `follow_verdict` that set# `stream_coverage_unmeasured` on every low reading would waive real follow failures and every-# consumer test would still pass. These drive the real writer.+# consumer test would still pass.++# ---------------------------------------------------------------------------------------diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.pyindex 12cbe558bff..c6a12153023 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_ladder_ratio_identity.py@@ -257,8 +257,8 @@ def test_a_refused_resume_rolls_back_every_row_it_wrote(tmp_path):assert dropped > 0assert paths.payload_jsonl.read_bytes() == before- # The two ways the leftovers were visible: a rung the payload never owed, and a failed gate- # charged to somebody else's evidence.+ # The two ways the leftovers were visible: a rung the payload never owed, and a failed gate charged+ # to somebody else's evidence.assert recorded_ladder(paths.payload_jsonl) == ["1K", "10K", "100K"]assert [r for r in _rows(paths) if r.get("row_type") == "gate" and r.get("passed") is Falsediff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.pyindex 56bf2facebd..d607a5b96a7 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_launch_cleanup.py@@ -59,8 +59,8 @@ def launched(monkeypatch, tmp_path):}# `raising = False` so this fixture also builds against a lifecycle without the constant, which- # is what makes the test below fail on the unfixed code for the reason it is about rather than- # on the way in.+ # makes the test below fail on the unfixed code for the reason it is about rather than on the way+ # in.monkeypatch.setattr(lifecycle, "PID_DISCOVERY_TIMEOUT_S", 0.0, raising = False)# Stubbed for the same reason and, for every test but the two about it, so that whatever this# machine happens to have on :5399 cannot decide the answer.@@ -70,8 +70,7 @@ def launched(monkeypatch, tmp_path):monkeypatch.setattr(lifecycle, "_find_unsloth_bin", lambda install: "/bin/true")monkeypatch.setattr(lifecycle, "_read_bootstrap_password", lambda *a, **k: "secret")monkeypatch.setattr(lifecycle, "wait_for_healthz", lambda *a, **k: state["healthy"])- # Recorded rather than dropped: a launch refused before the spawn has to be shown not to have- # spawned anything.+ # Recorded rather than dropped: a launch refused before the spawn has to be shown not to have spawned anything.monkeypatch.setattr(subprocess, "Popen", lambda *a, **k: state["spawned"].append(a))def fake_run(cmd, *a, **k):@@ -200,9 +199,8 @@ def test_the_probe_itself_gives_both_answers_against_a_real_socket():port = listener.getsockname()[1]assert lifecycle.port_is_busy(port) is True- # Closed, so the same port is now the negative case. A port the kernel has just released can- # linger in TIME_WAIT for a connect, which is why the assertion below is on a port that was- # never bound at all rather than on this one.+ # Closed, so the same port is now the negative case. A port the kernel has just released can linger+ # in TIME_WAIT for a connect, which is why the assertion below is on a port never bound at all.with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:probe.bind(("127.0.0.1", 0))free_port = probe.getsockname()[1]diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.pyindex 3b026602a98..21345ce890c 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_lifecycle_refs.py@@ -48,8 +48,8 @@ def _git("GIT_COMMITTER_NAME": "studiobench","GIT_COMMITTER_EMAIL": "[email protected]",# No user or system git config: an `init.defaultBranch`, a `commit.gpgsign` or a- # `clone.defaultRemoteName` on the machine running the tests would otherwise decide- # what this repository looks like.+ # `clone.defaultRemoteName` on the machine running the tests would otherwise decide what this+ # repository looks like."GIT_CONFIG_GLOBAL": os.devnull,"GIT_CONFIG_SYSTEM": os.devnull,"HOME": str(cwd),@@ -70,8 +70,8 @@ def _origin(tmp_path: Path) -> tuple[Path, str, str]:(origin / "install.sh").write_text("second\n")_git("commit", "-am", "second", cwd = origin)second = _git("rev-parse", "HEAD", cwd = origin).stdout.strip()- # A bare mirror, because a fetch from a non-bare checkout of the same branch is a special case- # and the real remote is a server.+ # A bare mirror, because a fetch from a non-bare checkout of the same branch is a special case and+ # the real remote is a server.bare = tmp_path / "origin.git"_git("clone", "--bare", str(origin), str(bare), cwd = tmp_path)return bare, first, seconddiff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.pyindex 64a4e5f4217..fc8f7b33f2b 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_live_marker_race.py@@ -41,15 +41,12 @@ import pytestREPO_ROOT = Path(__file__).resolve().parents[5]-#: SIZED BY MEASUREMENT, not by guesswork. Under `spawn` the per-trial collision rate on the broken-#: guard is well below the 50% a barrier reaches under `fork`, because each contender pays its own-#: interpreter startup before arriving. At 10 trials the two headline tests missed the defect in one-#: run out of four; at 40 they caught it in six runs out of six, and the whole file still costs-#: about ten seconds. A concurrency test that only usually fails on the broken code is not much-#: better than one that never does.-#:+#: SIZED BY MEASUREMENT. Under `spawn` the per-trial collision rate on the broken guard is well below+#: the 50% a barrier reaches under `fork`, because each contender pays its own interpreter startup+#: first. At 10 trials the two headline tests missed the defect in one run out of four; at 40 they+#: caught it in six runs out of six, and the file still costs about ten seconds.#: `spawn` rather than `fork` because pytest has already started threads by the time this runs, and-#: forking a multi-threaded process is deprecated for the good reason that the child can deadlock.+#: forking a multi-threaded process is deprecated because the child can deadlock.TRIALS = 40@@ -68,8 +65,8 @@ def _contend(repo_root: str, outdir: str, index: int, start, hold, q) -> None:except Exception as exc: # noqa: BLE001 - reported rather than lostq.put((False, f"UNEXPECTED {type(exc).__name__}: {exc}"))finally:- # Nobody releases the marker until everyone has attempted, so an admission is genuine- # overlap rather than sequential reuse of a directory the first run already let go.+ # Nobody releases the marker until everyone has attempted, so an admission is genuine overlap rather+ # than sequential reuse of a directory the first run already let go.try:hold.wait(timeout = 60)except Exception: # noqa: BLE001@@ -197,14 +194,12 @@ def test_a_crashed_run_does_not_lock_the_directory_forever(tmp_path):rec.close()-# ── a retained record is not a holder ────────────────────────────────-#-# The marker is deliberately never unlinked, so on a directory that has been used before it already-# contains the PREVIOUS run's `pid session` line. These two are deterministic on purpose: they stand-# in the window rather than racing for it, so they say the same thing on a loaded runner as on an-# idle one.+# A retained record is not a holder. The marker is deliberately never unlinked, so a directory used+# before already contains the PREVIOUS run's `pid session` line. These two are deterministic on+# purpose: they stand in the window rather than racing for it.+# ── a retained record is not a holder ────────────────────────────────def _stalled_holder(marker: Path,write_after_s: float,diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.pyindex 796b234d4c0..f6f7f37e012 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_payload_identity.py@@ -455,15 +455,14 @@ def test_a_session_that_died_before_its_first_cell_declares_no_mode(tmp_path):)-# ── the two fixture axes ────────────────────────────────────────────────────────────────────-#-# `--stream-tail-chars` and `--corpus-dollars` change what the last turn STREAMS, and neither of-# them moves `corpus_hash`, which covers the frozen units on disk and the generator's parameters.-# Nor does either move a `cell_id`. They are on `IDENTITY_AXES` for the same reason the tier is:-# without that, a resume under a changed fixture skips cells measured against a different film and-# the payload silently becomes one ladder built from two.+# The two fixture axes. `--stream-tail-chars` and `--corpus-dollars` change what the last turn+# STREAMS, and neither moves `corpus_hash` (which covers the frozen units on disk and the+# generator's parameters) or a `cell_id`. They are on `IDENTITY_AXES` for the same reason the+# tier is: without that, a resume under a changed fixture skips cells measured against a+# different film and the payload silently becomes one ladder built from two.+# ── the two fixture axes ────────────────────────────────────────────────────────────────────def _fixture_payload(tmp_path, name, session, **fixture):paths = Paths.under(tmp_path / name)_record(@@ -710,21 +709,19 @@ def test_a_payload_from_before_the_fixture_axes_is_refused_under_a_non_default_fassert ("stream_tail_chars" if "--stream-tail-chars" in flags else "corpus_dollars") in (message)- # The refusal has to say WHY an axis the payload never mentions is a difference, or the- # reader's next move is to go looking for a key that was never going to be there.+ # The refusal has to say WHY an axis the payload never mentions is a difference, or the reader's+ # next move is to go looking for a key that was never going to be there.assert "predates this axis" in message-# ── the measurement-mode axis ───────────────────────────────────────────────────────────────-#-# `--click-probe` runs a full `page.click`, a real mouse click, a dispatch, a focus and a hover-# over the thread BEFORE the film starts, and the tool's own help text for the flag says it "makes-# the cell's timings incomparable with a cell that did not run it". The cell then carries a-# `composer_click_ms` measured on a composer all of those paths have already been through, and a-# `click_attribution` block a cell without the flag does not have at all. None of that moves the-# cell id, so it is on `IDENTITY_AXES` for the same reason the tier is.+# The measurement-mode axis. `--click-probe` runs a full `page.click`, a real mouse click, a+# dispatch, a focus and a hover over the thread BEFORE the film starts, and the flag's own help+# text says it 'makes the cell's timings incomparable with a cell that did not run it'. The cell+# then carries a `composer_click_ms` measured on a composer all those paths have been through,+# and a `click_attribution` block a cell without the flag lacks. None of that moves the cell id.+# ── the measurement-mode axis ───────────────────────────────────────────────────────────────def test_resuming_after_toggling_the_click_probe_is_refused(tmp_path):"""REGRESSION. Both directions, because either one produces the same mixed ladder.@@ -935,8 +932,8 @@ def test_the_report_a_mode_change_would_have_produced(tmp_path):changed_mode = score_payload(_payload("changed", "base").payload_jsonl, [10_000])# Same mode: the retry reuses the cell id, so the crash is superseded and never reaches the- # ladder. Change the mode and the same crash is a cell of its own, first at its rung, and it- # is what the report prints -- over a rung the run being reported measured cleanly.+ # ladder. Change the mode and the same crash is a cell of its own, first at its rung, and it is+ # what the report prints, over a rung the run being reported measured cleanly.assert "died at 10K" not in (same_mode.rungs[0].incomplete_reason or "")assert "died at 10K" in (changed_mode.rungs[0].incomplete_reason or "")@@ -1008,15 +1005,13 @@ def test_the_readme_sequence_no_longer_scores_two_runs_as_one_ladder(tmp_path):assert payload["header"]["studio_ref"] == "main"-# ── the injection axis ──────────────────────────────────────────────────────────────────────-#-# `--inject-stream-cost-ms` burns a known amount of main-thread time per SSE chunk on the TREATMENT-# arm, so a treatment cell recorded with it is not the same reading as one recorded without it ---# and `cell_id` carries the rung, the arm and the repetition and nothing that could tell. It is on-# `IDENTITY_AXES` for the same reason the two fixture axes above are, and for a larger perturbation-# than either of them makes.+# The injection axis. `--inject-stream-cost-ms` burns a known amount of main-thread time per SSE+# chunk on the TREATMENT arm, so a treatment cell recorded with it is not the same reading as one+# recorded without, and `cell_id` carries the rung, the arm and the repetition and nothing that+# could tell.+# ── the injection axis ──────────────────────────────────────────────────────────────────────def _injected_payload(tmp_path, name, session, **fixture):paths = Paths.under(tmp_path / name)_record(@@ -1131,19 +1126,17 @@ def test_an_unchanged_injection_resumes(tmp_path):assert _resume_set(paths) == {"r10K.base.rep0", "r10K.treatment.rep0"}-# ── the external probe axis ─────────────────────────────────────────────────────────────────-#-# `SBENCH_EXTRA_INIT_SCRIPT` installs a caller's own init script into the page, and the run says so-# in its log: "this run carries an external probe and is NOT a clean measurement of the build".-# What it cannot end in is a wrong number, because `refuse_if_probed` reads EVERY `run_meta` in the-# file and every scoring entry point calls it. It is on `IDENTITY_AXES` for what that refusal-# costs: the refusal is whole-file and the payload is append-only, so a resume that toggles the-# probe takes the cells recorded before it down with it, and nothing here can put them back.-#-# It is also the axis most easily toggled by accident. The other four are typed on the command-# line every time; this one is an environment variable, and a variable that is still set is-# indistinguishable from one that was never set until the run is over.+# The external probe axis. `SBENCH_EXTRA_INIT_SCRIPT` installs a caller's own init script into the+# page, and the run's log says 'this run carries an external probe and is NOT a clean measurement+# of the build'. It cannot end in a wrong number, because `refuse_if_probed` reads EVERY+# `run_meta` and every scoring entry point calls it; it is on `IDENTITY_AXES` for what that+# refusal costs, since the refusal is whole-file and the payload append-only, so a resume that+# toggles the probe takes the cells recorded before it down with it.+# It is also the axis most easily toggled by accident: the other four are typed on the command+# line every time, while a variable that is still set is indistinguishable from one that was+# never set until the run is over.+# ── the external probe axis ─────────────────────────────────────────────────────────────────PROBE = "probes/paint_counter.js"diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.pyindex 46725965fff..d6040984da2 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_probe_survives_failure.py@@ -119,14 +119,14 @@ class _Pacer:class _Seeder:- #: `None`, so `measure_chars_per_token` cannot reach for the network from a unit test.+ #:`None`, so `measure_chars_per_token` cannot reach for the network from a unit test.auth = Nonedef seed(self, plan):- # No messages, so the mount wait is the selector and not a count that never arrives, and- # both markers `SeededThread` declares are present and `None` for the same reason:- # `_wait_for_thread` reads `last_marker` unconditionally, and a stub that omits it fails on- # the attribute rather than on the failure these tests are about.+ # No messages, so the mount wait is the selector and not a count that never arrives, and both+ # markers `SeededThread` declares are present and `None`: `_wait_for_thread` reads `last_marker`+ # unconditionally, and a stub that omits it fails on the attribute rather than on the failure+ # these tests are about.return types.SimpleNamespace(thread_id = "t-1",seconds = 0.0,@@ -210,8 +210,8 @@ def test_a_cell_that_dies_after_the_probe_still_reports_the_attribution(tmp_pathassert row["completed"] is Falseassert row["failure"]["kind"] == "TimeoutError"- # Out of the payload, not out of the returned dict: the file is what a failed run leaves- # behind, and it is where the diagnostic has to be readable from.+ # Out of the payload, not the returned dict: the file is what a failed run leaves behind, and where+ # the diagnostic has to be readable from.cells = _cell_rows(paths)assert len(cells) == 1attribution = cells[0]["click_attribution"]diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.pyindex 42d446de724..17e89fc44d4 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_readiness_live.py@@ -82,8 +82,7 @@ TURNS = 9MESSAGES = TURNS * 2 # 18, the number in the failure this work exists to fixWINDOW = 6#: The fixture's row height. The completeness tests step the traversal by two rows, because a-#: gesture whose stops do not overlap can only report NOT MEASURED, and a test that wants to see-#: coverage refuse something has to give the probe a sweep that actually covers the thread.+#: gesture whose stops do not overlap can only report NOT MEASURED.ROW_PX = 120_DOM_JS = _STUDIO_TESTS / "studiobench" / "scene" / "dom.js"@@ -215,10 +214,10 @@ def test_full_mount_is_admitted_in_full_mode(browser):assert r.conditions["end_present"] is Trueassert r.conditions["settled"] is Trueassert r.probe["mounted"] == MESSAGES- # AND THE ORDINAL CONDITIONS ARE NOT APPLICABLE HERE, which is not the same as passing.- # Unsloth publishes no aria-posinset anywhere, so a `full` arm has none to validate and must- # never be gated on them; `None` is the value the parity layer and this gate both use for a- # surface that was not measured rather than one that agreed.+ # AND THE ORDINAL CONDITIONS ARE NOT APPLICABLE HERE, which is not the same as passing. Unsloth+ # publishes no aria-posinset anywhere, so a `full` arm has none to validate; `None` is what the+ # parity layer and this gate both use for a surface that was not measured rather than one that+ # agreed.assert r.conditions["posinset_ordinals_valid"] is Noneassert r.conditions["posinset_reaches_end"] is Noneassert r.probe["posinset_count"] == 0@@ -248,7 +247,7 @@ def test_a_virtualised_thread_is_admitted_in_windowed_mode(browser):assert r.conditions["anchored_at_end"] is Trueassert r.conditions["end_present"] is True# The ordinals of a window at the END of an eighteen-message thread: 13..18, distinct, one per- # mounted row. This is the shape the three malformed modes below fail to produce.+ # mounted row. The shape the three malformed modes below fail to produce.assert (r.probe["min_posinset"], r.probe["max_posinset"]) == (MESSAGES - WINDOW + 1, MESSAGES)assert r.probe["posinset_distinct"] == WINDOW@@ -310,16 +309,15 @@ def _completeness(browser, mode: str, **kwargs) -> tuple[dict, list[str]]:def test_a_virtualised_thread_passes_the_completeness_probe(browser):out, _ = _completeness(browser, "windowed")assert out["head_reached"] is True, out- # AND THE COVERAGE VERDICT IS NOT MEASURED AT THE DEFAULT STEP, which is the honest answer- # rather than a flattering one. The gesture jumps 2,000px at a time and this fixture's whole- # thread is 2,160px, so it lands at the bottom and then at the top and the rows between the- # two stops were never in view. Nothing is known about them, so nothing is claimed.+ # AND THE COVERAGE VERDICT IS NOT MEASURED AT THE DEFAULT STEP, which is the honest answer. The+ # gesture jumps 2,000px and this thread is 2,160px, so it lands at the bottom and then the top+ # and the rows between were never in view.assert out["ordinal_coverage_complete"] is None, outassert out["sweep_continuous"] is Falseassert "never in view" in out["coverage_reason"]- # WHICH KIND OF NOT MEASURED. The arm publishes ordinals, so the question applies and the- # sweep failed to answer it, which `record_completeness_gate` refuses to score the cell on.- # The remedy is the smaller step the next test uses, not a gate that accepts the unknown.+ # WHICH KIND OF NOT MEASURED: the arm publishes ordinals, so the question applies and the sweep+ # failed to answer it, which `record_completeness_gate` refuses to score the cell on. The remedy+ # is the smaller step the next test uses.assert out["ordinal_coverage_state"] == COVERAGE_UNMEASURED, out@@ -448,10 +446,9 @@ def test_windowed_mode_also_admits_a_thread_short_enough_to_mount_whole(browser)assert r.ready, r.reasonassert r.probe["setsize"] is Noneassert r.conditions["total_declared"] is True- # THE SAME WAIVER COVERS THE ORDINALS, and for the same reason: there are none to publish and- # none to validate. It is waived on `mounted >= expected` AND on the absence of ordinals- # together, so an arm that publishes malformed ones cannot buy its way out by also mounting- # the whole thread.+ # THE SAME WAIVER COVERS THE ORDINALS: there are none to publish and none to validate. Waived on+ # `mounted >= expected` AND the absence of ordinals together, so an arm publishing malformed ones+ # cannot buy its way out by also mounting the whole thread.assert r.probe["posinset_count"] == 0assert r.conditions["posinset_ordinals_valid"] is Trueassert r.conditions["posinset_reaches_end"] is True@@ -479,8 +476,8 @@ def test_a_half_mounted_thread_is_refused_in_full_mode(browser):detail = caught.value.detailassert detail["ready"] is Falseassert detail["conditions"]["all_messages_mounted"] is False- # And the two NEW conditions caught it too, independently of the count. That matters: it is- # what makes the windowed mode safe, because the windowed mode has no count to rely on.+ # And the two NEW conditions caught it too, independently of the count, which is what makes the+ # windowed mode safe, since it has no count to rely on.assert detail["conditions"]["end_present"] is Falseassert detail["probe"]["mounted"] < MESSAGES@@ -508,8 +505,8 @@ def test_a_half_mounted_thread_is_refused_in_windowed_mode_too(browser):finally:page.close()conditions = caught.value.detail["conditions"]- # THREE independent refusals, not one. It is still growing, it has not reached the end of the- # thread, and it publishes no total.+ # THREE independent refusals: it is still growing, it has not reached the end of the thread, and+ # it publishes no total.assert conditions["settled"] is Falseassert conditions["end_present"] is Falseassert conditions["total_declared"] is False@@ -631,8 +628,8 @@ def test_a_bottom_window_numbered_from_one_is_refused(browser):assert conditions["posinset_ordinals_valid"] is Trueassert detail["probe"]["max_posinset"] == WINDOWassert conditions["posinset_reaches_end"] is False- # The TEXT of the last message is mounted; only the numbering disagrees with it. That split is- # the point: `end_present` reads the thread and this reads what the arm says about it.+ # The TEXT of the last message is mounted; only the numbering disagrees. That split is the point:+ # `end_present` reads the thread and this reads what the arm says about it.assert conditions["end_present"] is Trueassert conditions["anchored_at_end"] is True@@ -738,8 +735,8 @@ def test_evaluate_refuses_ordinals_that_are_not_positions():conditions = evaluate(from_one, from_one, 18, MODE_WINDOWED)assert conditions["posinset_ordinals_valid"] is Trueassert conditions["posinset_reaches_end"] is False- # PAST THE END OF THE SET THE SAME ROWS DECLARE. 19 of 18 is not a position either, and it is- # what an off-by-one in a virtualizer's index-to-ordinal arithmetic produces.+ # PAST THE END OF THE SET THE SAME ROWS DECLARE. 19 of 18 is not a position either, and it is what+ # an off-by-one in index-to-ordinal arithmetic produces.over = _windowed_probe(max_posinset = 19)assert evaluate(over, over, 18, MODE_WINDOWED)["posinset_ordinals_valid"] is False# And every one of them is NOT APPLICABLE in full mode, where nothing publishes ordinals.@@ -796,9 +793,8 @@ def test_ordinal_coverage_never_reports_a_gap_in_the_gesture_as_data_loss():}got_coarse = ordinal_coverage(coarse, 18)assert got_coarse["ordinal_coverage_complete"] is None- # And it is the kind of None that is NOT a pass: the ordinals apply to this arm and the sweep- # did not inspect them. `not_applicable` -- the other None -- is an arm publishing no ordinals- # at all, which is the case below in `test_ordinal_coverage_separates_...`.+ # And it is the kind of None that is NOT a pass: the ordinals apply to this arm and the sweep did+ # not inspect them. `not_applicable` is an arm publishing no ordinals at all.assert got_coarse["ordinal_coverage_state"] == COVERAGE_UNMEASUREDcontinuous = dict(coarse, sweep_continuous = True)got = ordinal_coverage(continuous, 18)@@ -923,6 +919,6 @@ def test_a_windowed_thread_with_no_viewport_is_refused(browser):finally:page.close()- # Named in the refusal, so the reader is told the surface is missing rather than left to infer- # it from a condition that is really about something else.+ # Named in the refusal, so the reader is told the surface is missing rather than left to infer it+ # from a condition about something else.assert "viewport_present" in str(caught.value), str(caught.value)diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.pyindex 7320f026147..66fa5c795a0 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_resume_commit.py@@ -54,9 +54,9 @@ class _Verdict:class _Bundle:- # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta`- # records the engine and `requested_identity` resolves the same way, so a stub that names a- # fixed one would make a legitimate resume look like an engine change off Linux and macOS.+ # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta` records the+ # engine and `requested_identity` resolves the same way, so a stub naming a fixed one would make a+ # legitimate resume look like an engine change off Linux and macOS.engine = browser_mod.default_engine()[0]engine_note = "stubbed for this test"browser = context = page = cdp = None@@ -100,8 +100,8 @@ def studio(monkeypatch, tmp_path):def fake_install(ref, home, *args, **kwargs):install = StudioInstall(home = Path(home), repo = Path(home).parent / "repo", branch = ref)# `setattr` rather than a constructor argument, so this fixture also builds against a- # `StudioInstall` that has no commit field and the tests below fail on the subject rather- # than on the way in.+ # `StudioInstall` that has no commit field and the tests fail on the subject rather than the way+ # in.install.commit = state["commits"].get(ref, f"c-{ref}-1")return installdiff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.pyindex a66e267ead4..1f8aae053cd 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py@@ -57,9 +57,9 @@ class _Verdict:class _Bundle:- # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta`- # records the engine and `requested_identity` resolves the same way, so a stub that names a- # fixed one would make a legitimate resume look like an engine change off Linux and macOS.+ # WHAT `browser.launch` WOULD HAVE RESOLVED on the machine running this test. `run_meta` records the+ # engine and `requested_identity` resolves the same way, so a stub naming a fixed one would make a+ # legitimate resume look like an engine change off Linux and macOS.engine = browser_mod.default_engine()[0]engine_note = "stubbed for this test"browser = context = page = cdp = None@@ -433,15 +433,14 @@ def test_a_plain_run_and_a_plain_resume_are_unaffected(studio):)-# ── the external probe on the record ────────────────────────────────────────────────────────-#-# `SBENCH_EXTRA_INIT_SCRIPT` is an ENVIRONMENT variable, not a flag, so it outlives the command-# that wanted it. A resume under a shell that still has it set runs the rungs still owed with the-# probe in the page and appends a probed `run_meta`, and `refuse_if_probed` reads every `run_meta`-# in a file: the cells recorded cleanly before it stop being scorable too, permanently, because a-# payload is append-only. So `run_meta` has to carry the probe and `--resume` has to compare it.+# The external probe on the record. `SBENCH_EXTRA_INIT_SCRIPT` is an ENVIRONMENT variable, so it+# outlives the command that wanted it: a resume under a shell that still has it set runs the rungs+# still owed with the probe in the page and appends a probed `run_meta`, and `refuse_if_probed`+# reads every `run_meta` in a file, so cells recorded cleanly before it stop being scorable too,+# permanently. So `run_meta` has to carry the probe and `--resume` has to compare it.+# ── the external probe on the record ────────────────────────────────────────────────────────class _ProbedBundle(_Bundle):"""A probe run attaches console and `pageerror` listeners, which a `None` page cannot take."""@@ -645,8 +644,8 @@ def test_a_fresh_probe_run_replaces_the_summary_it_inherited(studio, monkeypatchassert "NO SUMMARY" in textassert script.name in textassert "studiobench summary" not in text- # The evidence itself is untouched: the refusal replaces the report, never the payload the- # summary described, which is still on disk under its archived name.+ # The evidence itself is untouched: the refusal replaces the report, never the payload the summary+ # described, which is still on disk under its archived name.assert len(list(Paths.under(studio["out"]).out.glob("payload-*.jsonl"))) == 1@@ -843,11 +842,11 @@ def test_an_attached_null_control_registers_one_provider_for_the_one_studio(studargs = _args(studio, "--attach", url, "--attach-b", url, "--branch", "main", "--ab", "main")assert sb.run(args, ab_ref = "main") == 0- # THE SYMPTOM FIRST. Whatever the bookkeeping below says, the failure a cell actually meets is- # a seed script that SELECTS a provider the backend no longer has, so that is what this pins:- # every id named by a script scoped to this origin is still live there, and no live id is- # unnamed. Asserting only "nothing was deleted" would pass a future fix that deleted and then- # re-seeded both scripts with the survivor, and would fail a fix that never selects at all.+ # THE SYMPTOM FIRST. Whatever the bookkeeping says, the failure a cell meets is a seed script that+ # SELECTS a provider the backend no longer has, so that is what this pins: every id named by a+ # script scoped to this origin is still live there, and no live id is unnamed. Asserting only+ # 'nothing was deleted' would pass a fix that deleted and re-seeded both scripts with the+ # survivor.selected = _selected_provider_ids(scripts, url)assert selected == set(backend.live[url])assert selected, "the one Unsloth must still have a provider selected"diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.pyindex e8eaa73cf67..e0717c42f5f 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_drain.py@@ -138,9 +138,9 @@ class _Pacer:return list(self.streams)-#: (action name, timing key, value). The three actions `scoring.from_payload.ACTION_SOURCES`-#: reads, so a cell that completes carries enough weight to be scored rather than failing the-#: coverage floor for an unrelated reason.+#: (action name, timing key, value). The three actions `scoring.from_payload.ACTION_SOURCES` reads,+#: so a cell that completes carries enough weight to be scored rather than failing the coverage+#: floor for an unrelated reason.SCENE_ACTIONS = (("keystroke", "p95_ms", 40.0),("message_menu", "open_ms", 90.0),@@ -279,8 +279,8 @@ def cell_runner(monkeypatch, tmp_path):def _cell():- # NOT the 10K rung: that one additionally runs the seeded-vs-streamed equivalence check, which- # is a different subject.+ # NOT the 10K rung: that one additionally runs the seeded-vs-streamed equivalence check, which is a+ # different subject.return Cell(cell_id = "r1K.A0.rep0", rung = "1K", rung_tokens = 1_000, tier = "quick")@@ -331,8 +331,7 @@ def test_a_cell_whose_reply_never_finished_does_not_complete(cell_runner):assert row["completed"] is Falseassert row["failure"]["kind"] == "RuntimeError"assert "never finished" in row["failure"]["message"]- # The evidence ships in the same row as the failure: how long was waited, and how much the- # pacer actually delivered.+ # The evidence ships in the same row as the failure: how long was waited, and how much the pacer actually delivered.assert row["stream"]["finished"] is Falseassert row["pacer"]["last"]["chunks"] == 150assert row["pacer"]["streams"][0]["tag"] == "r1K.A0.rep0"diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.pyindex 0945f4b0418..0a329263885 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_stream_stats.py@@ -257,7 +257,6 @@ def test_a_cell_with_nothing_planned_is_not_checked():# ── level 2: what the cell does with it ──────────────────────────────────────────────────────-CENSUS = {"messages": 6, "elements": 1200, "highlight_spans": 200, "assistant_chars": 1120}@@ -394,9 +393,9 @@ def cell_runner(monkeypatch, tmp_path):thread_id = "t1",seconds = 0.5,messages = 0,- # Both markers `SeededThread` declares, present and None: the readiness gate- # reads `last_marker` unconditionally, so a stub that omits it fails on the- # attribute rather than on the stream accounting these tests are about.+ # Both markers `SeededThread` declares, present and None: the readiness gate reads `last_marker`+ # unconditionally, so a stub that omits it fails on the attribute rather than on the stream+ # accounting.first_marker = None,last_marker = None,),@@ -495,7 +494,7 @@ def _scene_with(rows: list[dict]):return _Fixed-#: A `send_turn` that RAN, loaded the pacer and pressed Enter, and whose own assertion failed.+#:A `send_turn` that RAN, loaded the pacer and pressed Enter, and whose own assertion failed.SEND_TURN_THAT_FAILED = {"action": "send_turn","ran": True,@@ -688,10 +687,9 @@ class _WirePage:if "messageCount" in expr:return self.messages# THE THREAD'S LENGTH AS WELL AS THE MOUNTED COUNT. `send_turn` proves a send worked by- # `threadTotal()` growing, not `messageCount()`, so that a windowed arm whose window slides- # is not read as a send that did nothing. This page models a fully mounted arm, where the- # two are the same number; without the second name the shipped action sees 0 both sides and- # every follow-up reports that it never started a reply.+ # `threadTotal()` growing, not `messageCount()`, so a windowed arm whose window slides is not read+ # as a send that did nothing. This page models a fully mounted arm; without the second name the+ # shipped action sees 0 both sides and every follow-up reports that it never started a reply.if "threadTotal" in expr:return self.messagesif "assistantChars" in expr:@@ -776,9 +774,9 @@ def test_a_healthy_multi_turn_cell_passes_the_check_over_real_wire_bytes(monkeypthread_id = "t1",seconds = 0.5,messages = 0,- # Both markers `SeededThread` declares, present and None: the readiness gate- # reads `last_marker` unconditionally, so a stub that omits it fails on the- # attribute rather than on the stream accounting these tests are about.+ # Both markers `SeededThread` declares, present and None: the readiness gate reads `last_marker`+ # unconditionally, so a stub that omits it fails on the attribute rather than on the stream+ # accounting.first_marker = None,last_marker = None,),diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.pyindex 37f3c6d611d..2e87f2a8649 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_token_refresh.py@@ -45,9 +45,9 @@ from studiobench.runtime.seeder import Seeder # noqa: E402#: How long the fake Unsloth's access tokens live. An hour compressed into something a test can sit#: through, and long enough that a loaded machine cannot expire one mid-request.TOKEN_TTL_S = 6.0-#: The margin the tests below drive `token()` with. The REAL ratio matters: a margin far shorter-#: than the token's life is what the harness runs with (15 minutes against 60), and a margin longer-#: than the whole life would make every call rotate and prove nothing.+#: The margin the tests below drive `token()` with. The REAL ratio matters: a margin far shorter than+#: the token's life is what the harness runs with (15 minutes against 60), and a margin longer than+#: the whole life would make every call rotate and prove nothing.TEST_MARGIN_S = 1.0PASSWORD = "studiobench-bench-password"@@ -62,8 +62,8 @@ class _State:self.logins = 0self.login_attempts = 0self.rejections = 0- #: Set to reject EVERY token once, whatever its `exp` says: an Unsloth restarted underneath- #: the run, or a clock this process cannot see.+ #: Set to reject EVERY token once, whatever its `exp` says: an Unsloth restarted underneath the run,+ #: or a clock this process cannot see.self.reject_next = 0self.lock = threading.Lock()@@ -194,8 +194,8 @@ def test_the_seeder_keeps_working_after_its_token_expires(studio, monkeypatch):logins_after_setup = state.loginstime.sleep(TOKEN_TTL_S + 0.5)- # The token the run was handed is dead by now -- the control above proves the server refuses- # it -- and this still has to work.+ # The token the run was handed is dead by now (the control above proves the server refuses it) and+ # this still has to work.assert seeder.create_thread()assert auth.rotations == 1assert state.logins == logins_after_setup + 1@@ -215,8 +215,7 @@ def test_the_token_is_replaced_before_it_expires_not_after_it_fails(studio, monkmonkeypatch.setattr(lifecycle, "TOKEN_REFRESH_MARGIN_S", TEST_MARGIN_S)base_url, state = studioauth = authenticate(base_url, "bench", PASSWORD, new_password = PASSWORD)- # Inside the margin and NOT yet expired: the server would still accept the old token, and it- # is replaced anyway.+ # Inside the margin and NOT yet expired: the server would still accept the old token, and it is replaced anyway.time.sleep(TOKEN_TTL_S - TEST_MARGIN_S / 2)assert auth_request_json(auth, f"{base_url}/api/chat/threads", method = "POST", body = {})diff --git a/tests/studio/studiobench/runtime/selftest/thread_fixture.js b/tests/studio/studiobench/runtime/selftest/thread_fixture.jsindex 40b5bddfe7c..70876b0a2de 100644--- a/tests/studio/studiobench/runtime/selftest/thread_fixture.js+++ b/tests/studio/studiobench/runtime/selftest/thread_fixture.js@@ -1,57 +1,49 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//// A SYNTHETIC THREAD, with a switch on how it mounts.-//// This exists so the readiness gate can be shown PASSING on a correctly virtualised thread and-// FAILING on a broken one, in a real browser, against the real `scene/dom.js` adapter and the real-// `runtime/readiness.py` probe. A gate that has only ever been shown passing is not a gate, and-// neither is one that has only ever been shown against a mock of itself.-//-// It reproduces exactly the contract the shipping app publishes and the readiness gate reads ---// `.aui-thread-root`, `.aui-thread-viewport`, `[data-role]`, `.aui-thread-scroll-to-bottom` with-// its `invisible` toggle, the composer textarea -- and nothing else. It is not a replica of Unsloth-// and must never be mistaken for one: it cannot tell you anything about the app's own timing. What-// it can do is put the gate in front of each of the situations it has to tell apart.+// FAILING on a broken one, in a real browser, against the real `scene/dom.js` adapter and the+// real `runtime/readiness.py` probe.+// It reproduces exactly the contract the shipping app publishes and the gate reads+// (`.aui-thread-root`, `.aui-thread-viewport`, `[data-role]`, `.aui-thread-scroll-to-bottom`+// with its `invisible` toggle, the composer textarea) and nothing else. It is not a replica of+// Unsloth and can say nothing about the app's own timing.(() => {const MODES = [// Everything mounted, the way the app ships today."full",- // Mounting is still in progress: the first K messages, growing. This is the state the gate- // exists to refuse, and the state the old count-based gate DID correctly refuse.+ // Mounting still in progress: the first K messages, growing. The state the gate exists to+ // refuse, and the one the old count-based gate did correctly refuse."mounting",// A correct window: the tail of the thread, aria-setsize/aria-posinset published, anchored at- // the bottom, and the head materialises when you scroll to the top.+ // the bottom, head materialising when you scroll to the top."windowed",// A window with no aria-setsize. Nothing outside the app can learn how long this thread is."windowed_no_total",- // A window over the HEAD of the thread rather than its end: settled, correct total, but the- // last message is nowhere.+ // A window over the HEAD of the thread rather than its end: settled, correct total, last message nowhere."windowed_at_top",- // A window at the end, correct total published, and a store that only holds what is mounted.- // Indistinguishable from `windowed` while you stand at the bottom of the thread, which is why- // the completeness probe has to walk to the top.+ // A window at the end with a correct total published and a store that only holds what is+ // mounted. Indistinguishable from `windowed` while you stand at the bottom, which is why the+ // completeness probe walks to the top."windowed_lost_head",// A store that kept the first page AND the last page and lost the middle. The head marker- // arrives when you scroll to the top, so the marker check alone calls it complete; the- // ordinals of one mounted window run 1,2,3 then 16,17,18, which is the loss.+ // arrives when you scroll to the top, so the marker check alone calls it complete; the ordinals+ // run 1,2,3 then 16,17,18, which is the loss."windowed_lost_middle",- // The same correct window, but with the ordinals written directly on the [data-role] element- // instead of on a row wrapper. Both placements must be accepted.+ // The same correct window with the ordinals written directly on the [data-role] element instead+ // of a row wrapper. Both placements must be accepted."windowed_flat",- // THE THREE MALFORMED ORDINAL CONTRACTS. Each publishes aria-posinset on every mounted row --- // enough for a gate that only counts attributes -- and each publishes numbers that are not- // positions in the thread, so nothing outside the app can locate the mounted window.- //- // Every row says 0. The attribute is there and it is out of range: aria-posinset is 1-based.+ // THE THREE MALFORMED ORDINAL CONTRACTS. Each publishes aria-posinset on every mounted row,+ // enough for a gate that only counts attributes, and each publishes numbers that are not+ // positions, so nothing outside the app can locate the mounted window.+ // Every row says 0. The attribute is there and out of range: aria-posinset is 1-based."windowed_zero_ordinals",// Every row says the same number, so six mounted rows claim one position between them."windowed_duplicate_ordinals",// A window at the BOTTOM of an 18-message thread numbered 1..6: the index within the window- // published as though it were the position in the thread. The most likely of the three to be- // written by accident, and the one that makes a window at the end look like a window at the- // start.+ // published as the position in the thread. The likeliest of the three to be written by+ // accident, and the one that makes a window at the end look like a window at the start."windowed_from_one",];@@ -96,8 +88,8 @@root.appendChild(composer);document.body.appendChild(root);- // Every message the "store" holds. `windowed_lost_head` throws most of them away, which is- // the data loss the completeness probe is looking for.+ // Every message the 'store' holds. `windowed_lost_head` throws most of them away, which is the+ // data loss the completeness probe looks for.const ROW_PX = 120;let store = [];for (let i = 0; i < turns; i += 1) {@@ -106,9 +98,9 @@}const declaredTotal = total;if (mode === "windowed_lost_head") store = store.slice(-windowSize);- // THE HEAD AND THE TAIL, AND NOTHING BETWEEN THEM. Half a window at each end, so the store- // is exactly one window long and every scroll position mounts all of it: the first message- // is always there for the marker check, and the hole is always there for the ordinals.+ // THE HEAD AND THE TAIL, AND NOTHING BETWEEN THEM. Half a window at each end, so the store is+ // exactly one window long and every scroll position mounts all of it: the first message is+ // always there for the marker check, and the hole is always there for the ordinals.if (mode === "windowed_lost_middle") {const keep = Math.max(1, Math.floor(windowSize / 2));store = store.slice(0, keep).concat(store.slice(-keep));@@ -116,11 +108,9 @@const state = { mode, store, declaredTotal, windowSize, ROW_PX, total };- // WHAT EACH ROW PUBLISHES AS ITS POSITION. Everything but the three malformed modes- // publishes the message's real position in the thread, which is what aria-posinset means.- // The malformed ones publish a number that is not one, each in a different way, so the gate- // can be shown refusing each of them separately rather than refusing "something about the- // ordinals".+ // WHAT EACH ROW PUBLISHES AS ITS POSITION. Everything but the three malformed modes publishes+ // the message's real position, which is what aria-posinset means; the malformed ones each+ // publish a different sort of non-position, so the gate can be shown refusing each separately.function publishedPos(message, indexInWindow) {if (mode === "windowed_zero_ordinals") return 0;if (mode === "windowed_duplicate_ordinals") return state.declaredTotal;@@ -138,11 +128,10 @@el.setAttribute("data-role", m.role);el.style.cssText = "height:" + ROW_PX + "px;overflow:hidden;";el.textContent = m.text;- // WHERE THE SHIPPING VIRTUALIZER PUTS THE ORDINALS: on the positioned row wrapper, not- // on the message. `thread-message-virtualizer.tsx` renders an absolutely positioned div- // per item and mounts `ThreadPrimitive.MessageByIndex` inside it, so the element that is- // a member of the set is the wrapper. `wrapped` reproduces that shape exactly; the- // unwrapped placement is kept too, because the gate must accept both.+ // WHERE THE SHIPPING VIRTUALIZER PUTS THE ORDINALS: on the positioned row wrapper, not the+ // message. `thread-message-virtualizer.tsx` renders an absolutely positioned div per item with+ // `ThreadPrimitive.MessageByIndex` inside, so the wrapper is the member of the set. `wrapped`+ // reproduces that; the unwrapped placement is kept because the gate must accept both.if (publishTotals && mode !== "windowed_flat") {const row = document.createElement("div");row.setAttribute("aria-setsize", String(state.declaredTotal));@@ -157,9 +146,9 @@}list.appendChild(el);}- // Spacers, so the scroll extent describes the WHOLE thread even though only a window is- // mounted. A virtualizer that omits these has a scrollbar that lies, which is one of the- // behavioural invariants in analysis/behaviour.py.+ // Spacers, so the scroll extent describes the WHOLE thread even though only a window is mounted.+ // A virtualizer that omits these has a scrollbar that lies, one of the behavioural invariants in+ // analysis/behaviour.py.const above = mode === "full" || mode === "mounting" ? 0 : startIndex * ROW_PX;const below =mode === "full" || mode === "mounting"@@ -175,10 +164,10 @@}state.renderWindowAround = (scrollTop) => {- // A store SHORTER than the thread it claims cannot be indexed by the thread's own- // indices: `windowed_lost_middle` holds one window's worth of messages for an- // eighteen-message thread, so every scroll position mounts the same rows and the ordinals- // on them are the only thing that says which messages they are.+ // A store SHORTER than the thread it claims cannot be indexed by the thread's own indices:+ // `windowed_lost_middle` holds one window's worth for an eighteen-message thread, so every+ // scroll position mounts the same rows and the ordinals are the only thing saying which+ // messages they are.if (mode === "windowed_lost_middle") {const start = Math.max(0,@@ -191,8 +180,8 @@0,Math.min(state.declaredTotal - windowSize, Math.floor(scrollTop / ROW_PX)),);- // `windowed_lost_head` only has the tail in its store, so an index into the full thread- // has to be translated. It will simply have nothing to show near the top.+ // `windowed_lost_head` only has the tail in its store, so an index into the full thread has to+ // be translated; it will simply have nothing to show near the top.const offset = mode === "windowed_lost_head" ? state.declaredTotal - state.store.length : 0;render(Math.max(0, first - offset), windowSize, true);};@@ -201,9 +190,9 @@render(0, state.store.length, false);pin();} else if (mode === "mounting") {- // GROWING. One message every 250ms, from the head, and it never finishes inside the- // window any test here gives it. The count climbs, the element count climbs, the scroll- // height climbs, and the last message is never reached.+ // GROWING: one message every 250ms from the head, never finishing inside the window any test+ // gives it. The count, element count and scroll height climb, and the last message is never+ // reached.let n = 1;render(0, n, false);pin();@@ -222,8 +211,8 @@pin();} else {// windowed, windowed_flat, windowed_lost_head, windowed_lost_middle and the three- // malformed-ordinal modes: all of them are a window at the END of the thread, and they- // differ only in what they publish about it.+ // malformed-ordinal modes are all a window at the END of the thread, differing only in what+ // they publish about it.render(Math.max(0, state.store.length - windowSize), windowSize, true);pin();viewport.addEventListener("scroll", () => {@@ -234,13 +223,11 @@);});}- // COPY-FROM-STORE, the shipping fix, reproduced on the same contract.- //- // Mirrors `decideThreadCopy` in- // studio/frontend/src/components/assistant-ui/thread-copy-from-store.ts: intervene only when- // the selection spans the whole mounted list AND the store holds more than is mounted. The- // unit tests over there stub `containsNode`; this is the only place the REAL Selection- // semantics are exercised, which is the part that could actually be wrong.+ // COPY-FROM-STORE, the shipping fix, reproduced on the same contract. Mirrors `decideThreadCopy`+ // in thread-copy-from-store.ts: intervene only when the selection spans the whole mounted list+ // AND the store holds more than is mounted. The unit tests there stub `containsNode`; this is+ // the only place the REAL Selection semantics are exercised.+ // Full path: studio/frontend/src/components/assistant-ui/thread-copy-from-store.ts.if (opts.copyFromStore) {viewport.addEventListener("copy", (event) => {const selection = window.getSelection();@@ -261,8 +248,7 @@},};- // The two-rAF paint promise the real harness installs from instruments/frames.js. The- // completeness probe awaits it.+ // The two-rAF paint promise the real harness installs from instruments/frames.js. The completeness probe awaits it.if (!window.__sbNextPaint) {window.__sbNextPaint = () =>new Promise((resolve) =>diff --git a/tests/studio/studiobench/runtime/session.py b/tests/studio/studiobench/runtime/session.pyindex fca5f7c84a9..a5e1062066c 100644--- a/tests/studio/studiobench/runtime/session.py+++ b/tests/studio/studiobench/runtime/session.py@@ -53,10 +53,9 @@ from .readiness import (from .seeder import Seeder, SeededThread, compare_signatures, dom_signature, measure_chars_per_tokenfrom .types import BenchContext, Cell, Paths, Recorder, Window, make_cell_id, new_session_id-# A 1x1 PNG, written to disk once per run so the image-upload action has a real file to attach.-# Generated rather than shipped as a binary asset: the artifact is a zipapp and a tester's-# machine has no fixture directory, and an action that reports `ran = False, no image path` at-# every rung is a hole in the suite that looks like a decision.+# A 1x1 PNG, written once per run so image-upload has a real file. Generated rather than shipped:+# the artifact is a zipapp with no fixture directory, and an action reporting `ran = False` at+# every rung is a hole that looks like a decision._PNG_1X1 = bytes.fromhex("89504e470d0a1a0a0000000d494844520000000100000001080600000""01f15c4890000000d49444154789c6360000002000100ffff03000006"@@ -73,29 +72,21 @@ def ensure_probe_image(paths: Paths) -> Path:IDLE_CALIBRATION_MS = 1500# The rung the seeded-vs-streamed equivalence is CHECKED at. 10K, because both paths are-# affordable there: streaming it takes under a minute at field cadence, and seeding it is instant.-# Below it the thread is one turn and there is nothing to seed; above it, streaming the whole-# thread is hours.+# affordable there; below it there is nothing to seed, above it streaming the thread is hours.EQUIVALENCE_RUNG = "10K"MOUNT_TIMEOUT_S = 180#: How much of the streaming phase the thread must stay pinned for `follows_the_stream` to pass.-#: 0.95 rather than 1.0: the sampler ticks four times a second and a legitimate pin lands a tick-#: or two late after a large append, which is a rendering delay and not a failure to follow. It is-#: paired with `ever_fell_behind`, which is absolute -- a thread that drifted past the tolerance-#: even once fails, however quickly it was yanked back, because being yanked back is itself the-#: other half of the intent contract being broken.+#: 0.95 rather than 1.0, since the sampler ticks four times a second and a legitimate pin can land+#: a tick late. Paired with `ever_fell_behind`, which is absolute: being yanked back is the other+#: half of the intent contract being broken.FOLLOW_PINNED_MIN = 0.95-#: How much of the STREAMING TIME the attached phases must cover before `pinned_fraction` is-#: allowed to stand as a verdict.-#:-#: `pinned_fraction` is computed over attached samples only. With `detached` latching on the first-#: deliberate scroll and never clearing, the shipped film -- which scrolls 1.5s into an 18s opening-#: stream and then streams twice more -- produced a verdict from the first ~3s: 11 attached samples-#: against 72 detached, 13% coverage, reported as 100% pinned and read as "the thread follows the-#: stream". The latch is fixed, and this makes the coverage a condition rather than a footnote, so-#: a future regression that strands the sampler cannot produce a confident pass over a sliver.+#: How much of the STREAMING TIME the attached phases must cover before `pinned_fraction` may+#: stand as a verdict. It is computed over attached samples only, and with `detached` latching on+#: the first deliberate scroll the shipped film produced a verdict from the first ~3s -- 13%+#: coverage, reported as 100% pinned. The latch is fixed; this makes the coverage a condition.+# That scroll is 1.5s into an 18s opening stream.FOLLOW_MIN_STREAM_COVERAGE = 0.50@@ -145,30 +136,19 @@ def follow_verdict(follow: Mapping[str, Any]) -> tuple[bool, dict[str, Any]]:pinned_ok = pinned is not None and pinned >= FOLLOW_PINNED_MINfell_behind = bool(follow.get("ever_fell_behind"))coverage_short = coverage is None or coverage < FOLLOW_MIN_STREAM_COVERAGE- # THE DETACHMENT HAS TO BE THE SCHEDULE'S, AND ONLY A RE-ATTACHMENT PROVES THAT IT WAS.- #- # The waiver's whole justification is that `attached_fraction_of_stream` is set by the film and- # is therefore identical on both arms. Half of it is: the harness scrolls away on a fixed- # schedule. The other half is the BUILD'S -- `scene/dom.js` clears `detached` only when a run- # that began after the gesture is observed at the bottom, so an arm that stops re-pinning when- # the user submits a new turn never re-attaches and every remaining sample lands in the- # detached branch. That branch `return`s before the pinned and fell-behind accounting, so the- # opening stream's `pinned_fraction: 1.0` and `ever_fell_behind: False` survive untouched while- # coverage collapses -- exactly the shape this conjunction was about to waive, on the one- # failure the gate exists to catch. A reply that leaves the viewport is unmounted and stops- # costing anything to paint, so the cell is CHEAPER for the defect, and `readings_by_arm` would- # have admitted it to the ratios against a healthy partner.- #- # `reattachments` is the sampler's own record of the app coming back, and it is >= 1 on every- # cell the carve-out was derived from: the measured 0.481 coverage is unreachable without one,- # since `detached` latches 1.5s into an 18s stream and only a re-attachment resumes counting- # attached samples. Absent (an old payload, or a sampler that never ran) is not evidence of a- # re-attachment, and the absent-instrument case is already waived by `follow_attempted`.+ # THE DETACHMENT HAS TO BE THE SCHEDULE'S, AND ONLY A RE-ATTACHMENT PROVES IT. The waiver rests+ # on `attached_fraction_of_stream` being set by the film and identical on both arms, but half of+ # it is the BUILD'S: `scene/dom.js` clears `detached` only when a run begun after the gesture is+ # observed at the bottom, so an arm that stops re-pinning never re-attaches and every sample+ # lands in the detached branch, leaving `pinned_fraction: 1.0` intact while coverage collapses.+ # A reply that leaves the viewport stops costing anything to paint, so the cell is CHEAPER for+ # the defect and `readings_by_arm` would have admitted it against a healthy partner.+ # `reattachments` is the sampler's own record of the app coming back.reattached = bool(follow.get("reattachments"))- # THE COVERAGE AS A NUMBER, WHATEVER THE VERDICT, next to the floor it is read against. It was- # only ever visible as a pass or a fail, and that is why it took a campaign to notice the floor- # sat above the film's ceiling: every reader saw "FAILED its stream-follow gate", nobody saw- # 0.481. A quantity that decides whether a run is quotable has to be legible as a quantity.+ # THE COVERAGE AS A NUMBER, WHATEVER THE VERDICT, next to the floor it is read against: it was+ # only ever visible as a pass or a fail, which is why it took a campaign to notice the floor sat+ # above the film's ceiling.+ # Every reader saw "FAILED its stream-follow gate"; nobody saw the 0.481.recorded: dict[str, Any] = {"stream_coverage": coverage,"stream_coverage_floor": FOLLOW_MIN_STREAM_COVERAGE,@@ -187,12 +167,10 @@ def follow_verdict(follow: Mapping[str, Any]) -> tuple[bool, dict[str, Any]]:# How long the composer may take to accept the click that starts the film. Not a performance-# budget: the point is that the cell survives and the number gets recorded. See `_press_send`.-# 90s because it has to be well clear of the worst real reading and still bounded, since a click-# that never lands is a different fact from a slow one.+# budget: the point is that the cell survives and the number gets recorded. 90s, well clear of+# the worst real reading and still bounded.COMPOSER_CLICK_TIMEOUT_S = 90-# Above this the log says so out loud, because a multi-second click is the user complaint itself-# and it should not be something you only find by reading the payload afterwards.+# Above this the log says so out loud, because a multi-second click is the user complaint itself.SLOW_COMPOSER_CLICK_MS = 1_000@@ -281,8 +259,7 @@ class Session:yield wfinally:w.t_close_ms = self._now_ms()- # REVERSE order on close, so an instrument that wrapped another's state unwinds after- # the one it wrapped.+ # REVERSE order on close, so an instrument that wrapped another's state unwinds after the one it wrapped.for inst in sorted(self.instruments, key = lambda i: i.name, reverse = True):got = self._safe(inst, "close", w)if got is not None:@@ -340,39 +317,31 @@ class CellRunner:log: Callable[[str], None] = printimage_path: Optional[Path] = Nonecadence: str = "field"- #: Record the NORMALISED signature text beside each digest. Off by default and deliberately- #: so: the text is megabytes per capture. `sweep/parity_null_control.py --hunt` is the only- #: consumer, and it needs it because a digest pair can only say THAT two DOMs differ, never- #: which bytes moved -- and "which bytes" is the entire difference between fixing a- #: normalisation gap and guessing at one.+ #: Record the NORMALISED signature text beside each digest. Off by default: the text is megabytes+ #: per capture, and `sweep/parity_null_control.py --hunt` is the only consumer -- it needs it+ #: because a digest pair can say THAT two DOMs differ, never which bytes moved.parity_raw: bool = False- #: Directory for the per-action viewport PNGs, or None. Set only when the caller intends to- #: produce before/after evidence; the encode is cheap but the files are not free.+ #: Directory for the per-action viewport PNGs, or None. Set only when the caller intends+ #: before/after evidence; the encode is cheap but the files are not free.parity_shots: Optional[str] = None- #: Which ARM this runner drives. It is burned into every filename, because a screenshot pair- #: whose halves cannot be told apart is worse than no pair: both arms share a fixture, a- #: password and a film, so the image itself carries nothing that identifies the side.+ #: Which ARM this runner drives, burned into every filename: both arms share a fixture, a password+ #: and a film, so the image itself carries nothing that identifies the side.arm_label: str = "base"- # Set once the 10K check fails, and it then labels every LARGER rung, which is the whole- # point: those rungs are mostly seeded and their fidelity depends on this one answer.+ # Set once the 10K check fails, and it then labels every LARGER rung: those rungs are mostly+ # seeded and their fidelity depends on this one answer.equivalence_failed: bool = False- # Run the click attribution probe before the film starts. Off by default: it costs a handful- # of seconds at small rungs and a great deal at large ones, and it makes the cell's timings- # incomparable with a cell that did not run it. See `_click_attribution`.+ # Run the click attribution probe before the film. Off by default: it costs a great deal at large+ # rungs and makes the cell's timings incomparable with a cell that did not run it.click_probe: bool = False- # WHICH READINESS GATE. `full` is the default and is what every normal arm runs: every seeded- # message must be mounted, plus the settle and end-present conditions the gate gained. An arm- # that mounts a WINDOW on purpose sets `windowed` and is then held to a different set of- # conditions, none of them weaker. See runtime/readiness.py.- #- # It is a per-TARGET setting, so a base-versus-virtualised A/B runs the base arm on `full` and- # only the treatment arm on `windowed`, and the base arm keeps the strict gate it always had.+ # WHICH READINESS GATE. `full` is the default every normal arm runs; an arm that mounts a WINDOW+ # on purpose sets `windowed` and is held to a different set of conditions, none of them weaker.+ # See runtime/readiness.py.+ # Per TARGET, so a base-versus-virtualised A/B keeps the base arm on its strict gate.readiness_mode: str = MODE_FULL- # Scroll a windowed thread to its top once per cell, before the measured window, to prove the- # arm still holds the head of the conversation. Costs a full traversal, so it is off for- # `full` (where the whole thread is mounted and there is nothing to prove) and on by default- # for `windowed`, where it is the only check that separates a virtualised thread from one that- # has lost most of its messages.+ # Scroll a windowed thread to its top once per cell, before the measured window, to prove the arm+ # still holds the head of the conversation. Costs a full traversal, so it is off for `full` and+ # on by default for `windowed`, where it is the only check that separates a virtualised thread+ # from one that has lost most of its messages.completeness_probe: Optional[bool] = Nonedef run(self, cell: Cell, plan: RungPlan) -> dict:@@ -398,9 +367,8 @@ class CellRunner:"target_tokens": plan.target_tokens,"instruments": {},}- # Cleared HERE, not beside the click that produces it, so the preservation in the `finally`- # below cannot attach the PREVIOUS cell's attribution to a cell that died before its own- # probe ran. A reading filed under the wrong cell id is worse than a missing one.+ # Cleared HERE, not beside the click that produces it, so the preservation in the `finally` below+ # cannot attach the PREVIOUS cell's attribution to a cell that died before its own probe ran.self._click_attribution_result = Nonetry:self._run_inner(cell, plan, row)@@ -412,10 +380,8 @@ class CellRunner:"traceback": traceback.format_exc()[-3000:],}if isinstance(exc, ThreadNotReady):- # WHICH CONDITION, not just that it timed out. The old gate's message was "the- # thread mounted 9 of 18 messages", which named a count and left the reader to- # guess whether the app was slow, the thread was short or the arm mounts a window- # on purpose. The verdict now carries every condition and the last probe reading.+ # WHICH CONDITION, not just that it timed out: the old message named a count and left the reader+ # to guess whether the app was slow, the thread short, or the arm windowed on purpose.row["failure"]["readiness"] = exc.detailrow["readiness"] = exc.detailrec.gate(@@ -430,35 +396,23 @@ class CellRunner:dump_diagnostics(page, self.paths.logs, f"fail_{cell.cell_id}", self.log)finally:row["instruments"].update(s.each_instrument("end_cell", cell))- # A cell that could not complete is a FIRST-CLASS RESULT with its failure mode and its- # RSS at death, not a gap in the table.+ # A cell that could not complete is a FIRST-CLASS RESULT with its failure mode and its RSS at+ # death, not a gap in the table.rss = row["instruments"].get("rss") or {}row["rss_at_death_mb"] = rss.get("rss_peak_mb") if not row["completed"] else None- # AND SO IS THE PROBE THAT ALREADY RAN. `--click-probe` finishes inside `_press_send`,- # and everything after it there -- the `page.fill`, the send button lookup and its- # click -- still runs under the default 8s action timeout, which is exactly what a- # large rung exceeds. Assigned only on the way out of `_run_inner`, the whole- # attribution was dropped from the cell it was measured for, and unlike- # `composer_click_ms` it has no window row of its own to survive in: nothing else in- # the payload records it. So it is filed from here, on both paths.+ # AND SO IS THE PROBE THAT ALREADY RAN: `--click-probe` finishes inside `_press_send`, and+ # everything after it there still runs under the default 8s action timeout, which a large rung+ # exceeds. Assigned only on the way out of `_run_inner`, the attribution was dropped from the+ # cell it was measured for, and unlike `composer_click_ms` it has no window row to survive in.if self._click_attribution_result is not None:row["click_attribution"] = self._click_attribution_resultrec.emit(row)- # A TERMINAL MARKER FOR A CELL THAT DID NOT FINISH, so a reader scanning FORWARD can- # discard its windows without having to join backwards to the cell row.- #- # `window` rows are written as the film runs; the `cell` row is written when the film- # ends. An in-flight or aborted cell therefore leaves a complete-looking set of window- # rows behind with no completed cell owning them, and an analysis that reads windows- # directly picks them up. `floor_table.cell_metrics` has always guarded this; nothing- # else did.- #- # This cost a headline. Reading `stream:gap` windows without the guard reported the- # 1M rung at 28.7 fps and 21.8% of frames over 33 ms against a 46.7 fps baseline, an- # apparent large regression at the rung closest to the user complaint, drawn entirely- # from a cell that had not finished. An incomplete cell is not a shorter film but a- # different one: at 1M the completed cell emitted 6 `stream:gap` windows against 17 at- # 100K, so an in-flight cell contributes whichever phase it had reached.+ # A TERMINAL MARKER FOR A CELL THAT DID NOT FINISH, so a reader scanning FORWARD can discard its+ # windows without joining backwards. `window` rows are written as the film runs and the `cell`+ # row when it ends, so an aborted cell leaves a complete-looking set nothing owns. It cost a+ # headline: reading `stream:gap` windows without the guard reported the 1M rung at 28.7 fps+ # against a 46.7 fps baseline, drawn entirely from an unfinished cell.+ # And 21.8% of frames over 33 ms.if not row["completed"]:rec.emit({@@ -495,8 +449,8 @@ class CellRunner:raise ValueError(f"unknown readiness mode {self.readiness_mode!r}")readiness = self._wait_for_thread(page, seeded)row["readiness"] = readiness.as_dict()- # RECORDED AS A GATE, so no reader can pick up a windowed cell's frame rate without also- # seeing that its readiness was established a different way.+ # RECORDED AS A GATE, so no reader can pick up a windowed cell's frame rate without also seeing+ # that its readiness was established a different way.rec.gate(f"thread_ready:{self.readiness_mode}",True,@@ -504,9 +458,8 @@ class CellRunner:cell_id = cell.cell_id,)- # THE COMPLETENESS PROBE, before the idle window and therefore before anything is measured.- # It scrolls the whole thread, which mounts rows and dirties the page, so it has to be- # followed by the idle window rather than preceding a measurement directly.+ # THE COMPLETENESS PROBE, before the idle window and therefore before anything is measured. It+ # scrolls the whole thread, which mounts rows and dirties the page, so the idle window follows.do_probe = (self.completeness_probeif self.completeness_probe is not None@@ -521,8 +474,8 @@ class CellRunner:)row["completeness"] = completenessrecord_completeness_gate(rec, cell, completeness)- # Back to the resting state the gate described, or the idle calibration below runs- # against a page that is still settling from the traversal.+ # Back to the resting state the gate described, or the idle calibration below runs against a page+ # still settling from the traversal.self._wait_for_thread(page, seeded)# ── the enforced idle window ────────────────────────────────@@ -536,9 +489,8 @@ class CellRunner:w.note("clamp", clamp)row["clamp"] = clampif clamp.get("clampMs") is None:- # NOT fatal, and NOT silently zero. Blocked time is a subtraction against this floor,- # so without it busy_pct is null with the reason attached and every other column- # stands.+ # NOT fatal, and NOT silently zero: blocked time is a subtraction against this floor, so without+ # it busy_pct is null with the reason attached and every other column stands.self.log(f" timer clamp NOT established: {clamp.get('reason')}")rec.gate("timer_clamp", False, clamp, cell_id = cell.cell_id)else:@@ -575,7 +527,6 @@ class CellRunner:f"(measured via {cpt.get('source')})")- # ── the film ────────────────────────────────────────────────unit = plan.streamed_unitself.pacer.reset()self.pacer.load(@@ -592,24 +543,19 @@ class CellRunner:f"content chars, cadence {self.cadence}, {expected_ms / 1000:.0f}s expected")- # RESET THE FOLLOW SAMPLER FOR THIS CELL, immediately before the film starts.- #- # Necessary because the counters were just made to survive a navigation, by persisting to- # sessionStorage -- and a cell boundary IS a navigation, to the next seeded thread on the- # same origin. Without this, cell 2 reports cell 1's samples plus its own, cell 3 reports- # all three, and a single bad cell early in a session poisons every reading after it while- # each one still looks like a per-cell number.+ # RESET THE FOLLOW SAMPLER FOR THIS CELL, immediately before the film starts: the counters now+ # survive a navigation via sessionStorage, and a cell boundary IS a navigation, so without this+ # cell 2 reports cell 1's samples plus its own and one bad cell poisons every later reading.with contextlib.suppress(Exception):page.evaluate("() => window.__sb.follow && window.__sb.follow.reset()")before_metrics = cdp_metrics(s.ctx.cdp)self._composer_click_ms = None- # `click_attribution` is NOT filed here. It is filed in `run`'s `finally`, because a cell- # that dies after the probe has to keep it: see there.+ # `click_attribution` is NOT filed here but in `run`'s `finally`, because a cell that dies after+ # the probe has to keep it.t0 = self._press_send(page)- # On the cell rather than in `actions`, because it happens before the first slot opens and- # filing it as an action would put a reading outside the film into a list the scoring layer- # pairs by slot. It is still a per-cell timing and grows with the rung like any other.+ # On the cell rather than in `actions`, because it happens before the first slot opens and filing+ # it as an action would put a reading outside the film into a list scoring pairs by slot.row["composer_click_ms"] = self._composer_click_msscene = scene_schedule.SCENES.get(self.tier, scene_schedule.QUICK)@@ -630,15 +576,13 @@ class CellRunner:"parity_shots": self.parity_shots,"arm_label": self.arm_label,"image_path": str(self.image_path) if self.image_path else None,- # The follow-up turns `send_turn` streams mid-film, and- # the pacer it reloads to serve them.+ # The follow-up turns `send_turn` streams mid-film, and the pacer it reloads to serve them."_pacer": self.pacer,"_stream_queue": [{"reasoning": u.reasoning, "content": u.content, "kind": u.kind}for u in (plan.follow_up_units or [])],- # Shared and MUTABLE, so consecutive sends advance- # through the queue. See send_turn.+ # Shared and MUTABLE, so consecutive sends advance through the queue. See send_turn."_stream_cursor": {"i": 0},"_input_instrument": next((i for i in s.instruments if i.name == "input"), None),},@@ -654,15 +598,11 @@ class CellRunner:drained = self._drain_stream(page, expected_ms)w.note("drained", drained)row["stream"] = drained- # DID THE THREAD FOLLOW THE STREAM? Read once, here, after the last window of the film has- # closed, so the reading is charged to nothing. See the sampler in scene/dom.js.- #- # This is a GATE, not a column, because of the specific way it fails. A thread that stops- # following lets the streamed message leave the viewport; a windowed list then unmounts- # it, and the streaming cost collapses because the renderer is no longer rendering the- # thing being measured. The frame rate that comes out is excellent and is about nothing.- # A number with a caveat attached is still a number people quote without the caveat, so- # the caveat is a gate row that a reader has to step over.+ # DID THE THREAD FOLLOW THE STREAM? Read once, here, after the last window has closed, so the+ # reading is charged to nothing. A GATE, not a column, because of how it fails: a thread that+ # stops following lets the streamed message leave the viewport, a windowed list unmounts it, and+ # the streaming cost collapses -- an excellent frame rate about nothing. A number with a caveat+ # attached is still quoted without the caveat.follow = self._read_follow(page)row["follow"] = followpinned = follow.get("pinned_fraction")@@ -670,29 +610,18 @@ class CellRunner:passed, recorded = follow_verdict(follow)follow.update(recorded)rec.gate("follows_the_stream", passed, follow, cell_id = cell.cell_id)- # THE OTHER HALF OF THE CONTRACT, RECORDED AND DELIBERATELY NOT GATED.- #- # It was a gate for one run and it failed on BOTH arms at nearly the same rate (32 of 79- # on the base, 38 of 85 on the treatment), which is the signature of a reading about the- # film rather than about the build. The film re-pins legitimately and often: `send_turn`- # and `stop_generation` each START A RUN, and pinning to the bottom on run start is the- # intended behaviour, not a violation; `scroll_after` ends its own gesture near the bottom- # by design. Separating a legitimate re-pin from a yank requires knowing which pins the- # app was ASKED for, which this sampler does not know.+ # THE OTHER HALF OF THE CONTRACT, RECORDED AND DELIBERATELY NOT GATED. It was a gate for one run+ # and failed on BOTH arms at nearly the same rate, the signature of a reading about the film:+ # `send_turn` and `stop_generation` each START A RUN, where pinning is intended, and+ # `scroll_after` ends its gesture near the bottom by design. Separating a legitimate re-pin from+ # a yank needs to know which pins the app was ASKED for, which this sampler does not know, so it+ # is a per-arm figure compared BETWEEN arms where the confounds cancel.#- # So it is reported as a per-arm figure to be compared BETWEEN arms, where the confounds- # are common to both and cancel, and it carries the reason it is not a verdict. Gating on- # it would have failed the shipped build.row["scroll_intent"] = {- # THE ATTESTATION, without which this block fails the bare-zero ban and takes the whole- # report with it. `yanked_back_samples: 0` beside a non-zero `detached_samples` is the- # GOOD outcome -- the user scrolled away and the app left them there -- and the walker- # in scoring/schema.py cannot tell that from a counter nobody wrote. `follow` itself- # already carries `follow_attempted` and is covered by it; this block is derived from- # the same read and had nothing, so a real CI session refused to render at all with- # `bare zeros found: $.cells[0].scroll_intent.yanked_back_samples = 0`. False here- # rather than absent: the sampler that was never installed reports None for both- # counters, so "not measured" stays distinguishable from "measured zero".+ # THE ATTESTATION, without which this block fails the bare-zero ban: `yanked_back_samples: 0`+ # beside a non-zero `detached_samples` is the GOOD outcome, and the walker in scoring/schema.py+ # cannot tell that from a counter nobody wrote. False here rather than absent, so "not measured"+ # stays distinguishable from "measured zero"."follow_attempted": bool(follow.get("follow_attempted")),"detached_samples": follow.get("detached_samples"),"yanked_back_samples": follow.get("yanked_back_samples"),@@ -722,11 +651,10 @@ class CellRunner:f"after the user scrolled away"+ (" -- THE USER WAS YANKED DOWN" if follow.get("yanked_after_scroll") else ""))- # EVERY STREAM THE CELL SERVED, not just the last one. `last_stats()` describes whichever- # turn finished last, so for a multi-turn cell it says nothing at all about the opening- # reply -- and the opening reply is the one the rung is named for. Everything stays under- # the `pacer` key because that subtree is exempt from the payload's bare-zero rule: a- # pacer counter of 0 is a true reading, not a missing one.+ # EVERY STREAM THE CELL SERVED, not just the last one: `last_stats()` describes whichever turn+ # finished last, so for a multi-turn cell it says nothing about the opening reply the rung is+ # named for. Everything stays under `pacer` because that subtree is exempt from the bare-zero+ # rule.streams = self.pacer.all_stats()planned = self._planned_streams(cell, plan, row)row["pacer"] = {@@ -734,58 +662,32 @@ class CellRunner:"streams": streams,"check": check_planned_streams(streams, planned),}- # A REPLY THAT NEVER FINISHED IS A FAILED CELL, not a completed one with a note.- #- # `_drain_stream` reports rather than raises, and the value it reports was recorded here- # and read by nothing: no gate, no report column, no `--assert-liveness` check and no- # exit code. So the one state this tool exists to catch -- the app still generating three- # times past its own cadence and 120 s beyond that, after the whole film has run -- came- # back as `completed: true`, `ok` in the summary and exit 0, with its actions and its- # frame windows scored and paired into the A/B ratio against an arm that DID finish.- # That is the crash-beats-limp rule inverted: a build that cannot finish reads as a build- # that had nothing to say.- #- # Raised AFTER the drain reading AND the pacer's own counters are on the row, so the two- # facts that say WHY it never finished -- how long was waited, and how much the pacer- # actually delivered -- ship in the same row as the failure. `CellRunner.run` catches- # this, records `failure`, dumps the diagnostics and keeps the cell as a first-class- # incomplete result; the rung then scores INCOMPLETE, which is exactly how this harness- # reports a build that died at 500K. The censuses below are not taken, because a census of- # a thread that is still growing describes nothing that was measured.+ # A REPLY THAT NEVER FINISHED IS A FAILED CELL, not a completed one with a note. `_drain_stream`+ # reports rather than raises, and the value was read by nothing, so an app still generating three+ # times past its own cadence came back as `completed: true`, exit 0, paired into the A/B ratio.+ # And 120 s beyond that, after the whole film had run.+ # Raised AFTER the drain reading and the pacer's counters are on the row, so how long was waited+ # ships with the failure. The censuses below are not taken, because a census of a still-growing+ # thread describes nothing that was measured.if not drained.get("finished"):raise RuntimeError(f"the reply never finished: {drained.get('reason') or 'the run was still going'} "f"({drained.get('drain_ms')}ms waited, {drained.get('expected_ms')}ms expected)")- # A CELL THAT DID NOT STREAM WHAT IT PLANNED IS A FAILED CELL, for the same reason.- #- # The drain check above only asks whether the UI stopped running, and a later turn that- # finishes satisfies it on behalf of an earlier one that did not. So an opening reply that- # disconnected part way through left a complete-looking cell whose thread was thousands of- # characters short of its rung, averaged into the A/B ratio against arms that streamed in- # full. Raised here, after the check is on the row, so the reason and the per-turn evidence- # ship with the failure and the cell is excluded by name rather than quietly included.+ # A CELL THAT DID NOT STREAM WHAT IT PLANNED IS A FAILED CELL, for the same reason: the drain+ # check only asks whether the UI stopped running, and a later turn that finishes satisfies it on+ # behalf of an earlier one that did not, so an opening reply that disconnected left a+ # complete-looking cell thousands of characters short of its rung.check = row["pacer"]["check"]if check["checked"] and not check["ok"]:self.log(f" the cell did not stream what it planned: {check['reason']}")raise RuntimeError(f"the cell did not stream what it planned: {check['reason']}")- # AND THE SAME RULE FOR THE TURN THAT STREAMED BUT NEVER LANDED.- #- # The stream check asks whether the bytes went out. `send_turn` asserts something the pacer- # cannot see: that the thread GREW. A send whose bytes were served in full but whose reply- # never joined the thread leaves every later action, the peak census and the equivalence- # mirror reading a thread one turn short, and the stream check alone would pass it.- #- # Scoped to `send_turn` DELIBERATELY, and this is the whole of the generalisation. An- # action whose own assertion fails already has its own timing voided by- # `scoring/from_payload._action_measure`, which returns `Measure.failed` for- # `expect_ok is False` and says so. What that does NOT cover is an action whose failure- # changed the workload the REST of the cell measured, and `send_turn` is the only one that- # can: it is the only action whose outcome decides how much content the thread carries for- # everything after it. A `select_text` that selected nothing or a `message_menu` that did- # not open leaves the workload intact, and failing the cell for those would throw away a- # whole cell's frame readings -- which really were taken, over the real thread -- for a- # gesture that missed.+ # AND THE SAME RULE FOR THE TURN THAT STREAMED BUT NEVER LANDED: the stream check asks whether+ # the bytes went out, while `send_turn` asserts that the thread GREW. Scoped to `send_turn`+ # DELIBERATELY -- an action whose own assertion fails already has its timing voided, and what+ # that does not cover is an action whose failure changed the workload the REST of the cell+ # measured. `send_turn` is the only one that can.+ # The covered case is `expect_ok is False`.missed_turns = [afor a in (row["actions"] or [])@@ -802,38 +704,26 @@ class CellRunner:row["census_after"] = dom_signature(page)row["cdp"] = cdp_counters(before_metrics, cdp_metrics(s.ctx.cdp))- # THE PEAK, over every window's census, not the state at the end of the film.- #- # The film ENDS with thread_reopen and delete_message, so an end-of-cell census reports- # the thread the benchmark has just deleted. The first working run recorded 0 assistant- # messages and 0 characters against a reply the pacer's own log proved it had delivered in- # full: 150 chunks, 3,581 characters, at exactly the 73ms cadence. The occupancy that- # every per-action cost has to be read against is the peak, and it is now recovered from- # the per-window censuses rather than from a single reading taken at the worst moment.+ # THE PEAK, over every window's census, not the state at the end of the film: the film ENDS with+ # thread_reopen and delete_message, so an end-of-cell census reports the thread the benchmark has+ # just deleted. The first working run recorded 0 assistant messages against a delivered reply.+ # The pacer's own log proved it: 150 chunks, 3,581 characters, at exactly the 73 ms cadence.+ # 0 messages and 0 characters.censuses = [w.get("census") for w in row["actions"] if isinstance(w.get("census"), dict)]censuses = [c for c in censuses if c.get("elements")]peak = max(censuses, key = lambda c: c.get("elements", 0)) if censuses else {}row["census_peak"] = peakrow["census_peak_attempted"] = bool(censuses)- # WHICH ACTION THE PEAK CAME FROM, and a standing refusal to compare it across arms.- #- # `census_peak` is whichever per-action census had the most elements. The census attached- # to an action is taken after the action returns, and `reasoning_toggle` opens every pane- # and then closes them again, so that census races the close. Which action wins the max()- # therefore depends on the race and DIFFERS BETWEEN ARMS.- #- # Measured on a null control -- the same bundle served on both arms -- the winner flipped- # between `settings` (panes collapsed, 64,648 elements) and `reasoning_toggle` (panes open,- # 106,067), a 70.1% swing in `highlight_spans` WITHIN one arm.- #- # This produced a published wrong number: main was reported as mounting 48% more Shiki- # spans than a pre-campaign baseline, and an attribution bisect was nearly commissioned to- # find the cause. Settled, the two trees mount the same document to within 0.3%.- #- # It stays in the payload because it is useful as a diagnostic high-water mark. It carries- # its provenance and an explicit refusal so that no analysis can quote it across arms- # without stepping over a flag that says not to.+ # WHICH ACTION THE PEAK CAME FROM, and a standing refusal to compare it across arms. The census+ # attached to an action is taken after it returns, and `reasoning_toggle` opens every pane and+ # closes them again, so that census races the close and which action wins the max() DIFFERS+ # BETWEEN ARMS. Measured on a null control the winner flipped between two actions, a 70.1% swing+ # WITHIN one arm, which produced a published wrong number. Kept as a diagnostic high-water mark,+ # carrying its provenance and an explicit refusal.+ # The winner flipped between `settings` at 64,648 elements and `reasoning_toggle` at 106,067,+ # published as main mounting 48% more Shiki spans.+ # Settled, the two trees mount the same document to within 0.3%.peak_from = next((w.get("action")@@ -858,10 +748,9 @@ class CellRunner:if chars is None:chars = page.evaluate("() => window.__sb.dom.assistantChars()")row["assistant_chars_in_dom"] = chars- # The span density the fixture ACHIEVED, measured in the DOM rather than assumed. The- # field capture ran 5.6 characters per span; a corpus that lands far from that is not- # standing in for the same highlighter load per character, and the report should say so- # rather than quietly compare two different workloads.+ # The span density the fixture ACHIEVED, measured in the DOM rather than assumed: the field+ # capture ran 5.6 characters per span, and a corpus far from that is not standing in for the same+ # highlighter load per character.row["chars_per_span"] = round(chars / spans, 2) if spans else Nonerow["chars_per_span_target"] = 5.6self.log(@@ -882,9 +771,9 @@ class CellRunner:cell_id = cell.cell_id,)if not eq.get("equivalent"):- # A FINDING, printed, not a bug to hide. It says exactly which of this tool's- # numbers are about the streaming path and which are about a thread that was put- # there, and it is the reason the higher rungs carry a fidelity label at all.+ # A FINDING, printed, not a bug to hide: it says which of this tool's numbers are about the+ # streaming path and which are about a thread that was put there, and it is why the higher rungs+ # carry a fidelity label at all.self.log(" SEEDED IS NOT EQUIVALENT TO STREAMED at the 10K rung. Rungs above it ""are labelled fidelity: seeded_only."@@ -901,9 +790,8 @@ class CellRunner:" seeded and streamed agree on CONTENT at the 10K rung within "f"{eq['tolerance']:.0%}")- # Passing the content gate is not the same as the two threads being identical,- # and the difference is large enough that leaving it unsaid would mislead: a- # seeded rung carries the same rendered content and materially less mounted DOM.+ # Passing the content gate is not the same as the two threads being identical: a seeded rung+ # carries the same rendered content and materially less mounted DOM.fields = eq.get("fields") or {}for key in ("reasoning_spans", "highlight_spans", "assistant_chars"):field = fields.get(key) or {}@@ -995,17 +883,11 @@ class CellRunner:"""s = self.sessionpage = s.ctx.page- # THE STREAMED SIDE IS THE PEAK, AND THE PEAK IS TAKEN AT AN UNSTABLE MOMENT. Recorded on- # the row rather than left implicit, because the seeded side below is read after an- # explicit 4 s wait -- a settled moment -- so this gate differences a racing census against- # a stable one. That is the same shape as the defect that made `census_peak` unquotable- # across arms.- #- # It is NOT the same harm and the behaviour is deliberately left alone here: both sides of- # this comparison come from ONE cell on ONE build, so the instability widens the gate's- # tolerance rather than pointing a difference in a direction. Changing which census feeds- # it would change what the gate asserts, and that cannot be justified without measuring it- # against a browser. Said out loud so the next reader does not have to rediscover it.+ # THE STREAMED SIDE IS THE PEAK, AND THE PEAK IS TAKEN AT AN UNSTABLE MOMENT, recorded on the row+ # because the seeded side below is read after an explicit 4 s wait -- so this gate differences a+ # racing census against a stable one, the same shape as the defect that made `census_peak`+ # unquotable across arms. NOT the same harm: both sides come from ONE cell on ONE build, so the+ # instability widens the tolerance rather than pointing a difference in a direction.streamed = row.get("census_peak") or row.get("census_after") or {}streamed_from = "census_peak" if row.get("census_peak") else "census_after"follow_ups = self._streamed_follow_ups(plan, row)@@ -1042,20 +924,17 @@ class CellRunner:out["seeded_census_settled"] = Trueout["readiness_mode"] = self.readiness_modeif self.readiness_mode == MODE_WINDOWED:- # SAID OUT LOUD RATHER THAN SCORED QUIETLY. Both sides of this check are loaded by the- # SAME build, so under a windowed arm both censuses count the mounted window and not- # the thread. The comparison is still like for like and its pass still means something- # -- the two paths render the same window the same way -- but it is no longer evidence- # that seeding reproduces the whole streamed thread, because neither side has the whole- # thread in the DOM to compare.+ # SAID OUT LOUD RATHER THAN SCORED QUIETLY: both sides are loaded by the SAME build, so under a+ # windowed arm both censuses count the mounted window. The comparison is still like for like, but+ # it is no longer evidence that seeding reproduces the whole streamed thread.out["scope"] = "the mounted window only, not the whole thread"out["caveat"] = ("this arm mounts a window, so `assistant_messages`, `content_spans` and ""`content_code_blocks` are counts over what is mounted at the end of the thread. ""A pass is equivalence of the WINDOW, not of the thread.")- # What the mirror was built from, so a drift can be read against the corpus it compared- # rather than against an assumption about which turns were in the thread.+ # What the mirror was built from, so a drift can be read against the corpus it compared rather+ # than an assumption about which turns were in the thread.out["mirrored_follow_ups"] = len(follow_ups)out["planned_follow_ups"] = len(plan.follow_up_units or [])return out@@ -1142,47 +1021,29 @@ class CellRunner:page.evaluate("() => document.body.offsetHeight")return (time.monotonic() - started) * 1000.0- # FIRST, before anything else touches the page: the same trivial operation N times.- #- # This exists because the biggest number in the whole ladder is one I could not attribute.- # The first thing touched after a large thread mounts costs 11 to 24 seconds, and in every- # probe the cost vanished, because whatever ran first absorbed it and by the time the- # interesting path ran the page was warm. Measuring the decay directly is the way out: if- # the first reading is seconds and the rest are milliseconds, the cost is ONE TIME and its- # size is the first reading minus the steady state. If they are all expensive, it is a- # per-interaction cost and the ladder's number was never a one-off.- #- # A no-op body on purpose. Every reading is the same work, so any difference between them- # is the page's state and not the operation.+ # FIRST, before anything else touches the page: the same trivial operation N times. The biggest+ # number in the ladder is one nobody could attribute -- the first thing touched after a large+ # thread mounts costs 11 to 24 seconds, and in every probe the cost vanished because whatever ran+ # first absorbed it. Measuring the decay directly is the way out: a first reading in seconds and+ # the rest in milliseconds means the cost is ONE TIME. A no-op body on purpose.decay = [settled(lambda: None) for _ in range(5)]out: dict[str, Any] = {- # The harness layer's attestation, and it is load-bearing rather than decorative.- # `scoring/schema._walk_for_bare_zeros` rejects any bare zero that is not covered by a- # sibling `*_attempted` flag, and this block has legitimate zeros: a thread with no- # mounted code blocks records `code_token_spans: 0`, and `blur_inpage_ms` and- # `forced_layout_ms` come from `performance.now()`, which Chromium coarsens to 100 us- # outside a cross-origin-isolated context, so an operation shorter than that reads- # exactly 0. Without this flag `--report` refuses the whole payload of a completed- # probe run.+ # The harness layer's attestation, load-bearing rather than decorative:+ # `scoring/schema._walk_for_bare_zeros` rejects a bare zero with no sibling `*_attempted` flag,+ # and this block has legitimate zeros -- a thread with no mounted code blocks records+ # `code_token_spans: 0`, and Chromium coarsens `performance.now()` to 100 us.+ # Which is what `forced_layout_ms` is built from."click_attribution_attempted": True,"first_touch_ms": decay[0],"settled_touch_ms": min(decay[1:]),"touch_decay_ms": [round(v, 1) for v in decay],}- # The blur, timed, and timed again from INSIDE the page.- #- # The decay series above says the first touch after mount costs 10.6 ms at 500K, so there- # is no expensive first interaction. Yet the reading taken right after the first blur came- # back at 10,052 ms and 10,017 ms in two of three runs. Two things about that: it sits- # between the cheap decay series and the cheap readings that follow, so the blur is what it- # attaches to; and 10.0 seconds to three digits is the shape of a TIMEOUT, not of work that- # happens to take ten seconds.- #- # `blur_inpage_ms` decides which. It runs the same blur and the same forced layout inside a- # single evaluate and times them with `performance.now()`, so the protocol is excluded. If- # the page reports ten seconds, it is real work. If the page reports a millisecond while- # the outer reading is ten seconds, the ten seconds is the driver or the transport and not- # the app, and must never be quoted as a user cost.+ # The blur, timed, and timed again from INSIDE the page. The decay series says the first touch+ # after mount costs 10.6 ms at 500K, yet the reading right after the first blur came back at+ # 10,052 ms in two of three runs -- and 10.0 seconds to three digits is the shape of a TIMEOUT.+ # And 10,017 ms in the other.+ # `blur_inpage_ms` decides which: it runs the same blur inside one evaluate, so if the page+ # reports a millisecond while the outer reading is ten seconds, the ten seconds is the driver.out["blur_outer_ms"] = settled(lambda: page.evaluate("() => document.activeElement && document.activeElement.blur()"))@@ -1210,14 +1071,12 @@ class CellRunner:page.mouse.move(2, 2)page.wait_for_timeout(250)out["hover_thread_ms"] = settled(lambda: page.mouse.move(x, 300))- # The reading that decides what `roundtrip_ms` meant. If the first forced layout of a huge- # thread is expensive and every later one is cheap, this comes back near zero and the cost- # is a ONE-TIME layout of the mounted thread, not a per-interaction cost. If it is- # expensive again, every interaction pays it and the story is the opposite.+ # The reading that decides what `roundtrip_ms` meant: near zero means the cost is a ONE-TIME+ # layout of the mounted thread, expensive again means every interaction pays it.blur()out["roundtrip_again_ms"] = settled(lambda: None)- # Measured INSIDE the page, so the protocol round trip is not in the number. `offsetHeight`- # is read after a write that dirties layout, so it cannot be served from a clean tree.+ # Measured INSIDE the page, so the protocol round trip is not in the number. `offsetHeight` is+ # read after a write that dirties layout, so it cannot be served from a clean tree.out["forced_layout_ms"] = page.evaluate("() => { const t = performance.now();"" document.body.style.minHeight = (1 + Math.random()) + 'px';"@@ -1268,24 +1127,16 @@ class CellRunner:page.wait_for_selector(selector, timeout = 60_000)if self.click_probe:self._click_attribution_result = self._click_attribution(page, selector)- # In a window, so every instrument covers it. At 500K this single click is the largest- # cost in the whole run by an order of magnitude, and it was the one moment the tool could- # not see inside: no window, so no frame recorder, no CPU profile, no RSS delta. A cost- # that big being outside every instrument is how it stayed unattributed.- #- # `setup`, NOT `action`. The scoring layer pools every non-excluded window of the cell into- # `max_frame_ms`, `jank_index` and `time_in_jank_pct`, and this window is mostly Playwright's- # own injected actionability script running ON THE PAGE'S MAIN THREAD -- it blocks frames- # exactly as app work would, so the frame recorder cannot tell them apart. Filed as an- # `action` it would put an 11 s driver stall into three weighted headline metrics whose- # `max_frame_ms` anchor calls 2,000 ms the worst case, on EVERY run including the ones that- # never asked for --click-probe. It is measured, reported and kept out of the film.+ # In a window, so every instrument covers it: at 500K this single click is the largest cost in the+ # run by an order of magnitude and was the one moment the tool could not see inside. `setup`, NOT+ # `action`: the scoring layer pools every non-excluded window into the three frame metrics, and+ # this window is mostly Playwright's own actionability script running ON THE PAGE'S MAIN THREAD,+ # so filed as an `action` it would put an 11 s driver stall into three weighted headline metrics.+ # It would peg `max_frame_ms`, `jank_index` and `time_in_jank_pct`.with self.session.window("setup:composer_click", kind = "setup"):- # Timed INSIDE the window, like `Window.duration_ms` itself: the session opens every- # instrument before this block and closes them after it, and at instrument level 1-3- # those hooks stop a CPU profile, collect coverage and write and analyse a trace.- # Timed around the `with`, `composer_click_ms` would grow with the instrument level- # and stop being the click.+ # Timed INSIDE the window, like `Window.duration_ms`: the session opens every instrument before+ # this block and closes them after, and at instrument level 1-3 those hooks stop a CPU profile,+ # collect coverage and analyse a trace, so timing around the `with` would grow with the level.clicked_at = time.monotonic()page.click(selector, timeout = COMPOSER_CLICK_TIMEOUT_S * 1000)self._composer_click_ms = (time.monotonic() - clicked_at) * 1000.0@@ -1301,16 +1152,15 @@ class CellRunner:raise RuntimeError("the send button is not on the page, so no reply can be started")t0 = time.monotonic()send.click()- # The composer must be EMPTY for the rest of the film, or the Stop control is replaced by- # a Queue control and the stop action presses the wrong button. Sending clears it, but the- # keystroke action refills it, which is why the stop action clears it again itself.+ # The composer must be EMPTY for the rest of the film, or the Stop control is replaced by a Queue+ # control and the stop action presses the wrong button. Sending clears it, but the keystroke+ # action refills it, which is why the stop action clears it again itself.return t0def _drain_stream(self, page, expected_ms: float) -> dict:"""Wait for the run to end, or say plainly that it did not."""- # Generous: the whole point of deficit scheduling is that the stream's own duration is- # machine-independent, so anything much past it is the RENDERER failing to keep up, which- # is a finding rather than a timeout to paper over.+ # Generous: deficit scheduling makes the stream's own duration machine-independent, so anything+ # much past it is the RENDERER failing to keep up, which is a finding rather than a timeout.deadline = time.monotonic() + (expected_ms / 1000) * 3 + 120started = time.monotonic()while time.monotonic() < deadline:@@ -1340,10 +1190,9 @@ def make_context(out_lock = None,) -> tuple[BenchContext, Session]:session_id = new_session_id()- # THE LOCK THE CALLER IS ALREADY HOLDING. `run()` takes the output directory before it archives- # a payload or installs an Unsloth, so the `Recorder` adopts that lock rather than opening a- # second one against the same path. Without a caller's lock it takes its own, which is what- # every test and every other consumer of a payload does.+ # THE LOCK THE CALLER IS ALREADY HOLDING: `run()` takes the output directory before it archives a+ # payload, so the `Recorder` adopts that lock rather than opening a second against the same path.+ # Without a caller's lock it takes its own.recorder = Recorder(paths.payload_jsonl, session_id, lock = out_lock)ctx = BenchContext(browser = browser_bundle.browser,@@ -1376,12 +1225,11 @@ def make_context(return ctx, Session(ctx = ctx, instruments = instruments)-#: The sources that may SIZE a rung. `measure_chars_per_token` also has a last-resort-#: whitespace-and-punctuation estimate, which it labels itself as "off by tens of percent on dense-#: code" -- and this corpus is mostly dense code, where that estimate reads 6.7 against tiktoken's-#: 3.3. Sizing the ladder from it would not make the axis honest, it would move the error and, past+#: The sources that may SIZE a rung. `measure_chars_per_token`'s last-resort whitespace estimate+#: labels itself "off by tens of percent on dense code", and this corpus is mostly dense code,+#: where it reads 6.7 against tiktoken's 3.3 -- sizing from it would move the error and, past#: `MANIFEST_CHARS_PER_TOKEN`, make `plan_rung` refuse the whole run on any machine with no-#: tokeniser. A real tokeniser sizes the rungs; the estimate is still measured and still reported.+#: tokeniser. The estimate is still measured and reported.LADDER_RATIO_SOURCES = ("tiktoken/cl100k", "studio /api/inference/chat/count_tokens")@@ -1478,10 +1326,9 @@ def build_cells(}out: list[tuple[Cell, RungPlan]] = []for rung in rungs:- # The ladder is sized by `ratio`, not by the raw `chars_per_token` argument: the argument- # may be None, meaning "measure it", and the measured value is what every cell's `meta`- # reports. Passing the argument straight through here would size the rungs from a number- # the payload does not carry.+ # The ladder is sized by `ratio`, not the raw `chars_per_token` argument: the argument may be None+ # meaning "measure it", and the measured value is what every cell's `meta` reports, so passing the+ # argument through would size the rungs from a number the payload does not carry.plan = plan_rung(corpus,rung,diff --git a/tests/studio/studiobench/runtime/types.py b/tests/studio/studiobench/runtime/types.pyindex ce077d98307..08b41502a51 100644--- a/tests/studio/studiobench/runtime/types.py+++ b/tests/studio/studiobench/runtime/types.py@@ -30,27 +30,24 @@ ROW_TYPES = frozenset("action","sample","failure",- # The A/B run order, recorded before the first cell. Written even when the order is- # UNBALANCED, because whether linear drift cancelled is a property of the run that a reader- # of the table has no other way to recover.+ # The A/B run order, recorded before the first cell. Written even when UNBALANCED, because+ # whether linear drift cancelled is a property of the run a reader cannot otherwise recover."ab_plan",- # A cell that did not finish, announced the moment it fails so a reader scanning FORWARD- # can discard its window rows without joining backwards to a cell row it may never reach.+ # A cell that did not finish, announced the moment it fails so a reader scanning FORWARD can+ # discard its window rows without joining backwards."cell_aborted",- # The comparability key: everything that must match for two payloads to be compared at- # all, hashed into one quotable token. Its own row rather than a field on `run_meta` so a- # reader can find it without parsing the whole meta block.+ # The comparability key: everything that must match for two payloads to be compared, hashed+ # into one quotable token. Its own row so a reader need not parse the whole meta block."comparability",- # One UI surface swept by the optional `--surfaces` phase: a route, a settings tab, a menu.- # A row type of its own rather than an `action` row with a different name, because a surface- # has no slot, no budget and no timing to miss -- and reusing `action` would put forty rows- # with a null `timings` into the column the report scores actions from.+ # One UI surface swept by the optional `--surfaces` phase. A row type of its own because a+ # surface has no slot, no budget and no timing to miss, and reusing `action` would put forty+ # null-`timings` rows into the column the report scores actions from."surface",})-# Required keys per row type. Enforced in Recorder.emit, because a row that silently lost its-# `ran` flag reads downstream as a fast action rather than as a missing one.+# Required keys per row type. Enforced in Recorder.emit, because a row that lost its `ran` flag+# reads downstream as a fast action rather than a missing one.ROW_REQUIRED: dict[str, tuple[str, ...]] = {"run_meta": ("tier",@@ -69,16 +66,12 @@ ROW_REQUIRED: dict[str, tuple[str, ...]] = {"comparability": ("key", "fields"),"sample": ("t_ms",),"failure": ("kind", "detail"),- # `reason` is REQUIRED, not optional. A surface row that lost its reason reads as a surface- # that was reached, which is the one thing a coverage sweep may never claim by default. It is- # null only on the success path, where `reached` is true.+ # `reason` is REQUIRED: a surface row that lost it reads as a surface that was reached, the one+ # thing a coverage sweep may never claim by default. Null only on the success path."surface": ("surface", "reached", "reason", "parity"),}-# ── the cell ────────────────────────────────────────────────────────--@dataclass(frozen = True)class Cell:"""One measured configuration: one rung, one arm, one repetition."""@@ -122,15 +115,14 @@ def make_cell_id(rung: str, arm: str, rep: int) -> str:return f"r{rung}.{arm}.rep{rep}"-# ── the window ──────────────────────────────────────────────────────-#: `gap` is the quiet stretch the scheduler holds between two slots. It is NOT `stream`: a gap-#: window is opened before every slot, so most of them sit long after the reply finished. See-#: SceneRunner._gap_window for what that mislabelling cost.-#: `setup` is work the harness has to do before the film can start -- the composer click is the-#: one that costs anything -- and it is NOT `action`. It is dominated by the driver rather than by-#: the app, so the scoring layer keeps it out of the frame pool. See+#: window opens before every slot, so most sit long after the reply finished. See+#: SceneRunner._gap_window.+#: `setup` is pre-film harness work (the composer click is the costly one) and is NOT `action`:+#: dominated by the driver rather than the app, so scoring keeps it out of the frame pool. See#: `scoring/from_payload.UNSCORED_WINDOW_KINDS`.+# ── the window ──────────────────────────────────────────────────────+WINDOW_KINDS = frozenset({"action", "stream", "gap", "idle", "setup", "settle", "teardown"})@@ -168,9 +160,6 @@ class Window:}-# ── actions ─────────────────────────────────────────────────────────--@dataclassclass ActionResult:"""The outcome of one action.@@ -184,23 +173,17 @@ class ActionResult:expect_ok: Optional[bool] = Noneexpect: dict = field(default_factory = dict)timings: dict = field(default_factory = dict)- # CORRECTNESS INVARIANTS, not timings, and kept apart from them on purpose.- #- # A count here answers "did the action still do the whole job", where a timing answers "how- # long did it take". They are scored the same way and they mean opposite things when they- # move: a timing falling is the result, a count falling is a regression.- #+ # CORRECTNESS INVARIANTS, not timings, and kept apart on purpose. A count answers 'did the+ # action still do the whole job'; they move oppositely, a timing falling is the result and a+ # count falling is a regression.# This exists because `select_all_copy` asserted only `chars > 0`. Its selection is taken over- # the viewport's DOM, so any change that stops mounting the whole thread -- windowing,- # virtualization, a progressive mount that never widens -- truncates the clipboard silently- # and still passes. From a user's point of view a copy that drops most of the conversation is- # data loss, and it is the classic regression of every list that starts unmounting rows.- #- # The reference is the OTHER ARM rather than an absolute threshold. Both arms of an A/B seed a- # byte-identical thread, so a treatment that truncates reads as a large negative delta scored- # against the null control's own spread, and nothing has to be calibrated per rung or per- # platform. A count is therefore only meaningful in a paired comparison, which is the only- # place it is read.+ # the viewport's DOM, so anything that stops mounting the whole thread truncates the clipboard+ # silently and still passes: data loss, and the classic regression of every list that starts+ # unmounting rows.+ # The reference is the OTHER ARM rather than an absolute threshold: both arms seed a+ # byte-identical thread, so a truncating treatment reads as a large negative delta against the+ # null control's own spread with nothing calibrated per rung or platform. A count is therefore+ # only meaningful in a paired comparison.counts: dict = field(default_factory = dict)reason: Optional[str] = Noneslot_missed: bool = False@@ -208,9 +191,8 @@ class ActionResult:def __post_init__(self) -> None:if not self.ran:self.timings = {}- # Same rule as `timings`, for the same reason: an action that did not happen has no- # invariant to report, and a zero left here would read as "the whole job was done, and- # it did nothing", which is the exact inversion of what happened.+ # Same rule as `timings`: an action that did not happen has no invariant to report, and a zero+ # here would read as 'the whole job was done, and it did nothing'.self.counts = {}self.expect_ok = Noneif not self.reason:@@ -287,9 +269,6 @@ class Instrument:def detach(self) -> None: ...-# ── paths and context ───────────────────────────────────────────────--@dataclassclass Paths:out: Path@@ -377,9 +356,9 @@ class OutDirLock:"""lock = cls(out)lock.out.mkdir(parents = True, exist_ok = True)- # The legacy per-session names, swept once. A directory left by an older build still has to- # be read, or the guard would quietly switch itself off on exactly the runs it was added- # for. Only the fixed name is ever a mutex.+ # The legacy per-session names, swept once: a directory left by an older build still has to be+ # read, or the guard switches itself off on exactly the runs it was added for. Only the fixed+ # name is ever a mutex.lock._refuse_if_legacy_marker_is_live()lock._acquire(session_id)return lock@@ -475,12 +454,11 @@ class OutDirLock:if fd is None:returnself._fd = None- # BLANKED BEFORE IT IS RELEASED, and only while this process still holds the lock, so- # nothing can be reading it as authoritative. The file stays (see above); what goes is- # its CONTENT, because a retained `pid session` line outlives the run that wrote it and- # the next contender to lose a race would otherwise be handed it as the holder. Not a- # substitute for the liveness test in `_read_marker_once_written`: a killed run never- # reaches this line.+ # BLANKED BEFORE IT IS RELEASED, and only while this process holds the lock, so nothing can read+ # it as authoritative. The file stays; the CONTENT goes, because a retained `pid session` line+ # outlives its run and the next contender to lose a race would be handed it as the holder. Not+ # a substitute for the liveness test in `_read_marker_once_written`: a killed run never reaches+ # this line.try:os.ftruncate(fd, 0)except OSError:@@ -562,21 +540,15 @@ class OutDirLock:deadline = time.monotonic() + budget_swhile True:got = cls._read_marker(path)- # A RETAINED RECORD IS NOT A HOLDER. The marker is deliberately never unlinked, so on- # any directory that has been used before it already contains the PREVIOUS run's- # `pid session` line. That record is non-empty, so a bare "did it read" test stops on- # it and the refusal names a run that finished hours ago as the current holder --- # worse than saying nothing, because it sends the reader after a specific dead pid.- # Measured: a clean `close()` leaves `2235618 sessionAAAA`, and a contender that meets- # a NEW holder inside this window was told `session sessionAAAA is still running as- # pid 2235618` while the actual holder was pid 2235621.- #- # Liveness is what separates them. The holder is by definition a running process, and- # the run that wrote a retained record has exited. `close()` also blanks the marker- # while it still holds the lock, so a clean exit leaves nothing to mistake; this covers- # the unclean ones. What is left is a retained record whose pid has been RECYCLED onto- # a live process, which needs an unclean exit and a wrapped pid space in the same- # directory.+ # A RETAINED RECORD IS NOT A HOLDER. The marker is never unlinked, so a reused directory already+ # holds the PREVIOUS run's `pid session` line; a bare 'did it read' test stops on it and names+ # a run that finished hours ago, sending the reader after a specific dead pid. Measured: a+ # clean `close()` leaves `2235618 sessionAAAA`, and a contender meeting a NEW holder in this+ # window was told that pid while the actual holder was 2235621.+ # Liveness separates them: the holder is by definition running, and the run that wrote a+ # retained record has exited. `close()` also blanks the marker under the lock, so this covers+ # only unclean exits. What is left is a retained record whose pid was RECYCLED onto a live+ # process, needing an unclean exit and a wrapped pid space in one directory.if got is not None and cls._alive(got[1]):return gotif time.monotonic() >= deadline:@@ -629,48 +601,34 @@ class Recorder:self.session_id = session_idself.t0 = t0 if t0 is not None else time.monotonic()# REFUSE A SECOND LIVE SESSION IN ONE OUTPUT DIRECTORY.- #- # Appending is correct across SHARDS, which are deliberate and sequential. It is never- # correct for two runs going at once: they contend with each other, so neither measures- # the machine the other thought it had, and they write the same `cell_id`s into one file- # where a reader keyed on `cell_id` sees only the last writer.- #- # Observed: a launcher started twice produced three `run_meta` rows and every `cell_id`- # twice, both marked completed, with `r1M.treatment.rep1` keystroke `p50_ms` reading- # 73.4 ms in one session and 144.5 ms in the other. Scored last-wins that payload showed a- # 149.8% regression that does not exist at that size.- #- # The marker carries the pid, so a crashed run leaves a marker that names a dead process- # and the next run says so rather than refusing forever.- #+ # Appending is correct across SHARDS, which are deliberate and sequential, and never correct for+ # two concurrent runs: they contend, so neither measures the machine the other thought it had,+ # and they write the same `cell_id`s where a reader keyed on `cell_id` sees only the last+ # writer.+ # Observed: a launcher started twice produced three `run_meta` rows and every `cell_id` twice,+ # both completed, with `r1M.treatment.rep1` keystroke `p50_ms` reading 73.4 ms in one session+ # and 144.5 ms in the other, scored last-wins as a 149.8% regression that does not exist.+ # The marker carries the pid, so a crashed run leaves a marker naming a dead process and the+ # next run says so rather than refusing forever.# ONE FIXED NAME, CREATED EXCLUSIVELY. The name used to be `.running.{session_id}`, and a- # per-session name cannot be a mutex however carefully it is written: the two contenders- # race on DIFFERENT paths, so each scanned the directory, each found only the other's- # absence, and each then created its own marker. Scan-then-write is two syscalls with a- # window between them, and the window is wide enough to lose. Measured on this guard- # before the change, two processes released from a barrier against one output directory:- # 123 of 200 trials admitted BOTH recorders, reproducing the defect-9 signature the guard- # exists to prevent -- two `run_meta` rows, one `cell_id` completed twice, 73.4 ms against- # 144.5 ms. Two processes merely launched back to back, with no barrier at all, still- # collided about 3% of the time.- #- # `os.open(..., O_CREAT | O_EXCL)` makes the check and the creation one atomic operation- # against other opens of the same name, which is the documented lock-file idiom and is- # available on Unix and Windows alike. `fcntl.flock` is deliberately NOT used: the `fcntl`- # module does not exist on Windows, and this tool is run by external testers there.- #- # TAKEN BEFORE THIS POINT WHEN THE CALLER HAS ONE, AND THAT IS THE NORMAL PATH. `run()`- # holds the directory from its first millisecond, because by the time the payload is opened- # a duplicate has already archived the live payload and installed two Unsloth instances on top of the- # run it is about to be refused in favour of. See `OutDirLock`. A `Recorder` built without- # one -- the tests, and any caller that only wants a payload -- still takes its own, so the- # guard cannot be switched off by forgetting to pass it.- #+ # per-session name cannot be a mutex: the contenders race on DIFFERENT paths, each finding only+ # the other's absence. Measured before the change, two processes released from a barrier onto+ # one directory admitted BOTH recorders in 123 of 200 trials; merely launched back to back they+ # still collided about 3% of the time.+ # `os.open(..., O_CREAT | O_EXCL)` makes the check and the creation one atomic operation against+ # other opens of the same name, the documented lock-file idiom, on Unix and Windows alike.+ # `fcntl.flock` is deliberately NOT used: `fcntl` does not exist on Windows, and external+ # testers run this there.+ # TAKEN BEFORE THIS POINT WHEN THE CALLER HAS ONE, AND THAT IS THE NORMAL PATH: `run()` holds+ # the directory from its first millisecond, because by the time the payload is opened a+ # duplicate has already archived the live payload and installed two Unsloth instances. A+ # `Recorder` built without one still takes its own, so the guard cannot be switched off by+ # forgetting to pass it.+ # See `OutDirLock`.# WHO OWNS THE LOCK IS RECORDED HERE, because it decides who may let go of it. A lock this- # `Recorder` took is this `Recorder`'s to release when it closes. A lock ADOPTED from the- # caller outlives the recording -- `run()` closes the recorder and then reads the payload- # back to render `ab.md` -- so releasing it in `close` would free the directory for the- # length of the report. See `close`.+ # `Recorder` took is its to release on close; a lock ADOPTED from the caller outlives the+ # recording, since `run()` closes the recorder and then reads the payload back to render+ # `ab.md`, so releasing it in `close` would free the directory for the length of the report.self._owns_lock = lock is Noneif lock is None:lock = OutDirLock.take(self.path.parent, session_id)@@ -693,8 +651,7 @@ class Recorder:row.setdefault("schema", SCHEMA)row.setdefault("ts_ms", self.now_ms())row.setdefault("session_id", self.session_id)- # default = str so a stray Path or dataclass degrades to a string instead of losing the- # whole row, and the run keeps going.+ # default = str so a stray Path or dataclass degrades to a string instead of losing the whole row.self._fh.write(json.dumps(row, default = str) + "\n")self._fh.flush()try:@@ -752,20 +709,13 @@ class Recorder:except OSError:pass# RELEASED, NOT DELETED, and ONLY IF THIS RECORDER TOOK THE LOCK ITSELF.- #- # AN ADOPTED LOCK IS NOT THIS OBJECT'S TO DROP. `run()` takes the output directory in its- # first millisecond and hands it here, then closes the recorder in the `finally` under the- # cells and goes on to READ THE PAYLOAD BACK -- `_render_ab` and `_summarise` both open- # `payload.jsonl` after that `finally` and before `run()`'s own outer one. Releasing the- # adopted lock here freed the directory for exactly that window, and a second invocation- # arriving in it was admitted: its `prepare_payload` renames the finished run's- # `payload.jsonl` to `payload-<stamp>.jsonl` before it clones anything, so the reporting- # step either fails with `FileNotFoundError` on a run whose cells all completed, or -- if- # the contender has got as far as opening a payload of its own -- reads THAT file and- # writes an `ab.md` describing another run's rows while still exiting 0.- #- # A `Recorder` that took its own lock -- the tests, and any caller that only wants a- # payload -- has nobody else to release it, so it still does so here.+ # AN ADOPTED LOCK IS NOT THIS OBJECT'S TO DROP. `run()` hands the directory here, then closes+ # the recorder in the `finally` and goes on to READ THE PAYLOAD BACK. Releasing the adopted+ # lock here freed the directory for that window, and a second invocation arriving in it renames+ # the finished `payload.jsonl` before cloning, so reporting either fails with+ # `FileNotFoundError` on a run whose cells all completed or writes an `ab.md` describing+ # another run's rows while exiting 0.+ # A `Recorder` that took its own lock has nobody else to release it, so it still does so here.lock = getattr(self, "_lock", None)if lock is not None and getattr(self, "_owns_lock", True):lock.release()diff --git a/tests/studio/studiobench/scene/actions.py b/tests/studio/studiobench/scene/actions.pyindex b192b154989..70b0135e7b8 100644--- a/tests/studio/studiobench/scene/actions.py+++ b/tests/studio/studiobench/scene/actions.py@@ -36,24 +36,10 @@ from . import register_actionSETTLE_TIMEOUT_MS = 8000-#: How long an action that needs the message ACTION BAR waits for it before reporting NOT RUN.-#:-#: The bar is mounted `hideWhenRunning`, so Copy, Delete and More do not exist while the thread is-#: generating. Every film opens these slots after a `send_turn`, spaced by the NOMINAL drain of the-#: follow-up turn -- FOLLOW_UP_CHARS over the field cadence, 4.59 s. The nominal drain is a floor-#: rather than an estimate: it assumes the pacer is the binding constraint, and at the 100K rung it-#: is not. Measured over six 100K cells here, the follow-up streamed for 4.4 to 4.7 s AFTER the-#: send window closed, so the reply settles within a few hundred milliseconds of the slot on a-#: quiet machine and a little after it on a loaded one. The CI payload of the run that failed the-#: liveness gate shows exactly that: one more SSE chunk arrived INSIDE the `message_menu` window-#: and the reply stopped growing inside it too.-#:-#: 1,500 ms, and deliberately NOT `ctx.budget_ms`. The remaining budget at entry is whatever the-#: previous window's teardown left over -- measured at 51 ms on one of those cells -- so a wait-#: capped by it is a wait that is not taken in exactly the conditions that need it. It is bounded-#: so a control that is genuinely absent still reports NOT RUN rather than eating the film: an-#: action that overruns pushes nothing (every slot has an absolute start) and the overrun is-#: recorded as `over_budget_ms` on the row.+#: How long an action needing the message ACTION BAR waits before reporting NOT RUN: 1,500 ms,+#: deliberately NOT ctx.budget_ms (51 ms left at entry on one measured cell), and bounded so a+#: genuinely absent control reports NOT RUN rather than eating the film. The wait happens BEFORE+#: the operation the action times, so `menu_open_ms` is unaffected.ACTION_BAR_WAIT_MS = 1500@@ -100,13 +86,10 @@ def _failed(raw: Any) -> str | None:return None-# ── the paint floor ─────────────────────────────────────────────────-#-# Two rAFs resolve no sooner than two vsync intervals, so ANY timing clocked across a double rAF-# has a ~33ms floor under it on a 60Hz display. An action that never happened still reports that-# floor, which reads as a plausible measurement rather than as a failure. Measured per cell and-# recorded so a reader can subtract it.+# Two rAFs resolve no sooner than two vsync intervals, so any double-rAF timing has a ~33ms floor+# on a 60Hz display; measured per cell and recorded so a reader can subtract it.+# ── the paint floor ─────────────────────────────────────────────────PAINT_FLOOR_JS = """async (samples) => {const values = [];@@ -129,13 +112,10 @@ def paint_floor_ms(page, samples: int = 9) -> float | None:return None+#: The bound that stops a wedged renderer from eating the slot; the wait itself ENDS when nothing+#: is in flight, and reaching this bound means a lost sample the coverage check fails on.# ── 1. keystroke to paint ───────────────────────────────────────────--#: How long the keystroke drain will wait for the last paint before giving up on it. Not a "wait-#: a bit longer" constant: the wait ENDS when nothing is in flight, and this is only the bound that-#: stops a wedged renderer from eating the slot. Reaching it means a sample was lost, which the-#: coverage check then fails on rather than quietly publishing the rest.KEYSTROKE_SETTLE_TIMEOUT_MS = 3000KEYSTROKE_SETTLE_POLL_MS = 25@@ -188,9 +168,8 @@ def keystroke(ctx: ActionContext) -> ActionResult:ctx.page.click(selector)started = time.monotonic()- # A real delay between characters. Typing with delay=0 sends the whole burst in one CDP- # message, which the renderer coalesces into a single input event and a single paint, so the- # measurement collapses to one sample no matter how many characters were sent.+ # A real inter-character delay: delay=0 sends the burst in one CDP message, which the renderer+ # coalesces into a single input event and a single paint.ctx.page.keyboard.type("a" * count, delay = 60)_settle_keystrokes(ctx, inst)got = inst.collect(count)@@ -210,23 +189,10 @@ def keystroke(ctx: ActionContext) -> ActionResult:"composer_grew_by": grew,"composer_text_length": got.get("text_length"),}- # EVERY KEYSTROKE ACCOUNTED FOR, not merely a composer that grew.- #- # `grew_by` alone was the whole check, and it is satisfied by a textarea whose value is right- # while the timings describe a subset of the typing. Two ways that happened, and the first- # INVERTS the metric: a keystroke whose paint had not resolved when the drain ran was simply- # absent, and the keystroke that has not painted yet is the slowest one -- measured here, a- # 500 ms keystroke vanished from a reading whose max was 20 ms, on the highest-weight metric in- # the table. A build that made typing worse read faster, and that reading feeds the null- # control's noise floor, so it would also tighten the floor every later comparison is judged- # against. The other way is quieter: a keystroke that never reached the instrument at all is- # missing from both `samples` and `coalesced`.- #- # So the reading stands only when nothing was left in flight and `samples + coalesced` covers- # every input the instrument saw, and every input the driver commanded arrived. A LOW sample- # count is not itself a failure -- on a jammed page most keystrokes coalesce behind a slow- # paint, and that is the finding, not a fault -- which is why this counts coverage rather than- # demanding a sample per character.+ # EVERY KEYSTROKE ACCOUNTED FOR, not merely a composer that grew: `grew_by` alone is satisfied+ # while the timings describe a subset, and the keystroke whose paint had not resolved is the+ # slowest one (a 500 ms keystroke vanished from a reading whose max was 20 ms). So the reading+ # stands only when nothing was in flight and samples + coalesced covers every input.covered = (seen is not Noneand seen >= count@@ -322,17 +288,14 @@ def _scroll(ctx: ActionContext, label: str) -> ActionResult:steps = int(ctx.args.get("steps", 14))step_px = int(ctx.args.get("step_px", 420))settle_ms = int(ctx.args.get("settle_ms", 200))- # THE FOLLOW SAMPLER IS SUSPENDED FOR THE DURATION OF THIS GESTURE. This action drags the- # viewport thousands of pixels off the bottom on purpose, twice, while the reply is still- # streaming. Samples taken during it record a thread that is not pinned, which is true and is- # the benchmark's own doing, and averaging them in makes `follows_the_stream` a reading about- # the film rather than about the app. See the sampler in scene/dom.js.+ # THE FOLLOW SAMPLER IS SUSPENDED FOR THIS GESTURE: it drags the viewport thousands of pixels off+ # the bottom while the reply streams, so samples taken during it make `follows_the_stream` a+ # reading about the film._ev(ctx, "() => window.__sb.follow && window.__sb.follow.suspend()")try:raw = _ev(ctx, SCROLL_JS, [steps, step_px, settle_ms])finally:- # Resumed even when the gesture raised, or one failed scroll silences the sampler for the- # rest of the cell and the arm reports a follow fraction built from nothing.+ # Resumed even when the gesture raised, or one failed scroll silences the sampler for the rest of the cell._ev(ctx, "() => window.__sb.follow && window.__sb.follow.resume()")err = _failed(raw)if err:@@ -341,10 +304,8 @@ def _scroll(ctx: ActionContext, label: str) -> ActionResult:return not_run(raw.get("reason", "the scroll did not run"))commanded = raw["commandedPx"]travelled = raw["travelledPx"]- # 90% of commanded. Unsloth replaces assistant-ui's autoscroll with an intent-aware one that- # snaps a move it reads as programmatic straight back to the bottom: measured on an earlier- # harness, the gesture landed where it started and the column timed a scroll that did not- # happen. Travel is the only thing that separates the two.+ # 90% of commanded: Unsloth's intent-aware autoscroll snaps a move it reads as programmatic+ # straight back to the bottom, so travel is the only thing separating a real scroll from none.ok = commanded > 0 and travelled >= 0.9 * commandedreturn ActionResult(ran = True,@@ -380,27 +341,13 @@ def scroll_after(ctx: ActionContext) -> ActionResult:return _scroll(ctx, "after")+# SETTLE ON THE DOM, NOT ON `data-state`: the attribute flips before the content it reveals has+# mounted, and the gap depends on the collapse mechanism an A/B is comparing. At 100K, reading+# `pre span` on the attribute frame gave a 41% phantom reduction across arms where a quiet-DOM+# read gave the same number on both. A null control cannot see a bias it shares, so both the+# timing and the census terminate on `quietFrames` unchanged frames.# ── 4. reasoning expand / collapse ──────────────────────────────────-# SETTLING ON THE DOM, NOT ON A STATE ATTRIBUTE.-#-# `data-state="open"` flips when the Collapsible's state changes. It says nothing about whether the-# content that state reveals has mounted, and how far apart those two frames are depends on the-# COLLAPSE MECHANISM -- which is exactly the thing an A/B across a collapse change is comparing. So-# an instant defined by the attribute means a different thing on each arm.-#-# Measured, 100K rung, both arms, same corpus. Reading `pre span` on the frame the attribute-# settles gives 74,917 on the measured-height arm and 44,075 on the grid-rows arm, a 41% apparent-# reduction. Reading it once the span count has stopped changing gives 74,250 on BOTH, to the span.-# The 41% was the grid arm being counted before it had finished mounting.-#-# Its null control did not catch this and could not have: the null runs one bundle against itself,-# so the skew is identical on both sides and cancels exactly. A null cannot see a bias it shares.-#-# So both the timing and the census now terminate on a QUIET DOM: the observed quantity has to stop-# changing for `quietFrames` consecutive animation frames before it is read. If it never goes-# quiet, nothing is returned for it and the reason is recorded, because silence beats a confident-# wrong answer.SETTLE_QUIET_FRAMES = 4REASONING_JS = """@@ -577,12 +524,8 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:if not raw.get("ran"):return not_run(raw.get("reason", "the reasoning toggle did not run"))- # THE REASON NAMES THE CLAUSE THAT ACTUALLY FAILED.- #- # This used to be built from the pane counts alone, so a run whose real failure was a censored- # timing printed "16 of 16 panes opened and 0 were still open after collapsing" -- a- # description of SUCCESS -- under the heading EXPECT FAILED. A reader who trusts the message- # concludes the assertion is broken rather than that the timing is missing.+ # THE REASON NAMES THE CLAUSE THAT ACTUALLY FAILED: built from pane counts alone it printed a+ # description of SUCCESS under EXPECT FAILED when the real failure was a censored timing.failures: list[str] = []if raw["openCount"] != raw["panes"]:failures.append(f"only {raw['openCount']} of {raw['panes']} panes opened")@@ -594,11 +537,9 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:failures.append(f"close_ms is censored: {raw.get('closeCensoredReason')}")ok = not failures- # A CENSORED TIMING IS ABSENT, NOT ZERO, and it is absent LOUDLY. `_action_timings` drops- # non-numeric values, so a censored cell silently leaves the pooled metric and the survivors- # are the fast ones -- survivorship bias wearing a mean. The censoring is therefore recorded- # as its own field so the scoring layer can refuse to pool a metric that is censored at some- # rungs and not others.+ # A CENSORED TIMING IS ABSENT, NOT ZERO, and absent loudly: `_action_timings` drops non-numeric+ # values, so censoring silently leaves the fast survivors behind. Recorded as its own field so+ # scoring can refuse to pool a metric censored at some rungs and not others.timings = {}if raw["openMs"] is not None:timings["open_ms"] = raw["openMs"]@@ -609,21 +550,16 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:ran = True,expect_ok = ok,expect = {- # SCOPED TO WHAT IS MOUNTED, and always was. `reasoningTriggers()` is a- # querySelectorAll, so on the shipped build it finds every pane in the thread and on a- # windowed mount it finds the panes in the window. The assertion below stays- # self-consistent either way (it opens N and expects N open), so the action still- # passes -- but its COST silently stops being a function of thread length and becomes- # a function of window size. That is the arm's point, and it is also the reason this- # timing is not comparable between a windowed arm and a full one without the pane- # count beside it.+ # SCOPED TO WHAT IS MOUNTED: `reasoningTriggers()` is a querySelectorAll, so on a windowed mount+ # its COST stops being a function of thread length and becomes one of window size. The assertion+ # stays self-consistent either way, so this timing needs the pane count beside it.+ # A windowed mount is `isWindowed()` in scene/dom.js."panes": raw["panes"],"panes_scope": "mounted","open_after_expand": raw["openCount"],"open_after_collapse": raw["afterClose"],- # Present ONLY when the span census went quiet. See the note above REASONING_JS: read- # on the frame the state attribute flips, this number was 41% wrong on one arm and- # right on the other, and its null control could not see the difference.+ # Present ONLY when the span census went quiet; read on the state-flip frame this number was 41%+ # wrong on one arm and right on the other (see the note above REASONING_JS)."highlight_spans_while_open": raw["spansOpen"],"highlight_spans_while_open_reason": raw.get("spansOpenReason"),"settled": raw["spansOpen"] is not None,@@ -641,14 +577,10 @@ def reasoning_toggle(ctx: ActionContext) -> ActionResult:)-#: THE SAME GESTURE A USER MAKES. `reasoning_toggle` above opens every pane in the thread at once,-#: which is a deliberate worst case and reads at 2.2 fps at the 100K rung. That number has been-#: quoted as though it described opening a reasoning pane, and it does not: a user opens ONE, and-#: almost always the newest one. This action measures that, so the two can be quoted apart.-#:-#: It is NOT in the standard film. The scene is a fixed-duration slot schedule, so adding a slot-#: shifts every window after it and voids comparability against every payload already on disk. Run-#: it in a purpose-built film until the next corpus or tier bump invalidates those payloads anyway.+#: THE SAME GESTURE A USER MAKES: `reasoning_toggle` opens every pane at once (2.2 fps at 100K), a+#: deliberate worst case that has been quoted as though a user did it. NOT in the standard film:+#: adding a slot shifts every window after it and voids comparability with every payload on disk.+# The fast film opens this slot 0.4 s after the worst-case drain.REASONING_ONE_JS = """async (timeoutMs) => {const D = window.__sb.dom;@@ -710,25 +642,19 @@ def reasoning_toggle_one(ctx: ActionContext) -> ActionResult:ran = True,expect_ok = ok,expect = {- # SCOPED TO WHAT IS MOUNTED, and always was. `reasoningTriggers()` is a- # querySelectorAll, so on the shipped build it finds every pane in the thread and on a- # windowed mount it finds the panes in the window. The assertion below stays- # self-consistent either way (it opens N and expects N open), so the action still- # passes -- but its COST silently stops being a function of thread length and becomes- # a function of window size. That is the arm's point, and it is also the reason this- # timing is not comparable between a windowed arm and a full one without the pane- # count beside it.+ # SCOPED TO WHAT IS MOUNTED: `reasoningTriggers()` is a querySelectorAll, so on a windowed mount+ # its COST becomes a function of window size rather than thread length, and this timing is not+ # comparable across arms without the pane count beside it."panes": raw["panes"],"panes_scope": "mounted",- # `panes_opened` is the assertion the two above are context for: this action opens- # exactly one pane whatever the thread holds, which is the whole point of it existing- # beside `reasoning_toggle`. Asserted by test_studiobench_reasoning_one_live.+ # `panes_opened` is the assertion the notes above are context for: this action opens exactly one+ # pane whatever the thread holds. Asserted by test_studiobench_reasoning_one_live."panes_opened": 1,"open_after_expand": raw["openCount"],"open_after_collapse": raw["afterClose"],"highlight_spans_while_open": raw["spansOpen"],- # The cost driver, so a reading can be normalised rather than compared across threads- # whose newest reply happens to differ in size.+ # The cost driver, so a reading can be normalised rather than compared across threads whose+ # newest reply differs in size."highlight_spans_added": raw["spansAdded"],},timings = {"open_ms": raw["openMs"], "close_ms": raw["closeMs"]},@@ -739,126 +665,50 @@ def reasoning_toggle_one(ctx: ActionContext) -> ActionResult:)+#: Fixed waits the throwaway turn costs once it starts: 80 ms for the composer, 600 ms to get the+#: turn going, 400 ms for in-flight chunks, 200 ms for the cleanup delete. RESERVED out of the+#: drain wait, or the overrun lands in `scroll_after`'s window as a bogus `slot_missed`. Split,+#: because the 80 ms is spent before the turn is sent.+# `scroll_after` has a 1,200 ms window on the fast film.# ── 5. stop generation ──────────────────────────────────────────────-#: What `stop_generation`'s throwaway turn costs in FIXED waits once it starts, on any machine: 80-#: ms for the composer to take the text, 600 ms to let the new turn get going so stop is measured-#: against a live stream rather than a starting one, 400 ms for the chunks already in flight to-#: land, and 200 ms for the cleanup delete to settle. Polling for the turn to start and for the-#: stream to stop is on top of it, so this is a floor.-#:-#: RESERVED OUT OF THE DRAIN WAIT rather than spent past the deadline. The drain wait below is-#: bounded by the slot's remaining budget, so a reply that finishes at the last moment of the slot-#: used to leave nothing at all for the 1.28 s that follows. `stop_generation` closes 300 ms before-#: `scroll_after` opens on the fast film and 500 ms before it on the quick one, and the runner is-#: sequential -- every slot has an absolute start, but nothing enters it until the previous action-#: returns. So the overrun came out of `scroll_after`'s own 1,200 ms window on the fast film and-#: recorded `slot_missed` there, whose reason blames the machine for reaching the slot late when-#: what happened is that this action was still cleaning up its scaffolding.-#:-#: SPLIT, because the two halves are reserved at different moments. The 80 ms that settles the fill-#: is spent BEFORE the turn is sent, so it is already gone by the time the turn-start wait is-#: bounded and a bound that subtracts the whole figure charges it twice.OWN_TURN_FIXED_AFTER_SEND_MS = 600 + 400 + 200OWN_TURN_FIXED_MS = 80 + OWN_TURN_FIXED_AFTER_SEND_MS-#: What the SAME throwaway turn costs BEYOND those sleeps: two polls and the driver round trips-#: underneath them, neither of which is free.-#:-#: MEASURED, because the first version of this reserve was not. It was sized from a page shim whose-#: `evaluate` is a Python call and whose clock only moves when `wait_for_timeout` is called, so the-#: waits it recorded were `[100 x29, 80, 600, 400, 200]` and the two polls cost nothing. Driving-#: the same shipped action against real chromium and a real clock, with a page standing in for the-#: five calls it makes, the stretch after the drain costs:-#:-#: a page that answers instantly 1,394 - 1,451 ms (13 - 17% over the fixed 1,280)-#: 120 ms to start, 90 ms to stop, 60 ms-#: to delete -- an unremarkable local app 1,723 - 1,938 ms (35 - 51% over)-#:-#: The gap is nine to thirteen CDP round trips (43 - 130 ms of pure driver time), the 50 ms-#: granularity of both poll loops, and the app's own latency in answering them. Against a 1,280 ms-#: reserve that put the action 506 - 516 ms past the end of its slot on the fast and quick films,-#: whose gaps before `scroll_after` opens are 300 ms and 500 ms, so `scroll_after` recorded-#: `slot_missed` and its reason blamed the machine.-#:-#: 700 ms covers the measured 443 - 658 ms with margin and still leaves every film a real drain-#: wait: the smallest stop slot is 3,000 ms, which keeps 1,020 ms of waiting after the reserve,-#: and the worst-case drain overhang the fast film has to absorb is 400 ms.+#: What the same turn costs BEYOND those sleeps: two polls plus the driver round trips. Measured+#: against real chromium, 1,394-1,938 ms against the fixed 1,280, so 700 ms covers the measured+#: 443-658 ms and still leaves every film a real drain wait.#:-#: SPLIT AT THE MOMENT THE TURN STARTS, because that is where the wait below is bounded and a total-#: is not a bound. The stop-settle poll, the cleanup delete's own round trips and the driver calls-#: between them are ALL still ahead at that point, and a bound that reserves only the fixed sleeps-#: leaves the action exactly `OWN_TURN_FIXED_MS` for a stretch that costs more than that. Measured-#: against real chromium on the same page as the totals above, with the turn starting on the last-#: millisecond the bound allows and the fast film's 3,000 ms stop slot:-#:-#: a page that answers instantly 60 ms after the sleeps slot + 0.0 s-#: 90 ms to stop, 60 ms to delete 227 ms after the sleeps slot + 0.15 s-#: 200 ms to stop, 150 ms to delete 424 ms after the sleeps slot + 0.34 s-#:-#: The fast film leaves 300 ms between this slot's deadline and `scroll_after`, so the last row-#: overran into `scroll_after`'s own window and was recorded there as `slot_missed` -- the same-#: symptom, one action later, that the reserve above was written to remove. 500 ms covers the-#: measured 60 - 424 ms; the 200 ms left over is the start poll's own share, which is what the-#: turn-start wait is allowed to spend.+#: SPLIT AT THE MOMENT THE TURN STARTS, because that is where the wait below is bounded and a+#: total is not a bound: the stop-settle poll, the cleanup delete and the driver calls between+#: them are all still ahead. Measured at 60-424 ms after the sleeps; 500 ms covers it.+# 90 ms to stop, 60 ms to delete, 227 ms after the sleeps; `STOP_CLEANUP_JS` is unbounded by the slot.+# Against the fast film's 3,000 ms stop slot.OWN_TURN_STOP_POLL_MS = 500OWN_TURN_START_POLL_MS = 200OWN_TURN_POLL_MS = OWN_TURN_START_POLL_MS + OWN_TURN_STOP_POLL_MS-#: What the throwaway turn needs in the slot, in total.-#:-#: RESERVED OUT OF THE DRAIN WAIT rather than spent past the deadline. The drain wait below is-#: bounded by the slot's remaining budget, so a reply that finishes at the last moment of the slot-#: used to leave nothing at all for the ~1.7 s that follows. `stop_generation` closes 300 ms before-#: `scroll_after` opens on the fast film and 500 ms before it on the quick one, and the runner is-#: sequential -- every slot has an absolute start, but nothing enters it until the previous action-#: returns. So the overrun came out of `scroll_after`'s own 1,200 ms window on the fast film and-#: recorded `slot_missed` there, whose reason blames the machine for reaching the slot late when-#: what happened is that this action was still cleaning up its scaffolding.-#:-#: A RESERVE IS NOT ENOUGH ON ITS OWN, which is why `stop_generation` also re-reads the clock-#: before it commits. The drain loop tests its deadline at the TOP, so one iteration -- a 100 ms-#: wait plus a round trip -- lands past it, and no constant chosen here can pay for time that has-#: already been spent. The reserve decides how much of the slot the drain may have; the check-#: decides whether what is actually left is enough to start a turn that cannot be abandoned once-#: it is running.+#: What the throwaway turn needs in the slot in total, reserved out of the drain wait rather than+#: spent past the deadline. A reserve is not enough on its own: the drain loop tests its deadline+#: at the TOP, so `stop_generation` re-reads the clock before committing to a turn.+#: The fixed waits are 80 ms for the composer to take the text, 600 ms to let the turn get+#: going, 400 ms for the chunks in flight and 200 ms for the cleanup delete.+# One iteration is a 100 ms tick.OWN_TURN_RESERVE_MS = OWN_TURN_FIXED_MS + OWN_TURN_POLL_MS-#: How long to wait for the throwaway turn to start, in total. The wait for a turn this action will-#: still MEASURE is bounded by the slot on top of this -- 8 s is 2.7x the whole budget of the fast-#: film's stop slot, so a turn that was slow to start used to overrun by seconds whatever had been-#: reserved for it -- but the turn does not stop existing when the slot runs out.-#:-#: ENTER IS PRESSED BEFORE THIS WAIT, which is what an earlier version of this comment had wrong-#: when it called cutting the wait short free. Something IS committed by then: the send is-#: accepted, the user turn is in the thread, and the reply starts whenever the relay gets round to-#: it. Returning at the slot bound left exactly the scaffolding the paragraph below refuses to-#: leave -- measured in chromium on a page that took 1,800 ms to start against the fast film's-#: 3,000 ms slot, the action returned `not_run` after 1,759 ms and the thread it handed on had two-#: extra messages in it and a live stream running through the next action's window.-#:-#: So the slot bounds how long the turn is WORTH MEASURING, and this bounds how long it is worth-#: waiting for so it can be taken back: `_reclaim_pending_turn` polls on to this deadline, stops-#: the turn if it ever becomes stoppable and deletes it, then reports `not_run`. The total spent-#: waiting for a start is therefore what it was before the slot bound existed, and the slot bound-#: still does its job on every run where the turn arrives.-#:-#: The stop-settle poll and `STOP_CLEANUP_JS` are deliberately NOT bounded by the slot at all: once-#: Enter is pressed the turn has to be stopped and removed or the thread every later action-#: measures keeps the scaffolding, and a settle poll cut short would report a stop that worked as-#: `still_running`.+#: How long to wait for the throwaway turn to start. Enter is pressed BEFORE this wait, so the+#: send is already committed: returning at the slot bound left two extra messages and a live+#: stream running into the next action's window. The slot bounds how long the turn is worth+#: MEASURING; this bounds how long it is worth waiting for so it can be stopped and deleted.+# The turn stays in the thread `still_running`.TURN_START_TIMEOUT_MS = 8000-#: The composer text the throwaway turn is sent with. Read back as well as written: a composer that-#: still holds it is a send the app REFUSED, and that is the one case on the timeout path where-#: nothing was committed and there is nothing to take back.+#: The composer text the throwaway turn is sent with, read back as well as written: a composer+#: that still holds it is a send the app REFUSED, the one case where nothing was committed.OWN_TURN_TEXT = "one more"-#: Remove the throwaway turn `stop_generation` created, so the thread it leaves behind is the-#: thread it found. Deletes the assistant turn and then the user turn that prompted it, in that-#: order, because deleting the user message first can take the reply with it and leave the count-#: ambiguous. Reports what it managed rather than asserting: a cleanup that half-worked must be-#: visible in the row, not swallowed.+#: Remove the throwaway turn: assistant first, then the user turn, because deleting the user+#: message can take the reply with it and leave the count ambiguous. Reports rather than asserts.STOP_CLEANUP_JS = """async (timeoutMs) => {const D = window.__sb.dom;@@ -968,9 +818,8 @@ def _reclaim_pending_turn(if removed is not None:ctx.page.wait_for_timeout(200)- # BOTH HALVES, REPORTED SEPARATELY. A turn that was deleted but never stopped and one that was- # stopped but not deleted leave the film in different states, and a row that says only "cleaned- # up" cannot be read in the light of either.+ # BOTH HALVES, REPORTED SEPARATELY: deleted-but-never-stopped and stopped-but-not-deleted leave+ # the film in different states.if running is not True:state = f"never started within the {TURN_START_TIMEOUT_MS}ms it was worth waiting for"elif stopped:@@ -997,9 +846,8 @@ def stop_generation(ctx: ActionContext) -> ActionResult:Queue control at the same position with the same class. Pressing it queues a message and thestream carries on, and the action reports a fast, precise, entirely wrong number."""- # THE SLOT, ON THE CLOCK THE ACTION ITSELF RUNS ON. Everything below that is bounded is- # bounded against this rather than against a constant, because a budget spent is not a budget- # available: `ctx.budget_ms` is what the slot HAD when the runner entered it.+ # THE SLOT, ON THE CLOCK THE ACTION ITSELF RUNS ON: `ctx.budget_ms` is what the slot HAD when the+ # runner entered it, and a budget spent is not a budget available.slot_deadline = time.monotonic() + ctx.budget_ms / 1000.0def remaining_ms() -> float:@@ -1010,39 +858,13 @@ def stop_generation(ctx: ActionContext) -> ActionResult:ctx.page.fill('textarea[aria-label="Message input"]', "")ctx.page.wait_for_timeout(120)- # STOP GETS ITS OWN GENERATION. It used to stop whatever the cell was streaming, which- # permanently truncated the measured reply: at the 10K rung the pacer got 5,304 of 17,737- # characters away before the socket broke, so the ten actions that follow -- all of them- # labelled "after the reply is complete" -- and the final census ran against a thread barely- # a third of the size the rung claims. The seeded-vs-streamed equivalence check then compared- # that truncated reply against a complete seeded one and reported 20% drift, which read as a- # finding about seeding and was a finding about this action.- #- # Sending its own short turn keeps the measured reply intact and means the action has- # something to stop at EVERY rung, instead of reporting `not_run` at the small ones where the- # main stream is already over by the time the slot opens.- #- # THE GUARD BELOW IS THE OTHER HALF OF THAT, and without it the paragraph above only held for- # the default fixture. The own-turn path was entered only when `isRunning()` was FALSE, so the- # moment anything WAS running this action fell straight through and clicked Stop on it -- the- # exact behaviour the paragraph above says was removed. The supported way to make that happen- # is `--stream-tail-chars`, whose entire purpose is a long opening reply: at 96,000 characters- # the reply streams for 291 s at field cadence against a 243 s standard film, so this slot- # opens at 28 s with the opening turn still in flight and kills it at about 9,200 characters.- # The reply-length axis the flag exists to provide is then never exercised, every later action- # still runs against a settled thread, `ran` is still true, and `--assert-liveness` -- which- # the flag's own help text sends the caller to -- still passes. A silently truncated fixture- # reported as a clean run is the most expensive failure shape this harness has.- #- # So the reply is given the rest of this slot's budget LESS what the throwaway turn then costs- # (`OWN_TURN_RESERVE_MS`) to finish on its own, which is all a marginally slow drain needs (the- # fast film opens this slot 0.4 s after the worst-case drain on the ladder, and 1.0 s of the- # fast film's 3 s budget is still left over for it). If it has not finished by then, NOTHING IS- # STOPPED and the row says why. An honest `not_run` costs a column; stopping the reply costs- # the whole cell, and spending the rest of the slot draining and THEN starting a turn of our- # own costs the next slot. With the default tail nothing is ever running when this slot opens- # -- the packing test in fixture/selftest holds every film to that -- so this path is- # unreachable on an unmodified run and the behaviour there is exactly what it was.+ # STOP GETS ITS OWN GENERATION. Stopping whatever the cell was streaming permanently truncated+ # the measured reply (5,304 of 17,737 characters at 10K), so every later action ran against a+ # third-sized thread. THE GUARD BELOW IS THE OTHER HALF: entering the own-turn path only when+ # `isRunning()` was false let `--stream-tail-chars` kill the opening reply while liveness passed.+ # At 96,000 characters the slot opens at 28 s and kills it at about 9,200.+ # So the reply gets this slot's budget less `OWN_TURN_RESERVE_MS`, and if it has not finished+ # NOTHING IS STOPPED and the row says why.if _ev(ctx, "() => window.__sb.dom.isRunning()"):drain_ms = max(0.0, ctx.budget_ms - OWN_TURN_RESERVE_MS)settle_deadline = time.monotonic() + drain_ms / 1000.0@@ -1061,13 +883,9 @@ def stop_generation(ctx: ActionContext) -> ActionResult:"and the final census measure, so nothing was stopped. Lower --stream-tail-chars ""or move this slot past the drain")- # AND THE RESERVE IS CHECKED AGAINST THE CLOCK, not assumed off the budget. The loop above- # tests `settle_deadline` at the top, so the iteration that finds the reply drained has- # already spent a 100 ms wait and a round trip past it; a drain that lands in the last- # iteration therefore leaves LESS than the reserve however the reserve is sized. Starting- # the turn anyway is the same overrun by a smaller amount, and the turn cannot be- # abandoned halfway -- once Enter is pressed the thread has to be stopped and cleaned up- # or every later action measures the scaffolding.+ # AND THE RESERVE IS CHECKED AGAINST THE CLOCK: the loop tests `settle_deadline` at the top, so a+ # drain landing in the last iteration leaves less than the reserve however it is sized, and the+ # turn cannot be abandoned once Enter is pressed.if remaining_ms() < OWN_TURN_RESERVE_MS:return not_run("the cell's own reply drained with only "@@ -1078,31 +896,18 @@ def stop_generation(ctx: ActionContext) -> ActionResult:"--stream-tail-chars or move this slot past the drain")- # READ BEFORE ENTER, because it is the only way to tell the turn this action added from the- # thread it was handed. `_reclaim_pending_turn` deletes only if this number grew.- #- # threadTotal, not messageCount, for the reason STOP_CLEANUP_JS gives above and in the same- # direction: under a windowed mount the window refills as the thread grows, so a send that- # WORKED reports after == before and the reclaim path then decides there is nothing to remove.- # The turn it already sent is left in the thread with its stream still running, which is the- # one outcome this whole path exists to prevent, and it contaminates every later action window- # and the final census. Identical on the shipped build, where the two are the same number.+ # READ BEFORE ENTER: the only way to tell the turn this action added from the thread it was+ # handed, and `_reclaim_pending_turn` deletes only if it grew. threadTotal, not messageCount:+ # under a windowed mount a send that WORKED reports after == before.messages_before = _ev(ctx, "() => window.__sb.dom.threadTotal()")ctx.page.fill('textarea[aria-label="Message input"]', OWN_TURN_TEXT)ctx.page.wait_for_timeout(80)ctx.page.keyboard.press("Enter")own_generation = Truesent_at = time.monotonic()- # HOW LONG THE TURN IS WORTH MEASURING. Bounded by the SLOT as well as by- # `TURN_START_TIMEOUT_MS`: 8 s is 2.7x the whole stop slot on the fast and quick films, so a- # turn slow to start overran by seconds no matter what had been reserved for it.- #- # WHAT IS RESERVED OUT OF IT is the rest of the turn, not only its sleeps. A turn starting on- # the last millisecond this allows still has the stop-settle poll, the delete and the driver- # calls between them ahead of it, and reserving `OWN_TURN_FIXED_MS` left 424 ms of that unpaid- # against real chromium -- 344 ms past a 3,000 ms slot with 300 ms before `scroll_after` opens,- # recorded there as a missed slot. `OWN_TURN_FIXED_MS` also counts the 80 ms above, which is- # already spent. See `OWN_TURN_STOP_POLL_MS`.+ # HOW LONG THE TURN IS WORTH MEASURING, bounded by the SLOT as well as by `TURN_START_TIMEOUT_MS`+ # (8 s is 2.7x the whole stop slot). What is reserved out of it is the rest of the turn, not only+ # its sleeps: reserving `OWN_TURN_FIXED_MS` left 424 ms unpaid against real chromium.start_wait_ms = max(0.0,min(@@ -1111,8 +916,7 @@ def stop_generation(ctx: ActionContext) -> ActionResult:),)deadline = time.monotonic() + start_wait_ms / 1000.0- # HOW LONG IT IS WORTH WAITING FOR, which is a different question and is NOT bounded by the- # slot. See `_reclaim_pending_turn`.+ # HOW LONG IT IS WORTH WAITING FOR, a different question, and NOT bounded by the slot. See `_reclaim_pending_turn`.reclaim_deadline = sent_at + TURN_START_TIMEOUT_MS / 1000.0started_late = (f"nothing was generating and a new turn did not start within {start_wait_ms:.0f}ms"@@ -1148,11 +952,9 @@ def stop_generation(ctx: ActionContext) -> ActionResult:chars_after = _ev(ctx, "() => window.__sb.dom.assistantChars()")ok = stopped_ms is not None- # LEAVE THE THREAD AS WE FOUND IT. The throwaway turn is scaffolding, not content: left in- # place it adds an assistant message and a reasoning pane that the rest of the film, the final- # census and the seeded-versus-streamed comparison all then measure. That showed up- # immediately as "streamed 5 assistant messages vs seeded 4", a drift introduced entirely by- # this action while it was busy fixing a different one.+ # LEAVE THE THREAD AS WE FOUND IT: left in place the throwaway turn adds an assistant message and+ # a reasoning pane that the rest of the film, the census and the seeded-versus-streamed+ # comparison all measure.removed = Noneif own_generation:removed = _ev(ctx, STOP_CLEANUP_JS, SETTLE_TIMEOUT_MS)@@ -1164,16 +966,15 @@ def stop_generation(ctx: ActionContext) -> ActionResult:"chars_before": chars_before,"chars_after": chars_after,"still_running": not ok,- # Which reply was stopped. A reader comparing `chars_added_after_stop` across- # rungs needs to know whether this was a throwaway turn or the cell's own.+ # Which reply was stopped: `chars_added_after_stop` reads differently for a throwaway turn than+ # for the cell's own."own_generation": own_generation,- # Whether the scaffolding was removed again. Reported rather than asserted: a- # cleanup that failed leaves an extra turn in the thread, and every census- # after this point needs to be readable in that light.+ # Whether the scaffolding was removed. Reported rather than asserted: a failed cleanup leaves an+ # extra turn every later census must be read against."scaffold_removed": (None if removed is None else bool(removed.get("removed"))),"scaffold_note": (None if removed is None else removed.get("reason")),- # A stop that worked leaves the text where it was, give or take the chunks- # already in flight. A large jump means the stream ran on.+ # A stop that worked leaves the text where it was, give or take the chunks already in flight; a+ # large jump means the stream ran on."chars_added_after_stop": (None if chars_after is None or chars_before is None else chars_after - chars_before),@@ -1287,8 +1088,8 @@ def model_change(ctx: ActionContext) -> ActionResult:return not_run("no model selector trigger on the page")before = _ev(ctx, "() => window.__sb.dom.currentModelLabel()")started = time.monotonic()- # Click the LABEL, not the trigger's right edge: a `span[data-eject-hit]` sits there and- # clicking it ejects the model instead of opening the picker.+ # Click the LABEL, not the trigger's right edge: a `span[data-eject-hit]` sits there and ejects+ # the model instead of opening the picker.trigger.click(position = {"x": 8, "y": 8})opened_ms = Nonedeadline = started + SETTLE_TIMEOUT_MS / 1000@@ -1411,10 +1212,8 @@ def copy_markdown(ctx: ActionContext) -> ActionResult:triggers a file download rather than a clipboard write. The action bar has `autohide`, so itis hovered first or the button is not in the tree to click."""- # Hover AND WAIT, on the same reasoning as `message_menu`: this bar is `hideWhenRunning`, so a- # slot that opens with the follow-up's last chunks still arriving finds no Copy button at all.- # The fixed 150 ms this replaced was a hover settle, not a wait for the reply, and it returned- # the same "no Copy button" whether the control was missing or merely not mounted yet.+ # Hover AND WAIT, on the same reasoning as `message_menu`: the bar is `hideWhenRunning`, so a slot+ # opening with the follow-up's last chunks still arriving finds no Copy button.found = _ev(ctx,"""@@ -1451,9 +1250,8 @@ def copy_markdown(ctx: ActionContext) -> ActionResult:return not_run("no Copy button on the last assistant message")ctx.page.wait_for_timeout(200)elapsed = (time.monotonic() - started) * 1000- # Read back from the clipboard. Headless Chromium grants clipboard-read only with the- # permission, which the browser factory requests; if it is not granted the action still RAN,- # and the assertion says it could not be proved rather than that it failed.+ # Read back from the clipboard: headless Chromium grants clipboard-read only with the permission+ # the browser factory requests, and without it the action still RAN but could not be proved.clip = Nonereason = Nonetry:@@ -1511,14 +1309,9 @@ def select_text(ctx: ActionContext) -> ActionResult:if not raw.get("ran"):return not_run(raw.get("reason", "selection did not run"))visible = raw.get("visibleChars") or 0- # NON-EMPTY, and the coverage fraction reported as evidence rather than gated on.- #- # "Half the visible characters" looked like a reasonable bar and is not one: innerText- # collapses runs of whitespace and skips nested scrollers, and Selection.toString applies its- # own normalisation, so the two counts are not the same quantity even when the selection- # covers the whole message. Measured on a real reply: 22 selected against 80 "visible", with- # the selection plainly spanning the entire message. The action's claim is that text in this- # message got selected; the fraction belongs in the evidence, not in the verdict.+ # NON-EMPTY, with the coverage fraction as evidence rather than a gate: innerText collapses+ # whitespace and skips nested scrollers while Selection.toString normalises differently, so the+ # two counts are not the same quantity.ok = raw["chars"] > 0return ActionResult(ran = True,@@ -1563,43 +1356,17 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:err = _failed(raw)if err:return not_run(err)- # A SENTINEL ON THE CLIPBOARD BEFORE THE COPY, so "the copy happened" is OBSERVED rather than- # assumed from the keystroke having been sent.- #- # THE FAILURE THIS EXISTS FOR, measured. Playwright's WebKit never performs a clipboard copy on- # Control+C. The action pressed the key, slept 250ms for the copy to land, and reported the- # sleep as `copy_ms`. In every WebKit payload on this machine that number is 258.5 ms at the 1K- # rung, 258.8 ms at 10K and 263.9 ms at 100K: a hundredfold change in the quantity under study- # moved the "measurement" by two percent, because it was measuring `wait_for_timeout(250)`.- # Chromium reads about 1,538 ms at 100K for the same action. Forty-three rows across eleven- # payloads reported a sleep as a result.- #- # A measurement that silently reports a sleep is the worst failure mode in this harness, so the- # test is on the CLIPBOARD and not on the engine name: an engine that starts working, or a new- # one, is admitted automatically, and an engine that stops working is refused automatically.- #- # WHAT THE CHECK NEEDS is a KNOWN PRE-COPY VALUE, and the sentinel is only the best way of- # getting one. Writing it can fail on its own: `writeText` needs transient user activation or- # the `clipboard-write` permission and throws `NotAllowedError` without either, while- # `readText` is gated separately -- so the two are not granted or refused together, and- # runtime/browser.py asks for the pair on Chromium and for neither on the other engines.- #- # Clearing the sentinel on that failure and carrying on is what re-admitted the exact defect- # above: with no pre-copy value, `clip == sentinel` can never fire, and a Control+C that did- # nothing leaves the clipboard holding whatever `copy_markdown` put there earlier in the same- # film -- a plausible, non-empty string that is read back as a fresh copy of the whole thread,- # with the 250 ms settle beside it as `copy_ms`.- #- # So a failed write falls back to SNAPSHOTTING what the clipboard already holds, which answers- # the same question ("did this keystroke change the clipboard"), and when even that cannot be- # established the action is NOT RUN. The residual, stated: if the clipboard already held- # character-for-character what the copy would produce, an honest copy is refused as a- # no-op. That direction is the safe one, it needs the write permission to be missing first,- # and the film never copies the same thread twice.+ # A SENTINEL ON THE CLIPBOARD BEFORE THE COPY, so the copy is OBSERVED rather than assumed from+ # the keystroke. Playwright's WebKit never copies on Control+C: the action reported its own+ # 250 ms sleep as `copy_ms` across forty-three rows. The test is on the CLIPBOARD, not the engine+ # name. A failed `writeText` falls back to SNAPSHOTTING what the clipboard already holds; when+ # even that fails the action is NOT RUN, because clearing the sentinel re-admits the defect.+ # Chromium reads about 1,538 ms at 100K for the same action.+ # Residual: an honest copy of character-identical content reads as a no-op.sentinel = f"__sb_clipboard_sentinel_{int(time.monotonic() * 1000)}__"sentinel_written = False- #: What the clipboard held before Control+C, and where that value came from. `None` means no- #: pre-copy value could be established at all, which is refused below.+ #: What the clipboard held before Control+C and where that value came from; None means no+ #: pre-copy value could be established, which is refused below.pre_copy: Optional[str] = Nonepre_copy_source = "sentinel"pre_copy_reason = None@@ -1630,19 +1397,10 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:ctx.page.keyboard.press("Control+C")ctx.page.wait_for_timeout(250)copy_ms = (time.monotonic() - copy_started) * 1000- # THE CLIPBOARD, NOT THE SELECTION.- #- # This action used to score itself on `Selection.toString().length`, and that is not what a- # user gets. The two came apart the moment an arm existed that unmounts messages AND handles- # the copy event: the selection can only ever cover mounted nodes, while a copy handler that- # reads the message store puts the whole conversation on the clipboard. Scored on the- # selection, a build that had FIXED the data loss would still be reported as losing data --- # the alarm would stay lit for a reason that no longer existed, which is the fastest way to- # get an alarm switched off.- #- # Read back before the selection is cleared, and the failure to read it is reported rather- # than treated as an empty clipboard: headless Chromium grants clipboard-read only with the- # permission the browser factory requests.+ # THE CLIPBOARD, NOT THE SELECTION: a selection covers only mounted nodes while a copy handler+ # reading the message store puts the whole conversation on the clipboard, so scoring on the+ # selection would report a build that FIXED the data loss as still losing it. Read back before+ # the selection is cleared, and a failed read is reported rather than read as an empty clipboard.clip = Noneclip_reason = Nonetry:@@ -1652,9 +1410,9 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:clipboard_chars = len(clip) if isinstance(clip, str) else Nonectx.page.evaluate("() => window.getSelection().removeAllRanges()")- # NO CONFIRMED COPY, NO TIMING. Reported NOT RUN rather than as a number, because a reader who- # sees `copy_ms` has no way to tell a real copy from a keystroke that went nowhere, and the- # engines that fail this do so silently and consistently enough to look like data.+ # NO CONFIRMED COPY, NO TIMING: a reader seeing `copy_ms` cannot tell a real copy from a+ # keystroke that went nowhere, and the engines that fail do so consistently enough to look+ # like data.if pre_copy is not None and clip == pre_copy:held = ("it still holds the sentinel written before the keystroke"@@ -1687,22 +1445,16 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:expect_ok = ok,expect = {"selected_chars": raw["chars"],- # WHAT ACTUALLY REACHED THE CLIPBOARD. The user-facing quantity, and the one the- # truncation alarm is scored on. A windowed mount cannot SELECT what is not in the- # DOM, but it can still COPY it, if the app's copy handler reads the message store --- # which is the fix, and which this is the only reading able to see.+ # WHAT ACTUALLY REACHED THE CLIPBOARD, the user-facing quantity the truncation alarm is scored+ # on: a windowed mount cannot SELECT what is not in the DOM but can still COPY it from the store."clipboard_chars": clipboard_chars,"clipboard_readable": clip_reason is None,"clipboard_note": clip_reason,- # WHICH pre-copy value the change was confirmed against: the sentinel this action- # wrote, or a snapshot of what the clipboard already held when the write was refused.- # A snapshot is the weaker of the two -- see the note above the write -- and a reader- # comparing rows has no other way to see which one a timing rests on.+ # WHICH pre-copy value the change was confirmed against: the sentinel this action wrote, or a+ # snapshot taken when the write was refused (the weaker of the two)."copy_confirmed_against": pre_copy_source,- # And the DOM coverage beside it, so the two can be told apart. Where- # `mounted_fraction` is well below 1 and `clipboard_chars` is nonetheless whole, the- # copy-from-store path is working; where both are short, the conversation is being- # lost. That is a user-visible defect and must not be filed as a measurement problem.+ # The DOM coverage beside it: `mounted_fraction` well below 1 with whole `clipboard_chars` means+ # the copy-from-store path works; both short means the conversation is being lost."messages_total": total,"messages_mounted": mounted,"mounted_fraction": (@@ -1716,15 +1468,9 @@ def select_all_copy(ctx: ActionContext) -> ActionResult:"copy_ms": round(copy_ms, 1),"total_ms": round((time.monotonic() - started) * 1000, 1),},- # `expect_ok` stays `chars > 0` deliberately. Within ONE run there is nothing to compare- # against: the selection is taken over the viewport's DOM, so if the DOM is windowed the- # selection and every DOM-derived reference shrink together and agree with each other.- # An absolute floor would need calibrating per rung and per platform, and would still be a- # guess. The counts below are the real check, and they are paired against the other arm.- #- # `clipboard_chars` FIRST, because it is the one a user would notice. `selected_chars` is- # kept beside it as the mechanism: if the clipboard held and the selection fell, the copy- # handler is reading the store; if both fell, the conversation is being lost.+ # `expect_ok` stays `chars > 0`: within one run the selection and every DOM-derived reference+ # shrink together, and an absolute floor would need per-rung, per-platform calibration.+ # `clipboard_chars` comes first as the user-visible number, `selected_chars` as the mechanism.counts = {"clipboard_chars": clipboard_chars,"selected_chars": raw["chars"],@@ -1752,10 +1498,9 @@ def send_turn(ctx: ActionContext) -> ActionResult:"""pacer = ctx.args.get("_pacer")queue = ctx.args.get("_stream_queue") or []- # A SHARED MUTABLE cursor, not a scalar in `args`. The scene runner builds each action's args- # as `{**base_args, **slot.args}`, so a scalar written back here lands in a dict that is- # discarded when the action returns: the second send re-sent the first turn, reported- # `turn_index: 1` twice, and failed because that message was already in the thread.+ # A SHARED MUTABLE cursor, not a scalar in `args`: the runner rebuilds each action's args, so a+ # scalar written back is discarded and the second send re-sent the first turn.+ # Slot args merge as `{**base_args, **slot.args}`.cursor = ctx.args.get("_stream_cursor")if not isinstance(cursor, dict):return not_run("no shared stream cursor was passed to the action")@@ -1765,21 +1510,16 @@ def send_turn(ctx: ActionContext) -> ActionResult:if index >= len(queue):return not_run(f"the stream queue is exhausted ({len(queue)} turns planned)")if _ev(ctx, "() => window.__sb.dom.isRunning()"):- # Sending while a reply is in flight queues the message instead of starting a stream, and- # the action would report a fast, precise number about a message that is merely parked.+ # Sending while a reply is in flight queues the message instead of starting a stream, and the+ # action would report a precise number about a message that is merely parked.return not_run("a reply was still streaming, so this send would have been queued")unit = queue[index]cursor["i"] = index + 1- # NO `pacer.reset()` HERE. It used to clear the pacer's stats before loading the next turn, and- # `CellRunner` records `last_stats()` -- so the opening reply's `StreamStats` were discarded by- # the first follow-up and the first follow-up's by the second. An opening reply that- # disconnected or delivered 4,624 of its 10,000 characters was then erased by a later turn that- # DID finish, and the cell was marked complete and scored against an undersized thread, because- # the only other liveness signal is the UI no longer running and the later turn satisfies it.- # Every turn is tagged, so keeping them all costs one small record each and lets- # `check_planned_streams` verify the cell streamed what it planned. `CellRunner` still resets- # once per cell, which is the boundary that reset is for.+ # NO `pacer.reset()` HERE: `CellRunner` records `last_stats()`, so resetting discarded the+ # opening reply's StreamStats and let a later completed turn mask a disconnected one. Every turn+ # is tagged instead, so `check_planned_streams` can verify what the cell streamed.+ # One turn delivered 4,624 of its 10,000 characters.tag = f"{ctx.args.get('cell_id', 'cell')}#turn{index + 1}"pacer.load(unit["reasoning"],@@ -1793,10 +1533,8 @@ def send_turn(ctx: ActionContext) -> ActionResult:return not_run("no composer on the page")ctx.page.fill(selector, f"studiobench follow-up {index + 1}")ctx.page.wait_for_timeout(80)- # THE THREAD'S LENGTH, not the mounted count. "The thread grew" is a statement about the- # conversation; a windowed mount answers it about the viewport, and the answer it gives is- # `after == before` for a send that worked perfectly, because the new pair arrives at the- # bottom while two messages leave the top of the window.+ # THE THREAD'S LENGTH, not the mounted count: under a windowed mount a send that worked perfectly+ # reports `after == before`, since the new pair arrives as two messages leave the top.before = _ev(ctx, "() => window.__sb.dom.threadTotal()")mounted_before = _ev(ctx, "() => window.__sb.dom.messageCount()")started = time.monotonic()@@ -1811,8 +1549,8 @@ def send_turn(ctx: ActionContext) -> ActionResult:ctx.page.wait_for_timeout(50)after = _ev(ctx, "() => window.__sb.dom.threadTotal()")mounted_after = _ev(ctx, "() => window.__sb.dom.messageCount()")- # A POSITIVE consequence: the turn actually started streaming AND the thread grew. A send that- # silently failed leaves both unchanged and would otherwise read as an instant send.+ # A POSITIVE consequence: the turn actually started streaming AND the thread grew, so a silently+ # failed send cannot read as an instant one.ok = (first_ms is not Noneand isinstance(after, int)@@ -1825,16 +1563,15 @@ def send_turn(ctx: ActionContext) -> ActionResult:expect = {"messages_before": before,"messages_after": after,- # What was MOUNTED either side, kept beside the thread totals. On the shipped build- # these two pairs are identical; where they differ, the difference is the size of the- # window and it is the reading the whole virtualization arm exists to produce.+ # What was MOUNTED either side, kept beside the thread totals: identical on the shipped build,+ # and where they differ the difference is the window size the virtualization arm measures."mounted_before": mounted_before,"mounted_after": mounted_after,"turn_index": index + 1,"queued_turns": len(queue),"streamed_chars": len(unit["reasoning"]) + len(unit["content"]),- # The tag this turn's stream carries, so the cell can check the turn against the- # pacer's own record of it rather than re-deriving the naming rule from the outside.+ # The tag this turn's stream carries, so the cell checks it against the pacer's own record rather+ # than re-deriving the naming rule."pacer_tag": tag,"unit_kind": unit.get("kind"),},@@ -1843,9 +1580,9 @@ def send_turn(ctx: ActionContext) -> ActionResult:)-#: What the page can tell us when the attachments button cannot be found. Reads geometry, style,-#: hit-testing and the surrounding chrome, so the three explanations that look identical from a-#: `not_run` string look different here.+#: What the page can tell us when the attachments button cannot be found: geometry, style,+#: hit-testing and surrounding chrome, so three explanations that look identical from a `not_run`+#: string look different here.IMAGE_BUTTON_DIAGNOSTIC = """() => {const sel = 'button[aria-label="Tools and attachments"]';const all = [...document.querySelectorAll(sel)];@@ -1889,30 +1626,20 @@ def image_upload(ctx: ActionContext) -> ActionResult:png = ctx.args.get("image_path")if not png:return not_run("no image path was supplied to the action")- # `:visible`, and the FIRST visible match.- #- # Two reasons, both measured. `document.querySelector` returns the first match in DOCUMENT- # ORDER and the composer exists more than once: the welcome-screen instance stays in the tree- # behind the docked one, and compare mode has two threads. The hidden duplicate comes first,- # so a plain query handed back a button that can never be clicked and the action reported "not- # visible" about a control that a direct probe found at 36x36 pixels, fully opaque, on screen.- #- # And Playwright's actionability wait does not give up quickly on an unclickable element: at- # its 30s default it blocked a 9s slot for more than three times the slot's budget, to report- # that nothing happened. Everything here is bounded by what is left of the slot.+ # `:visible`, and the FIRST visible match: `document.querySelector` returns document order and+ # the composer exists more than once (the welcome-screen instance stays in the tree, compare mode+ # has two threads), so a plain query handed back an unclickable button while a direct probe found+ # the real one. Playwright's actionability wait also blocks for its 30s default, so everything+ # here is bounded by what is left of the slot.locator = ctx.page.locator('button[aria-label="Tools and attachments"]:visible').firsttry:plus = locator.element_handle(timeout = 2000)except Exception: # noqa: BLE001plus = Noneif plus is None:- # WHY, not just THAT. A bare "not visible" is the opaque zero of the action layer: it- # conflates a button that is absent, a button that is present and covered, and a locator- # that disagrees with the page, and it has already cost three wrong hypotheses. A direct- # probe found the control at 36x36, fully opaque and hit-testable, on a fresh chat and- # after a settings round trip and under a 20,000-character composer fill, so whatever- # this is, it is none of those. Carrying the state into the row means the NEXT run- # answers it instead of the next investigation.+ # WHY, not just THAT: a bare "not visible" conflates a button that is absent, one that is covered+ # and a locator that disagrees with the page, and has already cost three wrong hypotheses.+ # Carrying the probe state into the row means the next run answers it.return not_run("no visible attachments button on the composer: "+ json.dumps(_ev(ctx, IMAGE_BUTTON_DIAGNOSTIC) or {})@@ -1956,20 +1683,12 @@ def image_upload(ctx: ActionContext) -> ActionResult:)+#: The end of the thread as a plain string that can be looked for after the rebuild. NOT the+#: seeder's `last_marker`: `send_turn` appends turns mid-film, so the seeded marker sits several+#: messages from the end while `end_present` requires it within two rows. `trim()` then a prefix,+#: never a whitespace-collapsing normalisation, since readiness.PROBE_JS matches RAW text.# ── 13. thread reopen ───────────────────────────────────────────────-#: The end of the thread, as a plain string that can be looked for again after the rebuild.-#:-#: NOT the seeder's `last_marker`, even though that is the string the readiness gate uses at the-#: start of the cell. `send_turn` appends turns to the thread MID-FILM, twice in every scene, so by-#: the time this slot opens the seeded marker sits several messages from the end -- and-#: `end_present` requires the marker within two rows of the end, so it could never be satisfied.-#: Reading the last user turn out of the DOM immediately before leaving gives the marker for the-#: thread as it actually stands at this point in the film.-#:-#: `trim()` and then a prefix, never a whitespace-collapsing normalisation: readiness.PROBE_JS-#: matches with `textContent.includes(marker)` against RAW text, and trimming plus slicing from the-#: front leaves a genuine substring of that raw text while collapsing internal runs would not._LAST_USER_TEXT_JS = """() => {const rows = Array.from(document.querySelectorAll('[data-role="user"]'));@@ -1980,15 +1699,10 @@ _LAST_USER_TEXT_JS = """}"""-#: How long the rebuilt thread is given to reach the readiness gate's definition of ready, in-#: seconds: what is left of the SLOT, floored and capped.-#:-#: Bounded by the slot because the condition below is now one a broken arm can fail to satisfy-#: forever -- a thread that never stops mutating never settles -- and the film's own last slot,-#: `delete_message`, is the one place an overrun has nowhere to go (see scene/schedule.py, which-#: records a sweep where exactly that lost delete on every base-arm cell). Floored at 10s so a-#: tight budget still gives a 1M-token thread a fair chance, and capped at the flat minute the-#: previous "declared its total" loop waited, so nothing waits longer than it used to.+#: How long the rebuilt thread is given to reach the readiness gate, in seconds: what is left of+#: the SLOT, floored at 10s so a 1M-token thread still gets a fair chance and capped at the flat+#: minute the previous loop waited. Slot-bounded because a broken arm can fail the condition+#: forever and `delete_message`, the film's last slot, has nowhere to absorb an overrun._REOPEN_READY_FLOOR_S = 10.0_REOPEN_READY_CEILING_S = 60.0@@ -2005,52 +1719,34 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:thread_id = ctx.args.get("thread_id")if not thread_id:return not_run("no thread id was supplied to the action")- # The THREAD's length, not the mounted count: a windowed arm that reopens with a different- # scroll anchor mounts a different number of rows, and the exact-equality assertion at the end- # of this action would then fail for a reason that has nothing to do with the rebuild.+ # The THREAD's length, not the mounted count: a windowed arm reopening at a different scroll+ # anchor mounts a different number of rows and would fail the exact-equality assertion.before = _ev(ctx, "() => window.__sb.dom.threadTotal()")mounted_before = _ev(ctx, "() => window.__sb.dom.messageCount()")- # AN INTEGER, not merely truthy. `_ev` hands back `{"__error": ...}` when the page throws, and- # a dict is truthy: it would reach the comparisons below and the readiness gate's own- # `expected_messages` as a length, and raise there instead of being reported here.+ # AN INTEGER, not merely truthy: `_ev` hands back `{"__error": ...}` when the page throws, and a+ # truthy dict would reach the readiness gate as a length and raise there.if not isinstance(before, int) or before <= 0:return not_run("the thread has no messages to rebuild")- # WHICH READINESS MODE THIS ARM IS HELD TO, decided from the mount that is on screen now and- # therefore before anything is torn down. A thread whose declared total is larger than what it- # has mounted IS a windowed mount -- that is `isWindowed()` in scene/dom.js -- and it is the- # same distinction runtime/readiness.py draws between its two modes. Read from the page rather- # than taken from a session flag so both arms are judged by one rule.+ # WHICH READINESS MODE THIS ARM IS HELD TO, decided from the mount on screen before anything is+ # torn down: a declared total larger than the mounted count IS a windowed mount. Read from the+ # page rather than a session flag so both arms are judged by one rule.mode = (MODE_WINDOWED if isinstance(mounted_before, int) and mounted_before < before else MODE_FULL)marker = _ev(ctx, _LAST_USER_TEXT_JS)if not isinstance(marker, str) or not marker:- # Refused HERE, before a single thing has been touched, so a thread this action cannot- # verify is also a thread it has not disturbed. Without a string that identifies the end of- # the conversation there is no way to tell a rebuilt thread from a half-rebuilt one, which- # is the entire question the wait below exists to answer.+ # Refused HERE, before anything is touched, so a thread this action cannot verify is also one it+ # has not disturbed: without a string identifying the end of the conversation there is no way to+ # tell a rebuilt thread from a half-rebuilt one.return not_run("the last user turn carried no text to identify the end of the thread with, so a ""finished rebuild could not have been told from a partial one")started = time.monotonic()- # NO FALLBACK ON THE WAY OUT. Workspace task #102 and the defect it left behind.- #- # That task taught this action to REFUSE a substituted navigation, because a goto is a FULL- # DOCUMENT NAVIGATION -- the SPA is torn down and reparsed, the bundle re-executes, the runtime- # rehydrates -- while the click is a client-side route change that rebuilds one React subtree.- # They are not the same operation and they do not cost the same thing; read as though it were- # the click, the substitution produced thread_reopen at 6.0 fps, a number about a page load- # quoted as a number about a thread.- #- # But it refused AFTER THE FACT, by inspecting `path` once `page.goto` had already run. The row- # was then honest and the rest of the film was collateral damage: the scene carried on from an- # empty new chat, and `delete_message` -- the last slot of every film, three slots later --- # found no messages and went unexercised for a reason that had nothing to do with deleting.- #- # So the substitution is declined BEFORE it happens. Nothing is clicked, nothing is navigated,- # the thread is still on screen, and the slots after this one still have the thread they were- # written for.+ # NO FALLBACK ON THE WAY OUT. A goto is a FULL DOCUMENT NAVIGATION while the click is a+ # client-side subtree rebuild, and read as the click the substitution produced thread_reopen at+ # 6.0 fps. Refusing AFTER `page.goto` had run left the scene on an empty new chat, so the+ # substitution is now declined BEFORE it happens.leave = _click_or_navigate(ctx,'button[aria-label="New chat"]',@@ -2069,19 +1765,11 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:"page navigation; a document reload is not a thread rebuild, so the action was not run "f"and no navigation was performed ({leave.reason})")- # BOTH CLOCKS START AT THE CLICK THAT WORKED, not at the first attempt on it.- #- # `_click_or_navigate` tries `handle.click` first, and Playwright's hit-target check retries- # for its whole 2,000 ms timeout against a control whose centre does not hit-test to it. The- # New chat button IS such a control by design -- `opacity-0 pointer-events-none` until its- # header group is hovered -- so that timeout was paid on EVERY run and landed inside- # `close_ms`, which sweep/floor_table.py harvests and quotes against the other arm. A metric- # that is two seconds of harness retry plus the unmount is the same defect as reporting a- # settle delay as `copy_ms`, one action along.- #- # The retry is not discarded, it is MOVED: `left_click_retry_ms` and `reopen_click_retry_ms`- # below carry it as the harness's own cost, which is what it is. `started` still governs the- # budget arithmetic, because the retry really did consume the slot's wall clock.+ # BOTH CLOCKS START AT THE CLICK THAT WORKED. `_click_or_navigate` tries `handle.click` first,+ # and Playwright's hit-target check retries for its full 2,000 ms against the New chat button,+ # which is `opacity-0 pointer-events-none` until its header group is hovered, so that timeout+ # landed inside the `close_ms` floor_table quotes. The retry is MOVED to the retry fields.+ # Moved to `left_click_retry_ms` and `reopen_click_retry_ms`.click_left_at = leave.started_at if leave.started_at is not None else started# Unmount FIRST, or "already back" is indistinguishable from "never left".closed_ms = None@@ -2095,11 +1783,9 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:return not_run("the thread never unmounted, so the rebuild could not be timed")reopen_started = time.monotonic()- # THE FALLBACK IS ALLOWED ON THE WAY BACK, and refused on the way out, for one reason: what it- # does to the scene. From an empty new chat, a navigation to the thread's own URL is what puts- # the thread back on screen for the slots that follow, so as a REPAIR it is the right tool --- # where on the way out it was the thing that broke the scene. It is still not a measurement of- # a rebuild, which is what the `not_run` below says, and no timing is reported for it.+ # THE FALLBACK IS ALLOWED ON THE WAY BACK: from an empty new chat, navigating to the thread's own+ # URL restores the scene for the slots that follow. Still not a measurement of a rebuild, so no+ # timing is reported for it.back = _click_or_navigate(ctx, f'[data-thread-id="{thread_id}"]', f"{ctx.args['base_url']}/chat?thread={thread_id}")@@ -2117,33 +1803,11 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:"the thread's sidebar row could not be clicked and a full page navigation was "f"substituted, so the rebuild was never timed ({back.reason})")- # WHEN IS THE REBUILD OVER?- #- # NOT when `threadTotal()` reaches the length it had. On a windowed arm `threadTotal()` reads- # `aria-setsize`, which is the STORE'S CLAIM about how long the conversation is, and the very- # first reopened row publishes it: the condition was satisfied while three of eighteen messages- # were mounted, with no bottom window, no final assistant content and no syntax highlighting- # yet. The action then reported that as `reopen_ms`, took its census and its parity digest off- # a half-built DOM, and still passed its own assertion because `after == before` was a- # statement about the same declared total. A declaration is not a rebuild.- #- # WHAT `reopen_ms` MEASURES NOW: from the click on the thread's sidebar row until the reopened- # thread satisfies runtime/readiness.py's gate -- the composer is present, the END of the- # conversation is mounted (the last user turn, found by the marker captured above), and the- # mount has SETTLED, meaning two samples STABLE_GAP_MS apart agree on the mounted count, the- # element count and the laid-out height. Plus, in `windowed` mode, that the thread declares the- # right total on every row and is anchored at the end.- #- # THAT SAME FUNCTION, not a second definition of "ready" written here. Two disagreeing- # definitions in one harness is a defect of its own, and this is the definition the cell was- # already admitted under: the gate that let the cell start is the gate that decides the rebuild- # is finished, so the number is the cost of getting back to the state the cell began in.- #- # COMPARABILITY: every condition gated in `full` mode is one a fully mounted thread reaches by- # construction, and each arm is judged in the mode its own mount is in, exactly as the cell's- # opening gate judged it. The extra `windowed` conditions are a contract that arm already had- # to meet to be scored at all. What this costs is a floor of one STABLE_GAP_MS, paid equally by- # both arms, because "settled" is a claim about two samples and cannot be made from one.+ # WHEN IS THE REBUILD OVER? Not when `threadTotal()` reaches its old length: on a windowed arm+ # that reads `aria-setsize`, the STORE'S CLAIM, which the first reopened row publishes while+ # three of eighteen messages are mounted. `reopen_ms` now runs from the sidebar click until+ # runtime/readiness.py's own gate passes. That same function, not a second definition of ready,+ # and each arm is judged in its own mode; the cost is one STABLE_GAP_MS paid equally by both.left_ms = ctx.budget_ms - (time.monotonic() - started) * 1000timeout_s = min(_REOPEN_READY_CEILING_S, max(_REOPEN_READY_FLOOR_S, left_ms / 1000))reopen_ms = None@@ -2159,13 +1823,12 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:reopen_ms = (time.monotonic() - click_back_at) * 1000readiness = ready.as_dict()except ThreadNotReady as exc:- # A REAL FINDING ABOUT THE ARM, and it keeps `ran = True` so the row carries which- # conditions were still outstanding. `timings["reopen_ms"]` stays null, which the scoring- # layer reads as "ran but recorded no reopen_ms" rather than as a fast rebuild.+ # A REAL FINDING ABOUT THE ARM: `ran` stays True with the outstanding conditions on the row, and+ # `reopen_ms` stays null so scoring does not read it as a fast rebuild.readiness = exc.detailexcept Exception as exc: # noqa: BLE001- # The probe itself failed (a closed page, an uninstalled `window.__sb`). Nothing was- # learned about the rebuild, so nothing is claimed about it.+ # The probe itself failed (a closed page, an uninstalled `window.__sb`), so nothing is claimed+ # about the rebuild.ctx.log(f" the readiness probe failed during the reopen: {type(exc).__name__}: {exc}")return not_run(f"the reopened thread could not be probed for readiness: {type(exc).__name__}: {exc}"@@ -2185,21 +1848,15 @@ def thread_reopen(ctx: ActionContext) -> ActionResult:"highlight_spans_after": spans,"left_via": leave.path,"reopened_via": back.path,- # THE HARNESS'S OWN COST, named as such and kept OUT of the two timings above. This is- # how long `_click_or_navigate` spent on attempts that failed before the one that- # worked -- almost always Playwright's 2,000 ms hit-target retry against a control it- # cannot hit at the centre. It belongs in the payload (a run where this grows is a run- # whose controls have moved) and it does not belong in a rebuild timing.+ # THE HARNESS'S OWN COST, kept OUT of the two timings above: how long `_click_or_navigate` spent+ # on attempts that failed, almost always Playwright's 2,000 ms hit-target retry."left_click_retry_ms": round((click_left_at - started) * 1000, 1),"reopen_click_retry_ms": round((click_back_at - reopen_started) * 1000, 1),- # WHICH GATE THE TIMING WAS TAKEN AGAINST, beside the timing itself. Two arms in- # different readiness modes are answering slightly different questions, and a reader- # comparing their `reopen_ms` has no other way to see that.+ # WHICH GATE THE TIMING WAS TAKEN AGAINST: two arms in different readiness modes are answering+ # slightly different questions."reopen_ready_mode": mode,- # The gate's own verdict: every condition, the last probe reading and how many samples- # it took. When the rebuild did not finish, this is what says which condition was still- # outstanding -- "mounted 3 of 18, aria-setsize 18" is the difference between a slow- # app and an arm that publishes a total it has not built.+ # The gate's own verdict: "mounted 3 of 18, aria-setsize 18" is the difference between a slow app+ # and an arm that publishes a total it has not built."reopen_readiness": readiness,},timings = {@@ -2229,18 +1886,10 @@ class Transition:ok: boolpath: str # "click", "navigate" or "failed"reason: str = ""- #: The monotonic instant the interaction that SUCCEEDED was issued, so a caller timing "from- #: the click" can start its clock at the click rather than at the first attempt. `None` when- #: nothing succeeded.- #:- #: THE FAILED ATTEMPT IS NOT PART OF THE GESTURE. Playwright's hit-target check reads- #: `elementFromPoint` at the control's centre and retries until the timeout, so a control that- #: is laid out but not hit-testable at its centre -- the sidebar's `New chat`, which ships- #: `opacity-0 pointer-events-none` until its group is hovered, and which therefore falls- #: through to the group underneath on every hit test -- costs the full 2,000 ms before the- #: reveal path below succeeds. Timed from the first attempt, `thread_reopen.close_ms` was- #: two seconds of Playwright retry with the unmount somewhere inside it, on every run, and- #: sweep/floor_table.py quotes it as a metric.+ #: The monotonic instant the interaction that SUCCEEDED was issued, so a caller can time from the+ #: click rather than the first attempt; None when nothing succeeded. The failed attempt is not+ #: part of the gesture: Playwright's hit-target check retries for the full 2,000 ms against the+ #: sidebar's `New chat`, which made `thread_reopen.close_ms` two seconds of retry.started_at: Optional[float] = None@property@@ -2248,9 +1897,8 @@ class Transition:return self.path == "navigate"-#: Where on an element to look for a point a user could actually hit. Fractions of the box, centre-#: first (so a normal control behaves exactly as before), then the lower band, which is what stays-#: uncovered when a sticky header overlaps the top of a control.+#: Where on an element to look for a point a user could hit: fractions of the box, centre first,+#: then the lower band that stays uncovered when a sticky header overlaps the top._HIT_POINTS = ((0.5, 0.5),(0.5, 0.75),@@ -2262,13 +1910,9 @@ _HIT_POINTS = ((0.5, 0.25),)-#: EVERY match, not the first. `document.querySelector` returns the first element in DOCUMENT-#: ORDER, and this app renders "New chat" twice -- once in the sidebar header and once on the chat-#: page -- exactly as it renders the composer twice. When the sidebar is collapsed its button is-#: still in the tree at zero size, comes first, and is the one a plain query hands back: the probe-#: then reports "no point on the control hit-tests to it" about a button that is 34x34 and fully-#: clickable a few hundred pixels away. This is the same trap `image_upload` documents at length,-#: and it cost this action a second time.+#: EVERY match, not the first: this app renders "New chat" twice, and a collapsed sidebar leaves+#: its zero-size button first in document order, so a plain query reported "no point hit-tests to+#: it" about a fully clickable button elsewhere. The same trap `image_upload` documents._HIT_TEST_JS = """([selector, points]) => {for (const el of document.querySelectorAll(selector)) {@@ -2297,18 +1941,14 @@ def _reachable_point(ctx: ActionContext, selector: str) -> tuple[float, float] |return None-#: How long the reveal transition needs, and how many times to look. `.sidebar-header-action`-#: transitions opacity over 150ms. One sample after a fixed sleep is a race that this test suite+#: `.sidebar-header-action` transitions opacity over 150ms, and one sample after a fixed sleep#: lost roughly one run in three on a loaded machine, so the hit test is retried instead._REVEAL_SETTLE_MS = 120_REVEAL_ATTEMPTS = 5-#: The control's own box, whatever its computed style says. An earlier version only offered a-#: hover point for a control that LOOKED hidden, which made the reveal conditional on a style read-#: taken at exactly the wrong moment: mid-transition `getComputedStyle` reports the old opacity,-#: and after a failed Playwright click the element can read as already revealed while not yet-#: being hit-testable. Since this only runs when the control is already known to be unreachable,-#: there is nothing to gain by being clever about whether it deserves a hover.+#: The control's own box, whatever its computed style says: mid-transition `getComputedStyle`+#: reports the old opacity, and after a failed click the element can read as revealed while not+#: yet hit-testable. This only runs when the control is already known to be unreachable._HOVER_TARGET_JS = """(selector) => {for (const el of document.querySelectorAll(selector)) {@@ -2398,19 +2038,11 @@ def _click_or_navigate(return Transition(ok = True, path = "click", started_at = attempt_at)except Exception as exc: # noqa: BLE001click_error = f"{selector} was not clickable: {type(exc).__name__}"- # THE CENTRE IS COVERED, BUT THE CONTROL IS NOT. Try the rest of it, the way a user would.- #- # Playwright clicks the centre of an element and refuses when something else hit-tests- # there. The sidebar's sticky group label overlaps the top of the New chat button, so the- # centre is taken and the click times out -- but most of the button is uncovered and a- # person clicks it every day without noticing. Giving up at the centre and calling the- # control unreachable is the harness being more fragile than a user, and it cost this- # action on every CI run.- #- # So the element's own box is hit-tested at a spread of points and the first one that- # resolves to the control is clicked with a real mouse event. If NO point on the control- # is reachable then it genuinely cannot be clicked, which is a finding worth reporting- # rather than a reason to substitute a page load.+ # THE CENTRE IS COVERED, BUT THE CONTROL IS NOT. Playwright clicks the centre and refuses when+ # something else hit-tests there; the sidebar's sticky group label overlaps the top of New chat+ # while most of the button is clickable. So the box is hit-tested at a spread of points and the+ # first that resolves to the control is clicked with a real mouse event; no reachable point is a+ # finding worth reporting rather than a reason to substitute a page load.point = _reachable_point(ctx, selector) or _reveal_by_hover(ctx, selector)if point is not None:try:@@ -2523,18 +2155,10 @@ async (opts) => {"""-#: How long an action that needs a message's ACTION BAR will wait for the reply to finish.-#:-#: Unsloth hides the action bar while a message is generating, so "no More button on the last-#: assistant message" and "no Delete button (a running message hides it)" are the same fact-#: reported by two actions. The film schedules `message_menu` about four seconds after a-#: `send_turn` whose reply runs for roughly fourteen, so the menu was being asked for on a message-#: that was still being written, and the action reported NOT RUN on every CI run.-#:-#: Waiting is what a USER does, and it is the only thing that makes the action measurable at all:-#: there is no menu to open until the reply lands. Bounded by the slot's own budget so a reply-#: that never finishes cannot swallow the rest of the film, and the wait happens BEFORE the-#: operation the action times, so `menu_open_ms` is unaffected.+#: How long an action that needs a message's ACTION BAR waits for the reply to finish. Unsloth+#: hides the bar while a message generates, and the film schedules `message_menu` about four+#: seconds after a `send_turn` whose reply runs for roughly fourteen, so the action reported NOT+#: RUN on every CI run. Bounded by the slot's budget, and waited BEFORE the timed operation._ACTION_BAR_WAIT_FRACTION = 0.6_ACTION_BAR_POLL_MS = 100@@ -2556,14 +2180,10 @@ def _wait_for_the_reply_to_land(ctx: ActionContext) -> bool:@register_action(name = "message_menu", default_budget_ms = 12000)def message_menu(ctx: ActionContext) -> ActionResult:- # TWO WAITS, AND THEY COVER DIFFERENT THINGS. This one waits for the STREAM to stop, bounded by- # a fraction of the slot's own budget, because a follow-up reply runs for seconds and Unsloth- # unmounts the whole action bar for its whole duration -- far longer than the page-side wait- # below is allowed to sit. `waitForButtonMs` then waits for the BAR TO MOUNT, which happens a- # few hundred milliseconds after the reply settles, and which this one cannot see because- # `isRunning()` is already false by then. Dropping either leaves a real NOT RUN on the table:- # without the first, a fourteen second reply outlasts a 1.5 s page-side wait; without the- # second, the slot asks for a control that has not been mounted yet.+ # TWO WAITS, COVERING DIFFERENT THINGS: this one waits for the STREAM to stop, bounded by a+ # fraction of the slot, while `waitForButtonMs` waits for the BAR TO MOUNT a few hundred ms+ # later, which this one cannot see because `isRunning()` is already false.+ # See `OWN_TURN_STOP_POLL_MS`.if not _wait_for_the_reply_to_land(ctx):return not_run("a reply was still generating when the slot's budget ran out, and Unsloth hides the "@@ -2581,9 +2201,8 @@ def message_menu(ctx: ActionContext) -> ActionResult:return not_run(raw.get("reason", "the menu action did not run"))waited_ms = raw.get("waitedMs")if waited_ms:- # Logged rather than swallowed. A cell that had to wait says so, because a wait means the- # film opened this slot before the follow-up had drained, and that is a fact about the- # PACKING which is worth seeing even though the action ran.+ # Logged rather than swallowed: a cell that had to wait says so, because the wait is a fact about+ # the film's PACKING.ctx.log(f" message_menu waited {waited_ms}ms for the action bar to be mounted")# Opened AND closed AND a non-zero item count. Any one of the three alone can be satisfied by# a menu that never rendered its items.@@ -2593,13 +2212,12 @@ def message_menu(ctx: ActionContext) -> ActionResult:expect_ok = ok,expect = {"items_while_open": raw["items"],- # Radix puts the body on the modal layer while the menu is up, which is the- # fan-out under suspicion. This proves the open really took that path.+ # Radix puts the body on the modal layer while the menu is up, which is the fan-out under+ # suspicion, so this proves the open really took that path."body_pointer_events_open": raw["bodyPointerEvents"],"body_pointer_events_closed": raw["bodyPointerEventsAfterClose"],- # In the payload, not only in the log: a run whose cells all waited is a run whose- # film opens this slot too early, and that is not visible from a row that only says- # `ran`.+ # In the payload, not only in the log: a run whose cells all waited is a run whose film opens+ # this slot too early."action_bar_wait_ms": waited_ms,},timings = {@@ -2682,9 +2300,8 @@ def delete_message(ctx: ActionContext) -> ActionResult:return not_run(err)if not raw.get("ran"):return not_run(raw.get("reason", "the delete action did not run"))- # The THREAD TOTAL dropped. A delete that detached the node but left the thread the same- # length is a different bug -- or, on a windowed mount, a node the virtualizer recycled -- and- # must not read as a successful delete.+ # The THREAD TOTAL dropped: a delete that detached the node but left the thread the same length+ # is a different bug, or on a windowed mount a node the virtualizer recycled.ok = raw["ms"] is not None and raw["after"] < raw["before"]return ActionResult(ran = True,diff --git a/tests/studio/studiobench/scene/dom.js b/tests/studio/studiobench/scene/dom.jsindex 7803f7b9ca2..3f41be6aa5a 100644--- a/tests/studio/studiobench/scene/dom.js+++ b/tests/studio/studiobench/scene/dom.js@@ -1,27 +1,17 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//-// The selector adapter for the REAL Unsloth chat UI.-//-// The salvaged action JS from playwright_heavy_thread.py calls `window.__heavyThread`, an API the-// smoke fixture exported. The shipping app exports nothing of the kind, so the actions are ported-// onto this: the same method names, backed by the app's own selectors. One file to edit when a-// class changes, instead of fifteen.-//-// The chat thread has essentially NO test ids -- exactly one, `composer-tool-status`, in the whole-// of components/assistant-ui and features/chat. So the contract is class hooks (`aui-*`,-// `unsloth-*`), `data-role`, `data-slot` and accessible names, all read out of the shipped TSX.-//-// Three selectors carry a trap and are handled here rather than in each action:-//-// - The composer exists TWICE in compare mode and once on the welcome screen vs once docked, so-// everything scopes to the first thread root.-// - The Stop button is REPLACED by a Queue button when the composer has text, because-// `queueDisabled` depends on `composerText.trim().length > 0`. An action that presses stop-// with text in the box presses queue and measures nothing.-// - Radix keeps a collapsed Collapsible's content element mounted for its animation, so-// "the reasoning pane is open" cannot be read from the content element's presence. It is read-// from `data-state` on the root, which is what Radix actually toggles.+// The selector adapter for the REAL Unsloth chat UI: the salvaged actions call the smoke+// fixture's `window.__heavyThread` API and are ported onto this instead. Same method names,+// the app's own selectors, one file to edit when a class changes.+// Salvaged from playwright_heavy_thread.py.+// The chat thread has essentially no test ids (exactly one, `composer-tool-status`), so the+// contract is class hooks (`aui-*`, `unsloth-*`), `data-role`, `data-slot` and accessible+// names read out of the shipped TSX.+// Three selectors carry a trap, handled here rather than per action: the composer exists TWICE+// in compare mode (so everything scopes to the first thread root); the Stop button is REPLACED+// by Queue when the composer has text, since `queueDisabled` follows `composerText`, so+// pressing stop with text measures nothing; and Radix keeps a collapsed Collapsible's content+// mounted for its animation, so "open" is read from `data-state` on the root.(() => {if (window.__sb && window.__sb.dom) return;@@ -31,8 +21,8 @@const qa = (sel, root) => Array.from((root || document).querySelectorAll(sel));// Accessible name for the app's TooltipIconButton, which renders the label in a visually- // hidden span rather than an aria-label. `getByRole(name:)` on the driver side would do this,- // but the actions run inside one page.evaluate so they need it here.+ // hidden span rather than an aria-label. `getByRole(name:)` would do this driver-side, but+ // the actions run inside one page.evaluate.const nameOf = (el) => {if (!el) return "";const aria = el.getAttribute("aria-label");@@ -43,11 +33,10 @@};// The two hooks the app publishes a part's progress through: assistant-ui's `data-status` on a- // text part (markdown-text.tsx) and `aria-busy` on the reasoning content (reasoning.tsx). Named- // once because the streaming scan and the control on it must never drift apart.- // Every control ComposerRightControls can put in the run-state slot, in no significant order.- // "Stop research" / "Stopping research" are included because they are the same slot, even though- // nothing in this benchmark can reach a research run today.+ // text part (markdown-text.tsx) and `aria-busy` on the reasoning content (reasoning.tsx).+ // Named once so the streaming scan and its control cannot drift apart.+ // Every control ComposerRightControls can put in the run-state slot. The research pair is+ // included because it is the same slot, even though nothing here can reach a research run.const RUN_STATE_CONTROLS = ["Stop generating","Stop queued message",@@ -87,133 +76,84 @@queueButton() {return q('button[aria-label="Queue message"]');},- // THE DISPATCHED HALF OF A QUEUE RUN. `ComposerRightControls` renders this once the queued- // entry has been dispatched, under `isQueueRunning && !thread.isRunning` (thread.tsx), so the- // thread reports itself NOT running and neither `stopButton()` nor `queueButton()` matches.- // Read on its own, that transient came out as an ordinary idle composer: `streaming` and- // `queued_idle` both false, exactly as a settled Send arm reads. An arm caught here against a- // settled one then had a differing composer with no run-state difference to explain it, which- // is the shape the comparison layer treats as a rendering regression.+ // THE DISPATCHED HALF OF A QUEUE RUN: `ComposerRightControls` renders this under+ // `isQueueRunning && !thread.isRunning`, so the thread reports itself NOT running and neither+ // `stopButton()` nor `queueButton()` matches. Read on its own that transient looked like a+ // settled Send arm, giving a differing composer with no run-state difference to explain it:+ // the shape the comparison layer treats as a rendering regression.stopQueuedButton() {return q('button[aria-label="Stop queued message"]');},isRunning() {return Boolean(D.stopButton() || D.queueButton());},- // WHICH RUN-STATE CONTROL THE COMPOSER IS RENDERING, as a token.- //- // `ComposerRightControls` renders exactly one of these at a time and which one is a function- // of the run state: Send when nothing is happening, Stop while a reply is being written, Queue- // while one is queued or while text sits in the box mid-reply, and the research pair while a- // research run is going. They are DIFFERENT SUBTREES, and the composer dock is inside- // `.aui-thread-root`, so `digest_scaffold` carries whichever one is up.- //- // Two arms showing different tokens differ in the scaffold FOR THAT REASON, with no rendering- // difference between them. `isRunning()` is too coarse to say so: it is true for Stop AND for- // Queue, so a queued-idle arm and a streaming arm agree on it while rendering different- // controls. The token is the thing the scaffold actually contains.+ // WHICH RUN-STATE CONTROL THE COMPOSER IS RENDERING, as a token. `ComposerRightControls`+ // renders exactly one at a time as a function of the run state, and they are DIFFERENT+ // SUBTREES inside `.aui-thread-root`, so `digest_scaffold` carries whichever is up and two+ // arms showing different tokens differ in the scaffold for that reason. `isRunning()` is too+ // coarse to say so, being true for Stop AND Queue.runStateControl() {for (const name of RUN_STATE_CONTROLS) {if (byName("button", name)) return name;}return "";},- // THE PROMPT QUEUE'S OWN SURFACE. `PromptQueueStack` renders the waiting prompts inside the- // composer root with the accessible name "Prompt queue, <n> of <m>", and that is the only- // place the queue states its own existence in the DOM. Scoped to the thread root so a queue- // running on ANOTHER thread, which the sidebar also renders, is not read as this one's.+ // THE PROMPT QUEUE'S OWN SURFACE: `PromptQueueStack` renders waiting prompts inside the+ // composer root with the accessible name "Prompt queue, <n> of <m>", the only place the queue+ // states its existence in the DOM. Scoped to the thread root so another thread's queue is not+ // read as this one's.promptQueue() {return q('[aria-label^="Prompt queue,"]', D.threadRoot());},- // A REPLY IS ACTUALLY BEING WRITTEN, which is NOT the question `isRunning()` answers.- //- // `isRunning()` has to accept the Queue button: with text in the composer a running thread- // renders that button and NO Stop button (`queueDisabled` depends on the composer having a- // queueable prompt), so an action that waited for Stop would wait out its budget on a live- // stream. Every caller that asks "may I send now" wants exactly that broad reading.- //- // It is the wrong reading for a positive control on the streaming probe, because- // `ComposerRightControls` renders "Queue message" in a SECOND place: under- // `isQueueRunning && !thread.isRunning`, while a queued prompt waits to be dispatched and- // nothing at all is generating. `isRunning()` cannot tell that queued-idle interval from a- // live stream whose `data-status` hooks have gone quiet, and those two need opposite- // treatment -- one is an ordinary settled thread, the other is an instrument that has stopped- // working.- //- // The queued-idle button always comes with the queue surface: `syncPromptQueueUI` marks the- // entry `dispatched` from the active item, and `getPromptQueueUIItemsForRun` drops only- // DISPATCHED items, so an undispatched active item -- the one that renders "Queue message"- // rather than "Stop queued message" -- is always in the list the stack renders.- //- // WHAT THIS GIVES UP, pinned in scene/selftest/test_studiobench_queued_idle_live.py: while a- // queue run holds further prompts AND a reply is streaming AND the composer has text, the- // queue surface is up and the only control is the Queue button, so this reads false and the- // control is not armed for that capture. Under-claiming, and cheap: probe blindness is a- // renamed selector, which is global, so every other capture in the run still catches it.- // A RESEARCH RUN IS A GENERATION THIS PREDICATE CANNOT SEE, and deliberately so for now.- // `ComposerRightControls` renders "Stop research" / "Stopping research" under- // `isResearchActive`, which neither `stopButton()` nor `queueButton()` matches, and a research- // report renders through `MarkdownPreview` rather than the assistant text part, so- // `streamingMessages()` finds no `data-status` for it either. All THREE are blind to it- // together.- //- // Not patched, because nothing here can start one: `ResearchMessage` is gated on- // `message.metadata` and the seeder writes `None` for every message, and no action, corpus- // unit or scene touches the composer's research toggle. Fixing one of the three would leave- // two that still disagree with it. If a research scene is ever added, move all three in the- // same change, and add the report's own busy hook to `STATUS_HOOK` at the same time.+ // A REPLY IS ACTUALLY BEING WRITTEN, which is NOT what `isRunning()` answers. `isRunning()`+ // must accept the Queue button, since with text in the composer a running thread renders no+ // Stop button, and every "may I send now" caller wants that broad reading. It is wrong for a+ // positive control on the streaming probe, because "Queue message" is also rendered under+ // `isQueueRunning && !thread.isRunning` while nothing is generating, and that queued-idle+ // interval needs the opposite treatment from a live stream whose hooks have gone quiet. The+ // queued-idle button always comes with the queue surface, since+ // `getPromptQueueUIItemsForRun` drops only DISPATCHED items. WHAT THIS GIVES UP (pinned in+ // test_studiobench_queued_idle_live.py): with a queue run, a streaming reply and text in the+ // composer this reads false and the control is not armed. Under-claiming, and cheap, since+ // probe blindness is a renamed selector and therefore global.+ // `syncPromptQueueUI` marks dispatched items.+ // A RESEARCH RUN IS A GENERATION THIS PREDICATE CANNOT SEE, deliberately: "Stop research"+ // matches neither `stopButton()` nor `queueButton()`, and a research report renders through+ // `MarkdownPreview` rather than the assistant text part, so `streamingMessages()` finds no+ // `data-status` either. Not patched, because nothing here can start one: `ResearchMessage` is+ // gated on `message.metadata` and the seeder writes `None`. If a research scene is added, move+ // all three in one change and add the report's busy hook to `STATUS_HOOK`.+ // The state is `isResearchActive`.generating() {if (D.stopButton()) return true;if (!D.queueButton()) return false;return !D.promptQueue();},- // WHICH MESSAGES ARE STILL BEING WRITTEN, read from the app's own published state rather than- // guessed at from `isRunning()` plus "it is probably the last one".- //- // assistant-ui gives every text part a status and Unsloth serialises it: markdown-text.tsx- // renders `<div data-status={status.type}>` around the Streamdown tree, so a part that is- // still arriving reads `data-status="running"` and a finished one reads `"complete"`. The- // reasoning pane publishes the same fact as `aria-busy` on `[data-slot="reasoning-content"]`- // (reasoning.tsx), which is a separate part of the same message and can be running while the- // answer is not, or the other way round. Either one means this message's DOM is mid-flight.- //- // Both are the APP's statements about its own state. Nothing here reads a timer, a character- // count or "the last assistant message", all of which are the benchmark inventing a fact the- // page already publishes -- and inventing it is how you end up attributing a stream to the- // wrong message on the arm that renders faster.+ // WHICH MESSAGES ARE STILL BEING WRITTEN, read from the app's own published state:+ // markdown-text.tsx renders `<div data-status={status.type}>`, and the reasoning pane+ // publishes the same fact as `aria-busy` on `[data-slot="reasoning-content"]`, a separate part+ // that can be running while the answer is not. Both are the APP's statements about its own+ // state; reading a timer, a character count or "the last assistant message" is how you+ // attribute a stream to the wrong message on the arm that renders faster.streamingMessages() {return qa("[data-role]").filter((m) => m.querySelector('[data-status="running"], [aria-busy="true"]') !== null,);},- // ── IS THE STREAMING PROBE BLIND, OR IS THERE SIMPLY NOTHING FOR IT TO SEE ──────────- //- // `streamingMessages()` returning nothing has three completely different causes, and the- // positive control on it is only worth having if they are told apart.- //- // THE HOOK IS GONE a build renamed or dropped `data-status`. It is one line in- // markdown-text.tsx and it is rendered for `complete` parts as- // well as `running` ones, so on a working build EVERY assistant- // message that has rendered a part carries it, and on a blinded- // build none does.- // THE ROW IS NOT MOUNTED a windowed arm scrolled away from the tail has unmounted the- // message it is writing into. That is what windowing is for.- // THE ROW HAS NO PARTS YET between the send being accepted and the reply's first part- // arriving, the assistant message is mounted with zero content- // parts and thread.tsx renders "Generating..." in its place. It- // publishes no status because it has nothing to publish yet, and- // `send_turn` returns the instant `isRunning()` flips, so a- // capture lands here twice a film.- //- // Scoped to ASSISTANT messages throughout: a user message never publishes `data-status` even- // on a perfectly working build, because only the assistant parts are rendered through- // `MarkdownText` (thread.tsx `ASSISTANT_PART_COMPONENTS`). A window holding only user rows- // would otherwise read as a build with no hook.+ // IS THE STREAMING PROBE BLIND, OR IS THERE NOTHING TO SEE? `streamingMessages()` returning+ // nothing has three causes worth telling apart. THE HOOK IS GONE: a build renamed+ // `data-status`, which is rendered for `complete` parts too, so on a working build every+ // assistant message that rendered a part carries it. THE ROW IS NOT MOUNTED: a windowed arm+ // scrolled away from the tail, which is what windowing is for. THE ROW HAS NO PARTS YET:+ // between the send being accepted and the first part arriving, thread.tsx renders+ // "Generating...", and `send_turn` returns the instant `isRunning()` flips, so a capture lands+ // here twice a film. Scoped to ASSISTANT messages, since only assistant parts render through+ // `MarkdownText`.statusHookPresent() {return D.assistantMessages().some((m) => m.querySelector(STATUS_HOOK) !== null);},- // Whether the message a reply would be written INTO is publishing parts this probe can read.- // False means it has none yet, which is the third case above and not a broken instrument.+ // Whether the message a reply would be written INTO is publishing parts this probe can read;+ // false means it has none yet, the third case above and not a broken instrument.lastAssistantPublishesStatus() {const last = D.lastAssistantMessage();return Boolean(last && last.querySelector(STATUS_HOOK));@@ -224,25 +164,16 @@messageCount() {return qa("[data-role]").length;},- // HOW LONG THE THREAD IS, as opposed to how much of it is mounted.- //- // On the shipped build these are the same number and this returns exactly what- // messageCount() does, so nothing about an existing run changes. They stop being the same- // number the moment an arm mounts a window, and at that point every before/after assertion in- // actions.py -- send_turn's "the thread grew", delete's "the count dropped", thread_reopen's- // "it came back with the same messages" -- is asking about the THREAD and being answered- // about the window. Under a windowed mount those three all read the wrong answer in the same- // direction: the window refills as fast as it empties, so a delete that worked reports- // `after == before` and fails, and a send that worked reports the same and fails.- //- // aria-setsize is where a windowed list is already required to publish this: WAI-ARIA says a- // list whose items are not all in the DOM must carry aria-setsize and aria-posinset. So this- // is not a private channel invented for the benchmark, it is the accessible name for the- // quantity, and an arm that does not provide it is one a screen reader cannot navigate.+ // HOW LONG THE THREAD IS, as opposed to how much is mounted. Identical to messageCount() on+ // the shipped build, and different the moment an arm mounts a window, at which point+ // send_turn's "the thread grew", delete's "the count dropped" and thread_reopen's "same+ // messages" ask about the THREAD and are answered about the window, all in the same direction,+ // because the window refills as fast as it empties. aria-setsize is where WAI-ARIA already+ // requires a windowed list to publish this, so it is the accessible name for the quantity+ // rather than a private channel.threadTotal() {- // On the message, or on the row wrapper a virtualizer positions it in. See the same walk in- // runtime/readiness.py: the ordinal belongs on the element that is a member of the set, and- // for a windowed list that is the positioned row rather than the message inside it.+ // On the message, or on the row wrapper a virtualizer positions it in. Same walk as+ // runtime/readiness.py: the ordinal belongs on the element that is a member of the set.const first = q("[data-role]");const owner = first ? first.closest("[aria-setsize]") : q("[aria-setsize]");if (owner) {@@ -251,8 +182,8 @@}return qa("[data-role]").length;},- // True when the thread is publishing a total that is larger than what it has mounted, i.e.- // this really is a windowed mount and not merely a short thread.+ // True when the thread publishes a total larger than what it has mounted, i.e. a windowed+ // mount and not merely a short thread.isWindowed() {return D.threadTotal() > qa("[data-role]").length;},@@ -264,19 +195,16 @@return all.length ? all[all.length - 1] : null;},- // ── following the stream ─────────────────────────────────────- //- // The jump-to-bottom control, which thread.tsx renders permanently and hides with- // `invisible` when the intent-aware autoscroll reports itself at the bottom. Reading the- // app's own state rather than recomputing it means the harness and the app cannot disagree- // about whether the thread is pinned, which is the whole question here.+ // The jump-to-bottom control, which thread.tsx renders permanently and hides with `invisible`+ // when the intent-aware autoscroll reports itself at the bottom. Reading the app's own state+ // means the harness and the app cannot disagree about whether the thread is pinned.jumpToBottomButton() {return q(".aui-thread-scroll-to-bottom");},appSaysAtBottom() {const jump = D.jumpToBottomButton();- // `null`, NOT `false`, when the control is absent. A build that does not render it has not- // told us it is scrolled up; it has told us nothing, and the two must not be summed.+ // `null`, NOT `false`, when the control is absent: a build that does not render it has told+ // us nothing, and the two must not be summed.return jump ? jump.classList.contains("invisible") : null;},distanceFromBottom() {@@ -285,7 +213,6 @@return Math.round(vp.scrollHeight - vp.clientHeight - vp.scrollTop);},- // ── reasoning ────────────────────────────────────────────────reasoningRoots() {return qa('[data-slot="reasoning-root"]');},@@ -297,25 +224,19 @@// content mounted for the animation, so a presence check reads every pane as open.return qa('[data-slot="reasoning-root"][data-state="open"]').length;},- // STILL MOUNTED IS NOT THE SAME AS STILL OPEN, and closing needs the other one.- //- // `reasoningOpenCount` flips on the click, because `data-state` is the state. The CHILDREN- // outlive it on both collapse mechanisms and by design: Radix's `Presence` suspends the- // unmount until `animationend` of `animate-collapsible-up`, and the grid arm's- // `UnmeasuredCollapsibleContent` renders `present && children` until the `grid-template-rows`- // `transitionend` or its 250 ms backstop. For that whole window every pane is closed, every- // span it contributed is still in the document, and a census asked whether it has stopped- // moving answers yes -- because it has not started.- //- // So a collapse is settled when the content is GONE, which is what this counts. Both arms are- // covered by one selector: Radix removes the element, the grid arm leaves it behind with- // `hidden`, and neither is a mounted pane.+ // STILL MOUNTED IS NOT STILL OPEN. `reasoningOpenCount` flips on the click, but the CHILDREN+ // outlive it on both collapse mechanisms by design: Radix's `Presence` suspends the unmount+ // until `animationend`, and the grid arm renders `present && children` until `transitionend`+ // or its 250 ms backstop. For that window every pane is closed while every span it contributed+ // is still in the document, and a census asked whether it has stopped moving answers yes+ // because it has not started. So a collapse is settled when the content is GONE, which one+ // selector covers on both arms.+ // The grid arm is `UnmeasuredCollapsibleContent`.reasoningContentMounted() {return qa('[data-slot="reasoning-content"]').filter((el) => !el.hasAttribute("hidden")).length;},- // ── action bar ───────────────────────────────────────────────actionBar(message) {const m = message || D.lastAssistantMessage();if (!m) return null;@@ -330,9 +251,9 @@const m = message || D.lastAssistantMessage();return m ? byName("button", name, m) : null;},- // Hover the last assistant message, which is what mounts its action bar. `autohide` on the- // bar unmounts it on every message that is not hovered, so a control read without this is- // read out of a tree it was never in.+ // Hover the last assistant message, which is what mounts its action bar: `autohide` unmounts+ // it on every message that is not hovered, so a control read without this is read out of a+ // tree it was never in.hoverLastAssistantMessage() {const m = D.lastAssistantMessage();if (m) {@@ -343,35 +264,18 @@return m;},- // WAIT for one of the action bar's controls, up to `waitMs`, instead of sampling for it once.- //- // WHY A WAIT AND NOT A SAMPLE, which is the whole point of this method. The assistant action- // bar is mounted with `hideWhenRunning` (thread.tsx), so while the thread is generating there- // is no Copy, no Delete and no More ANYWHERE in the tree -- not hidden, not disabled, absent.- // Every action that needs one of them is therefore scheduled after a `send_turn`, on the- // arithmetic that the follow-up turn drains in FOLLOW_UP_CHARS / the field cadence. That- // arithmetic is the NOMINAL drain: it assumes the pacer's cadence is the binding constraint,- // and at the 100K rung it is not -- the renderer is, so the reply arrives about 25% later- // than the cadence says, and the slot can open with the last few chunks still in flight.- //- // Measured on the CI runner, from the payload of the run that failed the liveness gate: the- // `message_menu` window opened at 32,000ms, took ONE more SSE chunk inside itself, and the- // reply stopped growing 71 characters later, inside the same window. The reply settled about- // a third of a second after the instant the action sampled the DOM. A single sample turns- // that third of a second into `NOT RUN -- no More button on the last assistant message`,- // which reads like a missing control and is really a clock being read too early.- //- // So the control is waited for. The wait is bounded, it is reported, and it happens BEFORE- // any measurement clock starts, so a cell that had to wait is not a cell whose open latency- // was inflated by the waiting. `fixture/selftest/test_studiobench_rung_plan.py` already- // asserts this behaviour in prose -- "a slot may legitimately open a little before the- // follow-up finishes and wait inside its own budget" -- and this is the code that makes the- // sentence true.- //- // Polling per paint, not per millisecond, and scoped: `actionButton` searches inside the last- // assistant message, so this is O(that message) rather than O(the thread), and it stops the- // instant the control appears (0 iterations on every cell where the reply has settled, which- // is the normal case).+ // WAIT for one of the action bar's controls, up to `waitMs`, instead of sampling once. The bar+ // is mounted with `hideWhenRunning`, so while the thread generates there is no Copy, Delete or+ // More anywhere: not hidden, absent. Every action needing one is scheduled after a `send_turn`+ // on the NOMINAL drain arithmetic, which assumes the pacer is the binding constraint; at 100K+ // the renderer is, so the reply arrives about 25% later. On the CI run that failed the+ // liveness gate the `message_menu` window opened at 32,000ms, took one more SSE chunk inside+ // itself, and the reply stopped growing 71 characters later inside the same window: a single+ // sample turns that third of a second into `NOT RUN -- no More button`. The wait is bounded,+ // reported, and happens BEFORE any measurement clock starts. Polling per paint and scoped to+ // the last assistant message, so it is O(that message) and stops as soon as the control+ // appears.+ // The drain arithmetic is FOLLOW_UP_CHARS over the field cadence; see test_studiobench_rung_plan.py.async waitForActionButton(name, waitMs, everyMs) {const started = performance.now();const budget = Math.max(0, Number(waitMs) || 0);@@ -383,17 +287,16 @@let el = D.actionButton(name);while (!el && performance.now() - started < budget) {await nextPaint();- // Re-hovered every pass: the bar unmounts again whenever the message re-renders, which- // during a stream is on every chunk.+ // Re-hovered every pass: the bar unmounts again whenever the message re-renders, which during+ // a stream is on every chunk.D.hoverLastAssistantMessage();el = D.actionButton(name);}return {el,waitedMs: Math.round((performance.now() - started) * 10) / 10,- // Recorded whether the control was found or not. A miss with `running: true` is the- // reply not having settled; a miss with `running: false` is a control that is genuinely- // not there, and those are different bugs.+ // Recorded whether the control was found or not: a miss with `running: true` is the reply not+ // having settled, a miss with `running: false` is a control that is genuinely not there.running: D.isRunning(),};},@@ -405,7 +308,6 @@return menu ? qa(".aui-action-bar-more-item", menu).length : 0;},- // ── settings ─────────────────────────────────────────────────settingsTrigger() {return q('button[aria-label="Settings"]');},@@ -421,7 +323,6 @@return q('[data-testid="settings-tab-' + id + '"]');},- // ── model picker ─────────────────────────────────────────────modelTrigger() {return q("button.unsloth-model-selector-trigger");},@@ -439,7 +340,6 @@return t ? (t.textContent || "").trim() : null;},- // ── composer plus menu / attachments ─────────────────────────plusButton() {return q('button[aria-label="Tools and attachments"]') || q('[data-tour="chat-plus-menu"]');},@@ -451,7 +351,6 @@);},- // ── sidebar / threads ────────────────────────────────────────threadRows() {return qa('[data-testid="recent-thread"]');},@@ -462,12 +361,10 @@return q('button[aria-label="New chat"].sidebar-header-action') || q('button[aria-label="New chat"]');},- // ── code blocks ──────────────────────────────────────────────codeCopyButtons() {return qa('button[title="Copy code"]');},- // ── census ───────────────────────────────────────────────────counts() {const started = performance.now();const out = {@@ -477,19 +374,16 @@reasoning_panes: qa('[data-slot="reasoning-root"]').length,reasoning_open: qa('[data-slot="reasoning-root"][data-state="open"]').length,code_blocks: qa("pre").length,- // Shiki spans. THE span density check: the field capture ran 90,262 characters against- // 16,186 spans, i.e. 5.6 characters per span, and a fixture that does not reproduce that- // is not measuring the same highlighter load per character.+ // Shiki spans. THE span density check: the field capture ran 90,262 characters against 16,186+ // spans, 5.6 characters per span, and a fixture that does not reproduce that is not measuring+ // the same highlighter load per character.highlight_spans: qa("pre span").length,- // WHERE the spans live, not just how many. A collapsed reasoning pane UNMOUNTS its- // children, so a thread with the same text can carry wildly different DOM depending on- // how that text got there. Without this split, "seeded has 20% fewer spans" is a number- // with three possible explanations and no way to choose between them.- // Tool components. TWO markers, because there are two renderers: a known tool gets a- // `tool-group-root`, and anything else gets the generic `tool-fallback-root` ("Used- // tool"). Counting only the first read ZERO on a thread that visibly contained tool- // blocks -- the second wrong selector in a row for this one component, and both times the- // reading was a confident zero rather than an error.+ // WHERE the spans live, not just how many: a collapsed reasoning pane UNMOUNTS its children,+ // so a thread with the same text can carry wildly different DOM. Without the split, "seeded+ // has 20% fewer spans" has three possible explanations.+ // Tool components, TWO markers because there are two renderers: a known tool gets a+ // `tool-group-root` and anything else a generic `tool-fallback-root`. Counting only the first+ // read ZERO on a thread that visibly contained tool blocks.tool_groups: qa('[data-slot="tool-group-root"]').length+ qa('[data-slot="tool-fallback-root"]').length,tool_groups_open: qa('[data-slot="tool-group-content"]').length@@ -499,29 +393,19 @@content_spans:qa("pre span").length - qa('[data-slot="reasoning-root"] pre span').length,content_code_blocks: qa("pre").length - qa('[data-slot="reasoning-root"] pre').length,- // Carried in the census so the peak occupancy and the character count come from the- // SAME reading. Two separate reads, taken either side of a destructive action, disagree.+ // Carried in the census so the peak occupancy and the character count come from the SAME+ // reading; two reads either side of a destructive action disagree.assistant_chars: D.assistantChars(),viewport_scroll_height: (D.viewport() || {}).scrollHeight || null,viewport_client_height: (D.viewport() || {}).clientHeight || null,- // ── DOES THE THREAD STILL FOLLOW THE STREAM? ──────────────- //- // Three readings, taken with every census, so the answer exists for every window of the- // film rather than being reconstructed afterwards from timings.- //- // This is here because of a specific way a virtualized arm can produce a beautiful and- // meaningless frame rate. If the thread stops following, the message being streamed- // drifts out of the viewport; a windowed list then UNMOUNTS it, and the streaming cost- // collapses to almost nothing. That is not a measurement of virtualization, it is a- // measurement of not rendering the thing being measured, and it flatters the arm in- // exactly the direction this campaign keeps having to catch. An fps number from such a- // run must never be readable without this beside it.- //- // `app_at_bottom` is the app's OWN state, not arithmetic: thread.tsx renders the- // scroll-to-bottom control permanently and hides it with `invisible` exactly when- // use-intent-aware-autoscroll considers itself at the bottom. `distance_from_bottom` is- // kept alongside because a virtualizer working from estimated row heights can sit a few- // pixels off while the app is quite correctly pinned.+ // DOES THE THREAD STILL FOLLOW THE STREAM? Three readings taken with every census, so the+ // answer exists for every window rather than being reconstructed from timings. If the thread+ // stops following, the streamed message drifts out of the viewport, a windowed list UNMOUNTS+ // it, and the streaming cost collapses to almost nothing: a beautiful frame rate that measures+ // not rendering the thing being measured. `app_at_bottom` is the app's OWN state (thread.tsx+ // hides the scroll-to-bottom control with `invisible` exactly when use-intent-aware-autoscroll+ // considers itself at the bottom); `distance_from_bottom` sits alongside because a virtualizer+ // working from estimated row heights can be a few pixels off while correctly pinned.viewport_scroll_top: (D.viewport() || {}).scrollTop || null,distance_from_bottom: D.distanceFromBottom(),app_at_bottom: D.appSaysAtBottom(),@@ -530,8 +414,8 @@return out;},- // Characters of assistant text currently in the DOM. Used for the seeded-vs-streamed- // equivalence check and for chars-per-span.+ // Characters of assistant text currently in the DOM, for the seeded-vs-streamed equivalence+ // check and chars-per-span.assistantChars() {let n = 0;for (const m of qa('[data-role="assistant"]')) n += (m.textContent || "").length;@@ -541,38 +425,21 @@window.__sb.dom = D;- // ── the follow-the-stream sampler ──────────────────────────────────────────────────────────- //- // WHY THIS IS NOT DONE FROM THE DRIVER. The question is "while a reply was streaming, was the- // thread pinned to the bottom", and the stream runs during the gap windows -- which are- // measured windows whose whole purpose is to observe the page doing nothing but stream. A- // `page.evaluate` per sample would put a CDP round trip and a forced style read inside exactly- // those windows, four times a second, which is the same class of mistake as the census that- // used to run inside the action windows.- //- // So it samples in the page and is READ ONCE PER CELL, outside every window. A 250ms timer,- // four ticks a second: frames.js documents a 1ms timer as costing nothing at ~150 ticks a- // second, so this is two orders of magnitude below that. It does no layout it has not already- // caused -- `classList.contains` is free, and `distanceFromBottom` is only read when a run is- // in progress.- //- // WHAT IT IS FOR. A thread that stops following unmounts the message being streamed, and the- // streaming cost then collapses because the renderer is no longer rendering the thing under- // measurement. That produces a superb frame rate and means nothing. `pinned_fraction` is the- // reading that makes an fps number from a windowed arm readable at all.- // THE COUNTERS SURVIVE A NAVIGATION, via sessionStorage.- //- // They did not, and the symptom was a confident "NOT MEASURED" on the arm that behaved best.- // The film ends with `thread_reopen`, which falls back to `page.goto` when the New chat control- // is covered; a full document navigation destroys the JS context and re-runs the init scripts,- // so a sampler that lived only in memory came back at zero and the whole cell's streaming- // phase was reported as never sampled. The treatment arm, whose thread_reopen did not run at- // all, kept its counters and looked like the only arm with data.- //- // sessionStorage is per-origin and per-tab and outlives a same-origin navigation, which is- // exactly the lifetime wanted: one cell, one page, however many documents it passes through.- // Saved on pagehide rather than on every tick, so the cost is one serialisation per navigation- // rather than four a second.+ // WHY THIS IS NOT DONE FROM THE DRIVER: the stream runs during the gap windows, whose whole+ // purpose is to observe the page doing nothing but stream, and a `page.evaluate` per sample+ // would put a CDP round trip and a forced style read inside them four times a second. So it+ // samples in the page and is READ ONCE PER CELL, outside every window: a 250ms timer, two+ // orders of magnitude below the 1ms timer frames.js documents as free, doing no layout it has+ // not already caused. `pinned_fraction` is what makes an fps number from a windowed arm+ // readable at all.+ // That timer runs at ~150 ticks a second.+ // THE COUNTERS SURVIVE A NAVIGATION, via sessionStorage. They did not, and the symptom was a+ // confident "NOT MEASURED" on the arm that behaved best: the film ends with `thread_reopen`,+ // whose `page.goto` fallback destroys the JS context and re-runs the init scripts, so an+ // in-memory sampler came back at zero while the treatment arm, whose thread_reopen did not+ // run, kept its counters and looked like the only arm with data. sessionStorage is per-origin+ // and per-tab and outlives a same-origin navigation, exactly the lifetime wanted. Saved on+ // pagehide rather than per tick.const FOLLOW_KEY = "__sb_follow_v1";const restore = () => {try {@@ -593,74 +460,50 @@yanked_back_samples: 0,stream_samples: 0,reattachments: 0,- // Set once the thread is seen to fall behind while a run is in progress, and never cleared.- // A thread that drifts away and is later yanked back to the bottom has still failed the- // contract, and an end-of-cell reading would show it pinned.+ // Set once the thread is seen to fall behind while a run is in progress, and never cleared: a+ // thread that drifts away and is later yanked back has still failed the contract, and an+ // end-of-cell reading would show it pinned.ever_fell_behind: false,}, restore() || {});window.addEventListener("pagehide", () => {try {window.sessionStorage.setItem(FOLLOW_KEY, JSON.stringify(F));} catch (e) {- // A full sessionStorage is not worth losing the page over; the reading degrades to- // "not measured", which is the honest outcome and is already handled.+ // A full sessionStorage is not worth losing the page over; the reading degrades to "not+ // measured", which is already handled.}});const FOLLOW_TICK_MS = 250;- // How far from the bottom still counts as following. Generous on purpose: a virtualizer- // working from estimated row heights corrects them as real rows are measured, so it can sit a- // little short of the exact bottom while behaving perfectly. 64px is under two lines of text,- // so it cannot hide a thread that has stopped following.+ // How far from the bottom still counts as following. Generous on purpose, since a virtualizer+ // working from estimated heights can sit short of the exact bottom while behaving perfectly;+ // 64px is under two lines, so it cannot hide a thread that has stopped following.const FOLLOW_TOLERANCE_PX = 64;- // TWO PHASES, BECAUSE THE INTENT CONTRACT HAS TWO HALVES.- //- // The contract (plans/proud-wiggling-falcon.md): autoscroll follows a stream, AND a user who- // has scrolled up is never yanked down. Those are opposite requirements about the same- // scrollTop, so one number cannot score both -- and the first version of this sampler tried,- // producing 47% to 50% pinned on BOTH arms with an identical 6,615px worst drift. That figure- // is the film: `scroll_during_generation` drags the viewport thousands of pixels up, twice,- // while the reply is still arriving, and the app then CORRECTLY declines to drag it back. The- // sampler was recording the app honouring the second half of the contract and scoring it as a- // failure of the first.- //- // ATTACHED (before the harness scrolls anywhere): the thread must stay pinned as content- // arrives. This is "does it follow", and it is what the gate scores.- // DETACHED (after the harness has deliberately scrolled): the thread must NOT come back to the- // bottom on its own. Any sample that finds it pinned again while still streaming is the app- // yanking the user down, which is the other half of the contract and is recorded as its own- // finding rather than mixed into the fraction.+ // TWO PHASES, BECAUSE THE INTENT CONTRACT HAS TWO HALVES (plans/proud-wiggling-falcon.md):+ // autoscroll follows a stream, AND a user who has scrolled up is never yanked down. One number+ // cannot score both: the first version reported 47-50% pinned on both arms with an identical+ // 6,615px worst drift, which is the film, since `scroll_during_generation` drags the viewport+ // thousands of pixels up twice and the app correctly declines to drag it back, so the sampler+ // scored the second half of the contract as a failure of the first. ATTACHED (before the+ // harness scrolls) is "does it follow" and is what the gate scores; DETACHED (after a+ // deliberate scroll) is "it must not come back on its own", recorded as its own finding.let detached = false;let suspended = 0;- // WHICH RUN THE USER SCROLLED AWAY FROM, and it is the difference between a yank and a return.- //- // `resume()` below clears `detached` only when the gesture itself ended at the bottom. On the- // real films it never does: `SCROLL_JS` jumps to the bottom and then steps 14 x 420px away from- // it, so at any rung whose thread is taller than 5,880px the gesture ends thousands of pixels- // up and `detached` is latched for the rest of the cell. The film then starts TWO MORE RUNS of- // its own -- `stop_generation` and `send_turn` both submit a turn -- and the app pins to the- // bottom for them, which session.py already documents as intended behaviour rather than a- // violation. Every one of those samples was landing in the detached branch, counted as a yank,- // and excluded from `attached_fraction_of_stream`.- //- // Measured, at head, across every 100K payload in outputs/: attached_fraction 0.07 to 0.15 with- // reattachments 0, on the BASE arm as well as the treatment and on pure null controls, so the- // `follows_the_stream` gate failed every 100K cell of every run including two copies of the- // shipped build. It passed only on the 1K smoke film, where the thread is short enough that the- // gesture's reversal happens to land back at the bottom -- i.e. the verdict was a property of- // the thread's height, not of the app.- //- // So a run the user STARTED is a fresh expression of intent to be at the end, exactly as- // returning to the bottom is. Re-attachment is granted only when a run that began AFTER the- // gesture is also OBSERVED at the bottom: an app that declines to pin stays detached and is- // scored as before, and a pin during the SAME run the user scrolled away from is still a yank.+ // WHICH RUN THE USER SCROLLED AWAY FROM, the difference between a yank and a return.+ // `resume()` clears `detached` only when the gesture ended at the bottom, which on the real+ // films it never does: `SCROLL_JS` steps 14 x 420px away, so above 5,880px `detached` latches+ // for the rest of the cell, and the film then starts two more runs whose intended pinning was+ // counted as a yank. Measured at head across every 100K payload: attached_fraction 0.07 to+ // 0.15 with zero reattachments, on the BASE arm and on pure null controls, so+ // `follows_the_stream` failed every 100K cell of every run and passed only on the 1K film. So+ // a run the user STARTED is a fresh expression of intent to be at the end: re-attachment is+ // granted only when a run that began AFTER the gesture is also observed at the bottom.let runSeq = 0;let wasRunning = false;let detachedAtRun = 0;setInterval(() => {F.samples += 1;// Read BEFORE the suspended early-return, or a run that starts and ends inside a deliberate- // gesture is never seen and the run after it is mistaken for the one the user scrolled away- // from.+ // gesture is never seen and the run after it is mistaken for the one scrolled away from.const running = D.isRunning();if (!running) wasRunning = false;else if (!wasRunning) { wasRunning = true; runSeq += 1; }@@ -668,7 +511,7 @@if (!running) return;const app = D.appSaysAtBottom();const distance = D.distanceFromBottom();- // "At the bottom" by the app's own answer, with the geometry standing in only when the build+ // "At the bottom" by the app's own answer, with geometry standing in only when the build// renders no jump control. `app === false` is the app saying no and is never overridden.const atBottom =app === true || (app === null && distance !== null && distance <= FOLLOW_TOLERANCE_PX);@@ -705,50 +548,36 @@window.__sb.follow = {// Called by any action that moves the viewport on purpose. Nested-safe, because more than one// scope may legitimately be open at once.- // Called by any action that moves the viewport on purpose. Nested-safe. The FIRST suspend- // also latches `detached`: from that moment the user has expressed an intent to be somewhere- // other than the bottom, and everything after it is scored against the second half of the- // contract instead of the first.- // `detachedAtRun` is stamped on every suspend rather than only the first, so a second gesture- // during a later run detaches from THAT run: without it, scrolling away during run 3 would be- // compared against run 1 and the very next at-bottom sample would re-attach.+ // The FIRST suspend also latches `detached`: from then the user has expressed an intent to be+ // somewhere other than the bottom, and everything after is scored against the second half of+ // the contract. `detachedAtRun` is stamped on every suspend rather than only the first, so a+ // second gesture during a later run detaches from THAT run.suspend() { suspended += 1; detached = true; detachedAtRun = runSeq; },resume() {suspended = Math.max(0, suspended - 1);- // RE-ATTACH IF THE GESTURE LEFT US AT THE END, and this is not a nicety.- //- // `detached` used to latch on the first suspend and never clear, so from the harness's- // first deliberate scroll onwards every remaining sample went to the detached branch and- // `running_samples` stopped growing. In the shipped film that scroll happens 1.5s into an- // 18s opening stream and is followed by two more streamed turns, so the follow verdict was- // computed from the first ~3s and covered 13% of the streaming time -- while reading, and- // being reported as, "the thread follows the stream". Measured: running_samples 11,- // detached_samples 72.- //- // The contract is about INTENT, and intent is re-expressed by coming back: a user who- // scrolls up is detached until they return to the end, at which point they are following- // again. That is the same re-attachment Unsloth's own intent-aware autoscroll implements.- // Only checked here, on the way out of a deliberate gesture, so the app silently pulling- // the viewport down on its own is still scored as a yank rather than laundered into a- // re-attachment.+ // RE-ATTACH IF THE GESTURE LEFT US AT THE END, and this is not a nicety. `detached` used to+ // latch on the first suspend and never clear, so from the harness's first deliberate scroll+ // every sample went to the detached branch: in the shipped film that scroll is 1.5s into an 18s+ // opening stream, so the verdict came from the first ~3s and covered 13% of the streaming time+ // while reporting "the thread follows the stream" (running_samples 11, detached_samples 72).+ // The contract is about INTENT, and intent is re-expressed by coming back, exactly as+ // Unsloth's own intent-aware autoscroll implements. Only checked on the way out of a+ // deliberate gesture, so the app pulling the viewport down on its own is still a yank.if (suspended === 0 && detached) {const app = D.appSaysAtBottom();const distance = D.distanceFromBottom();- // EITHER answer is enough, and the geometry is not merely a fallback for a build with no- // jump control. The control's `invisible` class is updated from a scroll LISTENER, and- // scroll events are dispatched asynchronously, so a gesture that has just returned the- // viewport to the end can reach this line while the class still says otherwise. Reading- // the class alone loses the re-attachment to a race and quietly restores the old- // never-reattach behaviour on exactly the fast gestures the harness performs.- // `distanceFromBottom()` is computed from scrollTop and cannot be stale.+ // EITHER answer is enough, and the geometry is not merely a fallback: the control's+ // `invisible` class is updated from a scroll LISTENER and scroll events are dispatched+ // asynchronously, so a gesture that has just returned the viewport to the end can reach this+ // line while the class still says otherwise. `distanceFromBottom()` cannot be stale.+ // `distanceFromBottom()` is computed from scrollTop.if (app === true || (distance !== null && distance <= FOLLOW_TOLERANCE_PX)) {detached = false;F.reattachments += 1;} else {- // Still away from the end. Re-stamp against the run in flight NOW, not the one that was- // in flight when the gesture began: a gesture long enough to span a run boundary would- // otherwise look like a scroll away from the previous run and be re-attached by the very- // first sample of the current one.+ // Still away from the end. Re-stamp against the run in flight NOW, not the one in flight when+ // the gesture began: a gesture spanning a run boundary would otherwise be re-attached by the+ // very first sample of the current run.detachedAtRun = runSeq;}}@@ -766,16 +595,14 @@yanked_back_samples: F.yanked_back_samples,// The second half of the intent contract, as its own verdict.yanked_after_scroll: F.yanked_back_samples > 0,- // null, not 1.0, when nothing was ever sampled mid-run. A cell whose stream finished- // before the first tick has not demonstrated that the thread follows; it has- // demonstrated nothing, and 1.0 would read as a pass.+ // null, not 1.0, when nothing was sampled mid-run: a cell whose stream finished before the+ // first tick has demonstrated nothing, and 1.0 would read as a pass.pinned_fraction: measured > 0 ? F.running_pinned / measured : null,pinned_fraction_reason:measured > 0 ? null : "no sample was taken while a reply was streaming",- // HOW MUCH OF THE STREAM THIS VERDICT ACTUALLY COVERS. `pinned_fraction` is computed- // over the attached phases only, so without this it can read 1.0 on a cell where the- // thread was attached for three seconds of an eighteen-second stream. Reported beside it- // so the one cannot be quoted without the other.+ // HOW MUCH OF THE STREAM THIS VERDICT COVERS: `pinned_fraction` is computed over the attached+ // phases only, so without this it can read 1.0 on a cell attached for three seconds of an+ // eighteen-second stream.stream_samples: F.stream_samples,attached_fraction_of_stream:F.stream_samples > 0 ? F.running_samples / F.stream_samples : null,diff --git a/tests/studio/studiobench/scene/parity.js b/tests/studio/studiobench/scene/parity.jsindex b1ce91a7b7d..486ead53cf7 100644--- a/tests/studio/studiobench/scene/parity.js+++ b/tests/studio/studiobench/scene/parity.js@@ -1,114 +1,38 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//// UI PARITY: a structural signature of the rendered thread, taken at the close of every action// window on BOTH arms of an A/B.-//-// WHY THIS AND NOT SCREENSHOTS. The screenshot parity pair (scripts PR 217) is the right artefact-// for a human reader of a pull request: it shows the surface, side by side, and a person checks it-// against `expect`. What it cannot do is breadth. Each pair costs two isolated installs, shows one-// surface, and needs somebody to open the composite. Eighteen PRs across eighteen action surfaces-// is 324 composites and nobody will look at them.-//-// This runs inside a comparison the tool is ALREADY doing. studiobench drives both arms through-// the same eighteen scripted actions against a byte-identical seeded thread inside one session, so-// the two DOMs are already meant to agree. Digesting them costs one `evaluate` per slot and turns-// "the UI is unchanged" from a claim into a per-action assertion. It is the cheap wide net; the-// screenshot pair stays the deep, human-readable one for the surfaces that matter most.-//-// ── WHAT THIS DIGEST CAN AND CANNOT SEE ──────────────────────────────────────────────────────-//-// Read this before quoting a passing parity check at anybody. The digest is a serialisation of the-// DOM, so its sensitivity is exactly the DOM and nothing else.-//-// IT SEES: tag structure and nesting, element order, every attribute name, every attribute value-// that is not on the volatile list below (so `class`, `data-slot`, `data-state`, `data-role`,-// `aria-hidden`, `hidden`, `disabled`, `role`, `title`, `alt`, `placeholder`), and all text content-// after time normalisation. Added elements, removed elements, reordered siblings, retitled buttons-// and flipped states all move the digest.-//-// IT DOES NOT SEE, and a green parity check asserts NOTHING about any of these:-// - CSS coming from a stylesheet. A class name is compared as a STRING. If `.aui-md-p` changes-// its margin in a `.css` file, every class list is identical and the digest is identical while-// the page reflows. Only the bounded computed-style probe below sees any of this, and it sees-// three properties on at most a few dozen elements.-// - Computed layout: geometry, sizes, positions, scroll extents, whether anything overflows or-// is clipped, whether two elements overlap.-// - Colour, typography, borders, shadows, opacity, transforms, animation.-// - Raster content: images, canvas, video, SVG paint. An `<img>` is compared by its attributes,-// and its `src` is normalised (see below), so a DIFFERENT PICTURE at the same normalised URL-// is invisible.-// - Anything outside the thread root and the overlay selectors: the sidebar, the header, toasts.-// - Anything the DOM does not carry at all: focus ring appearance, caret position, text-// selection, scroll position (deliberately normalised), `<input>` live values typed by a user-// (the DOM attribute is not the property), shadow DOM, and any state kept only in JS.-//-// THAT LIST HAS NOW BEEN MEASURED RATHER THAN JUST ASSERTED, and the result is worse than the-// wording above suggests. A concurrent campaign measuring the sidebar drag ran this digest against-// a real, visible change and got 0 of 34 differing pairs -- and its null control also returned 0 of-// 34, so the instrument was not discriminating in either direction on that change. Three-// purpose-built captures (sidebar-inclusive structure, sidebar inline style, custom-property reach)-// each found the same change 34 of 34 with the null at zero. The two blind spots that mattered were-// the two named above: the sidebar is outside the root, and computed layout is never read.-//-// So a PARITY OK verdict from this file means NO THREAD-STRUCTURE CHANGE WAS DETECTED. It does not-// mean the UI is unchanged, and the two are far enough apart that the second should never be-// written down on the strength of the first.-//-// Extending this into a real visual diff is the screenshot pair's job, not this one's. The point-// of the digest is breadth at near-zero cost per action, and its limits are stated here so nobody-// reads "18 actions, 0 differences" as "the UI is pixel-identical". It is not that claim.-//-// ── AND A NOTE ON ANY SCAN THAT CAN RETURN ZERO ──────────────────────────────────────────────-//-// `styleProbe` below walks a hand-written selector list. If Unsloth renames a class the list goes-// quiet, matches nothing, and its digest becomes the hash of an empty string -- identical on both-// arms, reported as a MATCH. A scan of nothing must never be reported as agreement, so-// `compare_styles` refuses a zero-element probe instead of matching it, and `elements` travels-// with every reading so the count can be checked rather than assumed.-//-// The same team hit the general form of this and it is worth recording: their CSSOM scan returned-// a clean zero because CSS nesting gives every `CSSStyleRule` a truthy but empty `cssRules`, so-// code that treats a truthy `cssRules` as "this is a grouping rule, recurse" silently skips every-// declaration in the document. They caught it only because they had gated on a positive control.-// Nothing in this file walks the CSSOM today, so that specific bug is not present here -- but any-// scan added later that can legitimately return zero needs a positive control, and a zero without-// one should not be believed.-//-// ── WHAT IS NORMALISED AWAY, and why each one has to be ──────────────────────────────────────-//-// A digest that trips on things that legally differ between two runs of the SAME build is worse-// than no digest: it trains you to ignore it. Every entry here was OBSERVED to differ in a-// base-vs-base null control; nothing is normalised on suspicion.-//-// generated ids Radix mints `radix-:r1a:` per mount, and aria-controls/labelledby point at-// them. They differ between two runs of one build.-// rendered durations the action bar prints things like `295ms`. PR 217 hit exactly this on-// unslothai/unsloth#9054: a 295 vs 310 difference that is wall clock, not-// content. Any number immediately followed by a time unit is collapsed.-// relative times "2 minutes ago" moves on its own.-// scroll state `--aui-scroll-stabilizer` and transform offsets encode where the thread-// happens to be scrolled, which the film varies deliberately.-// record identifiers thread ids, message ids and attachment ids are minted by the BACKEND when-// the fixture seeds the thread. The two arms of an A/B are two separate-// Unsloth installs with two separate databases, so these can never agree and-// carry no information about the frontend.-// absolute URLs `src`/`href` carry the arm's own origin, and the two arms are on two-// ports by construction. `http://127.0.0.1:5830/x` and `...:5831/x` are the-// same asset. Origins are stripped; the PATH is kept and still compared.-// blob and data URLs an uploaded image is addressed by a per-mount `blob:` UUID.-//-// WHAT IS DELIBERATELY KEPT. Tag structure, `data-slot`, `data-state`, `data-role`, `aria-hidden`,-// `hidden`, `disabled`, the class list, and all text content. `data-state` in particular: whether-// a reasoning pane reads open or closed IS the UI, and a perf change that quietly flips it is the-// exact defect this is meant to catch.+// Cheap breadth, unlike the screenshot parity pair: one `evaluate` per slot inside a comparison+// the tool already runs, versus 324 composites nobody will open.+// IT SEES: tag structure and nesting, element order, attribute names, non-volatile attribute+// values (class, data-slot, data-state, data-role, aria-hidden, hidden, disabled, role, title,+// alt, placeholder) and all text after time normalisation.+// IT DOES NOT SEE: stylesheet CSS (a class name compares as a string), computed layout and+// geometry, colour, typography and animation, raster content behind a normalised src, anything+// outside the thread root, and anything the DOM does not carry (focus, caret, selection, scroll+// position, live input values, shadow DOM, JS state).+// Measured, not asserted: against a real visible sidebar-drag change this digest found 0 of 34+// differing pairs with its null control also at 0, while three purpose-built captures found it+// 34 of 34. The blind spots that mattered were the sidebar being outside the root and computed+// layout never being read.+// So a PARITY OK verdict means NO THREAD-STRUCTURE CHANGE WAS DETECTED. It does not mean the UI is unchanged.+// Any scan that can return zero needs a positive control: `styleProbe` walks a hand-written+// selector list, so a renamed class makes it hash the empty string identically on both arms.+// `compare_styles` refuses a zero-element probe, and `elements` travels with every reading.+// Everything normalised below was OBSERVED to differ in a base-vs-base null control, nothing on+// suspicion: Radix generated ids, rendered durations (unslothai/unsloth#9054), relative times,+// scroll state, backend-minted record ids, absolute URLs (two ports; the path is kept), and+// blob/data URLs.+// A 295 vs 310 difference that is wall clock, not a build change.+// DELIBERATELY KEPT: tag structure, data-slot, data-state, data-role, aria-hidden, hidden,+// disabled, the class list and all text. A perf change that quietly flips a reasoning pane's+// data-state is the exact defect this catches.(() => {if (window.__sb && window.__sb.parity) return;window.__sb = window.__sb || {};- // Attributes whose VALUE is volatile between two runs of the same build. The attribute's- // presence is still recorded; only the value is dropped.+ // Attributes whose VALUE is volatile between two runs of the same build; the presence is still recorded.const VOLATILE_ATTRS = new Set(["id", "for", "aria-controls", "aria-labelledby", "aria-describedby", "aria-activedescendant","style", "data-radix-scroll-area-viewport", "data-testid-instance",@@ -120,14 +44,12 @@// Attributes that carry no rendered meaning at all.const IGNORED_ATTRS = new Set(["data-react-checksum", "data-reactroot"]);- // VIRTUALIZATION BOOKKEEPING. Dropped -- name and value -- from the VISIBLE-REGION digest only,- // and passed in by that caller rather than applied here, because the two comparisons want- // different things from these two attributes. The whole argument is at the call site, in- // `parityVisible.capture()`.+ // VIRTUALIZATION BOOKKEEPING, dropped name and value from the VISIBLE-REGION digest only and+ // passed in by that caller; the whole argument is at `parityVisible.capture()`.const VIRTUALIZATION_ATTRS = new Set(["aria-posinset", "aria-setsize"]);- // Attributes whose value is a URL. The origin is stripped and the path kept, so a genuinely- // different asset still moves the digest while the arm's own port does not.+ // URL-valued attributes: the origin is stripped and the path kept, so a genuinely different asset+ // still moves the digest while the arm's own port does not.const URL_ATTRS = new Set(["src", "href", "srcset", "action", "poster", "data-src", "formaction"]);// A UUID, or a long hex run. Both are how this app spells "an id the server just made up".@@ -155,8 +77,7 @@.replace(/\s+/g, " ").trim();- // FNV-1a, 32 bit, expressed as unsigned hex. Not cryptographic: this is a change detector, and- // the pair being compared is two runs of one page rather than an adversary.+ // FNV-1a, 32 bit, unsigned hex. Not cryptographic: a change detector over two runs of one page.const hash = (str) => {let h = 0x811c9dc5;for (let i = 0; i < str.length; i++) {@@ -172,29 +93,19 @@return normText(raw);};- // `dropAttrs`, when given, is a Set of attribute names this digest does not compare AT ALL --- // neither presence nor value, unlike VOLATILE_ATTRS which keeps the presence. Optional and off- // by default: every existing caller (the whole-thread digest, the per-message rows, the overlays- // and the node-driven unit tests) passes nothing and gets exactly what it got before.- //- // `elide`, when given, is a Set of ELEMENTS whose subtree this digest does not serialise. It is- // not a way of ignoring them: a marker carrying the tag and the element's `data-role` is written- // in their place, so the element's PRESENCE, its position among its siblings and its role are- // still compared and a message that vanished still moves the digest. Only the content inside it- // is withheld. The one caller is the streamed-message elision in `capture()`; the whole argument- // for it is stated there. Optional and off by default, so every existing caller passes nothing- // and gets byte-for-byte what it got before.+ // `dropAttrs` is a Set of attributes this digest does not compare AT ALL, unlike VOLATILE_ATTRS+ // which keeps the presence. `elide` is a Set of ELEMENTS whose subtree is not serialised: a marker+ // carrying the tag and `data-role` goes in their place, so presence, position and role still+ // compare and a vanished message still moves the digest. Both off by default.const signature = (root, dropAttrs, elide) => {if (!root) return "";const parts = [];const walk = (el, depth) => {- // A depth cap is a TRUNCATION, and a truncated signature that reads like a complete one is- // exactly the silent false negative this instrument exists to rule out. The marker is left- // in the signature so anything deeper is visible as "not walked" rather than as "absent".+ // A depth cap is a TRUNCATION, so the marker is left in the signature and anything deeper reads+ // as "not walked" rather than as absent.if (depth > 40) { parts.push("<!depth-cap>"); return; }if (elide && elide.has(el)) {- // Named, not silent. A reader of the raw signature sees exactly which element was held- // back and why, rather than a hole they have to infer from a length.+ // Named, not silent: a reader of the raw signature sees which element was held back and why.parts.push("<!in-flight " + el.tagName.toLowerCase() +" role=" + ((el.getAttribute && el.getAttribute("data-role")) || "?") + ">"@@ -217,21 +128,11 @@}}parts.push(">");- // ADJACENT TEXT NODES ARE JOINED BEFORE THEY ARE NORMALISED, and this is the difference- // between a normaliser that works and one that looks like it works.- //- // React renders `Thought for {n} seconds` as THREE text nodes -- "Thought for ", "3",- // " seconds" -- because the interpolation is its own child. Normalising each node on its- // own, `normText("3")` sees a bare digit with no unit after it, the rendered-duration rule- // cannot match, and a wall-clock number reaches the digest. Two runs of a BYTE-IDENTICAL- // build then disagree by one character, which is exactly what a null control here measured:- // `msg22(assistant):17334->17334c`, same length, `3` against `2`, hunted down to this line- // by `sweep/parity_null_control.py --hunt`. It was the single largest source of false- // alarms in that control, and every one of them read as "this pull request changed the UI".- //- // The run is joined RAW and normalised once, so what gets normalised is the text as the- // page renders it rather than the accident of where React split it. Element boundaries- // still break a run, so this cannot weld two separate labels into one string.+ // ADJACENT TEXT NODES ARE JOINED BEFORE THEY ARE NORMALISED. React renders `Thought for {n}+ // seconds` as three text nodes, so per-node normalisation sees a bare digit, the rendered-duration+ // rule cannot match, and a wall-clock number reaches the digest -- the largest single source of+ // false alarms in the null control. Joined raw and normalised once.+ // Found by `sweep/parity_null_control.py --hunt`.let run = "";const flush = () => {if (!run) return;@@ -254,19 +155,12 @@return parts.join("");};- // The bounded computed-style probe. THREE properties on at most `STYLE_CAP` elements, and its- // digest is reported SEPARATELY from the structural one rather than folded into it.- //- // Separate on purpose. `getComputedStyle` reads post-stylesheet, post-animation state, so it is- // the only part of this file that can see a `.css` change at all -- and for the same reason it- // is the part most likely to be caught mid-transition and differ between two runs of one build.- // Folding it into `digest` would put the whole instrument's credibility on its least stable- // reading. Kept apart, a structural mismatch stays a hard signal and a style mismatch is an- // advisory the caller can weigh.- //- // `display` and `visibility` because they are how CSS makes something disappear without touching- // the DOM. `pointer-events` because an overlay that stops or starts swallowing clicks is a real,- // shipped class of defect here and leaves no structural trace at all.+ // The bounded computed-style probe: three properties on at most `STYLE_CAP` elements, digested+ // SEPARATELY from the structural signature. `getComputedStyle` is the only part of this file that+ // can see a `.css` change, and the most likely to be caught mid-transition, so folding it in+ // would stake the instrument's credibility on its least stable reading. `display` and+ // `visibility` are how CSS hides something without touching the DOM; `pointer-events` because an+ // overlay that swallows clicks leaves no structural trace.const STYLE_CAP = 64;const STYLE_PROPS = ["display", "visibility", "pointer-events"];const STYLE_SELECTORS = [@@ -294,89 +188,50 @@}}const sig = parts.join("");- // `capped` travels with the reading. A style digest taken over the first 64 of 300 elements- // is a partial reading, and a partial reading that does not say so reads as a complete one.+ // `capped` travels with the reading: a style digest over the first 64 of 300 elements is partial,+ // and a partial reading that does not say so reads as a complete one.return { digest: hash(sig), chars: sig.length, elements: n, capped, props: STYLE_PROPS, sig };};const D = () => (window.__sb.dom || {});- // ── VISIBLE-REGION PARITY ────────────────────────────────────────────────────────────────────- //- // THE POLICY THIS SERVES. All changes must preserve UI and UX idempotency, with three- // exemptions: a difference may be accepted deliberately when performance improves dramatically;- // a difference that exists only OFF SCREEN is fine by definition, because rendering only what is- // visible is an accepted technique rather than a parity violation; and a select-all need not- // select all, PROVIDED the copy stays complete. Nothing in this file can see the third -- the- // clipboard is scored behaviourally, in analysis/behaviour.py -- so a selection that shrank is- // not a finding here.- //- // The whole-document digest above cannot express the second exemption. It compares everything- // that is in the DOM, so ANY deferred off-screen work fails it by construction -- virtualization,- // deferred syntax highlighting, content-visibility, lazy images. Refusing such a pair as- // NOT_APPLICABLE was the right instinct and it withholds a verdict. This supplies the verdict.- //- // THE CLAIM IT SUPPORTS, exactly: every message that was visible in the viewport at any point- // during the action is present on both arms and identical between them, and every difference- // lies outside the viewport.- //- // ── TWO BOUNDARY DECISIONS, MADE EXPLICITLY ──────────────────────────────────────────────────- //- // These are where a visible-region check goes wrong quietly, so they are decided here in the- // open rather than left implicit in whatever the code happens to do.- //- // 1. PARTIAL INTERSECTION COUNTS AS VISIBLE, AND THE ELEMENT IS DIGESTED IN FULL.- // A message one pixel into the viewport is visible to the user. The alternative -- digesting- // only the part inside the viewport -- is not definable on a DOM subtree without reading- // geometry per node, and reading geometry is the one thing this must not do (see below). So- // the whole element is compared. The error this admits is a FALSE ALARM: a difference in the- // off-screen tail of a partly-visible message is reported as a visible difference. The error- // it refuses to admit is a false pass. Given a parity gate, that is the right way round.- //- // 2. ANYTHING VISIBLE AT ANY POINT DURING THE ACTION IS COMPARED, not just at the end.- // An action that scrolls makes messages visible and then hides them again; a single sample at- // the close of the window would compare the DOM the scroll happened to land on and silently- // ignore everything the user actually saw on the way. So the observer is installed BEFORE the- // window opens and the compared set is the UNION of everything that ever intersected. The- // per-message digest is still the one taken at the close, which is a real limitation and is- // named as such in `now_visible` versus `ever_visible`: a message that was visible mid-action- // and has since changed is compared in its final state.- //- // ── WHY INTERSECTIONOBSERVER AND NOT GEOMETRY ────────────────────────────────────────────────- //- // `getBoundingClientRect()` / `getClientRects()` on content inside a `content-visibility` locked- // subtree makes Chromium render that subtree in order to answer, so a geometry-based visibility- // probe unlocks exactly what it came to observe and then reports that nothing was skipped. That- // is measured, not theoretical: one session reported 0 off-screen unrendered roots while the- // event counter recorded 22 simultaneously in the skipped state. See the content-visibility trap- // section of CONTRIBUTING-perf.md.- //- // IntersectionObserver is the correct instrument and not merely a safe one: it is the same- // mechanism Blink's own relevance machinery uses to decide whether a `content-visibility: auto`- // subtree is skipped, so observing with it neither forces rendering nor perturbs the decision.- // NOTHING in this section may call a geometry method on a candidate element.+ // THE POLICY: changes must preserve UI and UX idempotency, with three exemptions -- a dramatic+ // performance win, a difference that exists only OFF SCREEN, and a select-all that need not+ // select all provided the copy stays complete. The third is scored in analysis/behaviour.py.+ // The whole-document digest cannot express the off-screen exemption: it compares everything in+ // the DOM, so any deferred off-screen work fails it by construction. Refusing such a pair as+ // NOT_APPLICABLE withholds a verdict; this supplies one.+ // THE CLAIM: every message visible in the viewport at any point during the action is present on+ // both arms and identical between them, and every difference lies outside the viewport.+ // 1. PARTIAL INTERSECTION COUNTS AS VISIBLE and the element is digested IN FULL: digesting only+ // the visible part would need per-node geometry, which this must not read. The admitted error is+ // a FALSE ALARM, never a false pass.+ // 2. ANYTHING VISIBLE AT ANY POINT IS COMPARED, not just at the end: the observer is installed+ // BEFORE the window opens and the compared set is the UNION of everything that intersected. The+ // per-message digest is still the one taken at the close, named `now_visible` vs `ever_visible`.+ // GEOMETRY IS FORBIDDEN HERE: `getBoundingClientRect()` on content inside a `content-visibility`+ // locked subtree makes Chromium render it, so a geometry probe unlocks exactly what it came to+ // observe -- one session reported 0 off-screen unrendered roots while the counter recorded 22+ // skipped. IntersectionObserver is what Blink's own relevance machinery uses.+ // `getClientRects()` too.const VIS = {obs: null, mut: null, ever: new Set(), watching: false,- // Nodes already counted in `unplaced`, so a row offered and refused several times before it- // mounts does not inflate the diagnostic. NOT a "do not look at this again" set: placement is- // re-read on every offer, because a recycled row changes position. See `observeOne`.+ // Nodes already counted in `unplaced`, so a row refused several times before it mounts does not+ // inflate the diagnostic. NOT a "do not look at this again" set: placement is re-read on every+ // offer, because a recycled row changes position.unplacedSeen: new WeakSet(),- // Rows that were observed and could NOT be placed in the thread at all: no `aria-posinset`- // and not among the messages the DOM currently holds. Such a row is stamped with no ordinal,- // so it is silently absent from `ever_visible`, and a silence that cannot be counted is the- // failure this whole file is written against. Reported with the capture.+ // Rows observed that could NOT be placed in the thread at all: no `aria-posinset` and not among+ // the messages the DOM holds. Such a row is stamped with no ordinal and so is silently absent+ // from `ever_visible`, which is why it is reported with the capture.unplaced: 0,// The batch-scoped position index. See `positionIndex`.index: null,};const ordinalOf = (el, position) => {- // The message's position in the THREAD, not in the mounted list. This is what makes a windowed- // arm comparable with a fully mounted one at all: mounted index 0 is message 10 on one arm and- // message 1 on the other, so a per-index comparison compares different messages and reports- // every row as changed. `aria-posinset` is the windowed arm's own claim about position; a- // fully mounted thread does not publish it and does not need to, because there the DOM order- // IS the thread order, which is what `position` carries.+ // The message's position in the THREAD, not in the mounted list: mounted index 0 is message 10 on+ // one arm and message 1 on the other, so a per-index comparison reports every row as changed.+ // `aria-posinset` is the windowed arm's own claim; a fully mounted thread's DOM order IS it.const owner = el.closest ? el.closest("[aria-posinset]") : null;if (owner) {const n = Number(owner.getAttribute("aria-posinset"));@@ -385,41 +240,20 @@return position;};- // THE POSITION INDEX, AND WHY THE FALLBACK ORDINAL IS NOT A COUNTER.- //- // A row publishing no `aria-posinset` is stamped with its position among the thread's messages- // in the DOM AS IT STANDS. That has to be resolved when the row is OBSERVED rather than when an- // entry is delivered, which is the reason the ordinal is stamped onto the node in the first- // place: by delivery time the row may have been unmounted and `closest()` would answer nothing.- //- // A LIFETIME COUNT OF OBSERVED NODES IS NOT THAT POSITION, and the gap between them was a- // measured failure rather than a tidiness point. `thread_reopen` makes a fully mounted arm- // remove and recreate every message row inside one document. Those rebuilt rows legitimately- // publish no `aria-posinset` -- only a windowed arm publishes one -- so a counter that had- // already seen the thread's N rows stamped the rebuilt ones N+1..2N, while the windowed arm on- // the other side of the A/B stamped its real 1..N. `compare_visible` then saw two disjoint- // visible sets and reported "the two arms put DIFFERENT MESSAGES on screen" for a rebuild that- // was identical, failing thread_reopen on every base-versus-windowed pair.- //- // THE COST, WHICH IS WHY A COUNTER WAS TEMPTING. `observeAdded` runs inside the MEASURED action- // window, so an O(document) lookup per mutation is workspace task #102 exactly: the instrument- // charging its own walk to the action, on a DOM whose size is the quantity under investigation.- // Three things keep that walk off the per-mutation path:- //- // 1. `aria-posinset` is read FIRST, so a windowed arm -- the only kind that mounts rows by the- // hundred as it scrolls -- never builds an index at all.- // 2. The index is built at most ONCE PER MUTATION BATCH, and only by a batch that actually- // mounted a message element. A stream is text churn inside rows that are already mounted:- // it adds no elements, so it builds nothing, which is what- // `test_the_top_up_is_proportional_to_the_mutation_not_to_the_document` pins.- // 3. It is read from the live DOM inside the callback, i.e. after every mutation in the batch- // has been applied, so ONE read describes the batch's settled state rather than some- // intermediate one. A rebuild that removes N rows and adds N rows in one commit is read- // once, and reads N rows rather than 2N.- //- // What is left paying for it is an arm publishing no ordinals that mounts rows mid-action:- // send_turn's one or two new messages, and thread_reopen's single rebuild. That is a handful of- // document reads per action window, in exchange for the ordinals being right.+ // THE POSITION INDEX, AND WHY THE FALLBACK ORDINAL IS NOT A COUNTER. A row publishing no+ // `aria-posinset` is stamped with its position among the thread's messages in the DOM as it+ // stands, resolved when the row is OBSERVED, because by delivery time it may be unmounted.+ // A LIFETIME COUNT OF OBSERVED NODES IS NOT THAT POSITION: `thread_reopen` recreates every message+ // row inside one document, and rebuilt rows legitimately publish no ordinal, so a counter stamped+ // them N+1..2N against the windowed arm's real 1..N and `compare_visible` reported two disjoint+ // visible sets for an identical rebuild.+ // THE COST, which is why a counter was tempting: `observeAdded` runs inside the MEASURED window,+ // so an O(document) lookup per mutation is the trap. Three things keep that walk off the+ // per-mutation path: `aria-posinset` is read FIRST, so a windowed arm never builds an index; the+ // index is built at most once per mutation batch and only by a batch that mounted a message+ // element; and it is read from the live DOM inside the callback, so a rebuild reads N not 2N.+ // Workspace task #102. The per-mutation bound is held by+ // test_the_top_up_is_proportional_to_the_mutation_not_to_the_document.const positionIndex = () => {if (VIS.index) return VIS.index;const nodes = (D().messages && D().messages()) || [];@@ -429,100 +263,64 @@return map;};- //: `position` is the row's 1-based position among the thread's messages when the caller already- //: knows it -- the full scan below holds the whole ordered list and needs no index. Left- //: undefined, the position is resolved from the index, and only when no ordinal was published.+ // `position` is the row's 1-based position among the thread's messages when the caller already+ // knows it; left undefined it is resolved from the index, and only when no ordinal was published.const observeOne = (el, position) => {if (!VIS.obs) return;let ord = ordinalOf(el, typeof position === "number" ? position : null);if (ord === null) ord = positionIndex().get(el) || 0;- // The ordinal is stamped on the node, because by the time an entry is delivered the node may- // have been unmounted and `closest()` would return nothing.+ // The ordinal is stamped on the node: by the time an entry is delivered the node may be unmounted+ // and `closest()` would return nothing.if (ord > 0) {- // RE-READ RATHER THAN WRITE ONCE. A placed node used to be marked `seen` and skipped for the- // rest of the window, which assumed a row's position in the thread cannot change while the- // node lives. A virtualizer that RECYCLES rows breaks that assumption on purpose: it hands- // the same node to another item and renumbers it. The stamp then stayed at the old ordinal- // for the rest of the run, so the row's content was digested under a position it no longer- // held, and the message it now showed was never reported visible at all.- //- // Re-reading is bounded work. The full scan runs once; every other call arrives from- // `observeAdded`, which walks only what a mutation added, and `positionIndex` is built at- // most once per batch.+ // RE-READ RATHER THAN WRITE ONCE: a virtualizer that RECYCLES rows renumbers them, so a `seen`+ // stamp stayed at the old ordinal, the row's content was digested under a position it no longer+ // held, and the message it now showed was never reported visible. Re-reading is bounded: the full+ // scan runs once and every other call walks only what a mutation added.const had = el.__sbOrdinal;el.__sbOrdinal = ord;if (typeof had === "number" && had !== ord) {- // A RENUMBERED ROW THAT NEVER STOPPED INTERSECTING REPORTS NOTHING. IntersectionObserver- // delivers on a CHANGE of intersection, and this node's intersection did not change --- // only its identity did -- so `ever` would never learn the new ordinal. Re-registering the- // target makes the observer deliver an initial entry for it, which is the documented- // behaviour of `observe()` and the only way to ask "is this thing on screen right now"- // without calling a geometry method, which this section may not do.+ // A RENUMBERED ROW THAT NEVER STOPPED INTERSECTING REPORTS NOTHING: IntersectionObserver delivers+ // on a CHANGE of intersection and only this node's identity changed. Re-registering the target+ // makes the observer deliver an initial entry, the only way to ask "is this on screen now"+ // without calling a geometry method.VIS.obs.unobserve(el);}} else {- // NO ORDINAL RATHER THAN A MADE-UP ONE. A row that publishes nothing and is not in the- // thread's message list has no position this instrument can honestly claim, and a guessed- // one lands in `ever_visible` as a message the other arm never showed.- //- // NOT MARKED `seen`, WHICH IS THE WHOLE POINT. Marking it here is what made an unplaced row- // able to hide a VISIBLE one. A row is unplaceable when it is observed while detached --- // added and removed inside one task, so no paint could have shown it -- and a virtualizer- // that RECYCLES DOM nodes hands that same node back on a later batch, mounted and about to- // be shown. With the node already in `seen` the early return fired, it was never stamped,- // and every intersection it went on to report was dropped for want of an ordinal: visible- // on this arm, absent from `ever_visible`, and `compare_visible` free to call MATCH on the- // strength of the rows that did place. Left out of `seen` it is simply placed on the batch- // that mounts it, which is the batch that can place it.- //- // Counted once per node all the same. `unplacedSeen` is a separate WeakSet so a node that- // is offered and refused several times before it mounts does not inflate the diagnostic- // into a number nobody can interpret.+ // NO ORDINAL RATHER THAN A MADE-UP ONE: a guessed position lands in `ever_visible` as a message+ // the other arm never showed. NOT marked `seen`, which is the whole point -- a row observed while+ // detached is handed back by a recycling virtualizer, and with the node already in `seen` every+ // later intersection was dropped and `compare_visible` was free to call MATCH.if (!VIS.unplacedSeen.has(el)) {VIS.unplacedSeen.add(el);VIS.unplaced += 1;}}- // Idempotent by specification: `observe()` on a target already being observed returns without- // adding a second registration, so re-offering an unplaced row costs nothing.+ // Idempotent by specification: `observe()` on a target already being observed adds no second+ // registration, so re-offering an unplaced row costs nothing.VIS.obs.observe(el);};- //: The FULL scan. O(the document), so it runs exactly once, when the observer is installed --- //: which happens before the measured window opens.+ // The FULL scan. O(the document), so it runs exactly once, when the observer is installed, which+ // is before the measured window opens.const observeAll = () => {if (!VIS.obs) return;const nodes = (D().messages && D().messages()) || [];for (let i = 0; i < nodes.length; i++) observeOne(nodes[i], i + 1);};- // THE TOP-UP, AND WHY IT IS NOT A RESCAN.- //- // A windowed list mounts rows as it scrolls, so rows appearing after the observer was installed- // have to be picked up or a windowed arm is only ever asked about what it happened to have- // mounted at the start. The obvious way to do that is to re-run the full scan from a- // MutationObserver -- and the MutationObserver runs DURING the measured action window, so a- // full `querySelectorAll` per mutation batch would charge an O(document) walk to the action,- // once per batch, on a 64,000-element DOM, growing with exactly the quantity under- // investigation. That is workspace task #102 verbatim: the census and the parity digest running- // inside the window, which reported delete_message at 14.3 fps when it costs 49.0.- //- // So this walks only what was ADDED. `addedNodes` is small by construction -- a virtualizer- // mounts one or two rows per scroll step -- and the cost is proportional to the mutation rather- // than to the document. The one exception is stated in full at `positionIndex` above: an arm- // that publishes no ordinals needs the thread's current message list to place a row it just- // mounted, and that list is read ONCE for the whole batch and never once per row.+ // THE TOP-UP, AND WHY IT IS NOT A RESCAN. A windowed list mounts rows as it scrolls, but+ // re-running the full scan from a MutationObserver would charge an O(document) walk to the+ // measured action once per batch on a 64,000-element DOM, which reported delete_message at 14.3+ // fps where it costs 49.0. So this walks only what was ADDED, and the one document read an+ // ordinal-less arm needs is taken once for the whole batch.const observeAdded = (records) => {- // ONE INDEX PER BATCH AT MOST, and none at all for a batch that mounts nothing needing one.- // Dropped again on the way out so the next batch cannot be answered from a stale list: rows- // mount and unmount between batches, and a position read from the previous DOM is precisely- // the wrong answer -- the same class of mistake as the lifetime counter this replaced.+ // ONE INDEX PER BATCH AT MOST, dropped on the way out so the next batch cannot be answered from a+ // stale list: a position read from the previous DOM is precisely the wrong answer, the same+ // class of mistake as the lifetime counter this replaced.VIS.index = null;for (const rec of records) {- // A RENUMBERED ROW ARRIVES AS ITS OWN TARGET, not in anybody's `addedNodes`. The ordinal- // lives on `[aria-posinset]`, which `ordinalOf` reaches with `closest()`, so the row that- // has to be re-placed is the target's message descendant, or the target itself when the- // attribute is on the message.+ // A RENUMBERED ROW ARRIVES AS ITS OWN TARGET, not in anybody's `addedNodes`, so the row to+ // re-place is the target's message descendant, or the target itself when the attribute is on it.if (rec.type === "attributes") {const t = rec.target;if (!t || t.nodeType !== 1) continue;@@ -539,8 +337,8 @@if (node.hasAttribute && node.hasAttribute("data-role")) {observeOne(node);}- // A row wrapper arrives with the message inside it, so the added node is the ancestor.- // Bounded by the added subtree, never by the document.+ // A row wrapper arrives with the message inside it, so the added node is the ancestor. Bounded by+ // the added subtree, never by the document.if (node.querySelectorAll) {for (const inner of node.querySelectorAll("[data-role]")) {observeOne(inner);@@ -569,17 +367,11 @@}}, { root: vp, threshold: 0 });observeAll();- // childList, plus ONE attribute by name, and no character-data records. A row ENTERING the- // DOM is most of what needs observing, and a row leaving it has already contributed to- // `ever`. Text changing inside a mounted row -- which is what a stream is -- must not- // reach this callback at all, and with `attributeFilter` naming a single attribute it- // cannot: a stream mutates text, not `aria-posinset`.- //- // `aria-posinset` is here because it is the one attribute whose change means the row is a- // DIFFERENT MESSAGE. A virtualizer that recycles a still-connected row renumbers it in- // place, which is not a childList mutation, so without this the renumbering is invisible- // and the node keeps a stamp that now names the wrong message. Filtered to that name the- // callback fires when a windowed arm re-uses a row and at no other time.+ // childList plus ONE attribute by name and no character-data records: text changing inside a+ // mounted row -- which is what a stream is -- must not reach this callback. `aria-posinset` is+ // here because its change means the row is a DIFFERENT MESSAGE, which is not a childList+ // mutation when a virtualizer renumbers a still-connected row in place.+ // `ordinalOf` reaches `[aria-posinset]` with `closest()`.VIS.mut = new MutationObserver(observeAdded);VIS.mut.observe(vp, {childList: true,@@ -599,22 +391,13 @@if (!VIS.watching) {return { visible_attempted: false, reason: "the visibility observer was never installed" };}- // WAIT FOR A FRAME FIRST, and this is not defensive padding.- //- // IntersectionObserver computes intersections as a step of the rendering lifecycle, so- // with no frame between `observe()` and the read there is nothing to deliver and- // `takeRecords()` returns an empty list. An action that scrolls produces frames and hides- // this completely; a QUIET action -- open a menu, change a setting, type a character --- // may not, and the capture then reports that the viewport showed nothing at all. The- // analysis would correctly refuse such a pair as NOT_COMPARABLE, so it is not a false- // pass, but it would silently strip visible-region coverage from exactly the actions that- // are cheapest to get right. Observed directly: at rest, with no scroll, the observer- // reported an empty set while message 1 filled the viewport.+ // WAIT FOR A FRAME FIRST: IntersectionObserver computes intersections as a step of the rendering+ // lifecycle, so with no frame between `observe()` and the read `takeRecords()` returns nothing.+ // Observed directly, the observer reported an empty set while message 1 filled the viewport.await new Promise((resolve) => {requestAnimationFrame(() => requestAnimationFrame(resolve));});- // Then deliver anything still queued. Tearing down before this drops the last entries of- // the action -- the ones most likely to describe what it just put on screen.+ // Then deliver anything still queued: tearing down first drops the last entries of the action.const records = VIS.obs.takeRecords();for (const entry of records) {if (entry.isIntersecting && typeof entry.target.__sbOrdinal === "number") {@@ -623,25 +406,15 @@}const dom = D();const nodes = (dom.messages && dom.messages()) || [];- // THE SAME NODE SET THE POSITIVE CONTROL BELOW COUNTS. The rows used to re-walk the- // `data-status` / `aria-busy` selectors inline, so "the rows read the same hooks as the- // control" was true only while two copies happened to agree. Read from one call and the- // control cannot fire while a row still claims to be in flight, or the reverse.+ // THE SAME NODE SET THE POSITIVE CONTROL BELOW COUNTS: read from one call, so the control cannot+ // fire while a row still claims to be in flight, or the reverse.const live = new Set((dom.streamingMessages && dom.streamingMessages()) || []);const byOrdinal = {};let mounted_ever_visible = 0;- // TWO MOUNTED ROWS CANNOT BE ONE MESSAGE, and until this counter existed nothing said so.- // The assignment below is keyed by ordinal, so a second row reusing one SILENTLY REPLACED- // the first row's digest, and `VIS.ever` is a Set of numbers so it collapsed them too.- // Whether the ghost row was caught was pure DOM order: reproduced in a browser, the same- // ghost inserted BEFORE the real row returns MATCH and AFTER it returns DIFFER.- //- // COUNTED DIRECTLY, NOT DERIVED. `unmounted_at_capture` looks like it should notice, but- // the extra row and the vacancy cancel in exactly the renumber case, so it reads a clean 0- // over a live collision.- //- // The residual: a collision that had resolved to one row by the time `capture()` ran is- // not detectable here. Seeing it needs the clash recorded where the ordinal is stamped.+ // TWO MOUNTED ROWS CANNOT BE ONE MESSAGE: the assignment below is keyed by ordinal, so a second+ // row reusing one silently replaced the first row's digest and `VIS.ever` collapsed them too.+ // Counted directly, not derived: `unmounted_at_capture` reads a clean 0 over a live collision+ // because the extra row and the vacancy cancel.let ordinal_collisions = 0;const collided = [];for (let i = 0; i < nodes.length; i++) {@@ -649,31 +422,12 @@const ord = typeof el.__sbOrdinal === "number" ? el.__sbOrdinal : ordinalOf(el, i + 1);if (!VIS.ever.has(ord)) continue;mounted_ever_visible += 1;- // WITHOUT THE VIRTUALIZATION BOOKKEEPING, and the asymmetry is the reason.- //- // runtime/readiness.py deliberately accepts `aria-posinset` / `aria-setsize` either on- // the `[data-role]` message or on an ancestor row wrapper -- it walks with `closest()`,- // because the ordinal belongs on whichever element is the member of the set. An arm that- // takes the first option therefore carries two attributes on EVERY message that the- // fully mounted arm publishes on none, so the per-message digests differed on every- // message while the rendered content was identical, and visible-region parity reported a- // wall of differences for a DOM shape the gate explicitly permits. That makes auto-mode- // parity unusable for the very arm it exists to score.- //- // NOT normalised in the whole-document structural digest, which is a separate decision- // and a deliberate one. That digest is only ever applied to pairs where NEITHER arm is- // windowing (see `decide_modes`), and between two fully mounted arms an ordinal that- // appears, disappears or changes IS a change worth seeing. `signature` is shared with- // the thread digest, the per-message rows, the overlays and the node-driven unit tests,- // so the exclusion is passed in HERE by the one caller that wants it rather than baked- // into the function every caller uses.- //- // What this gives up: a windowed arm publishing a WRONG ordinal on a message is no- // longer visible in this digest. It is not unchecked -- runtime/readiness.py gates on- // the ordinals being real positions (at least 1, distinct, no larger than the declared- // set size, reaching the seeded total) and `probe_thread_completeness` walks them for- // holes -- and those are the checks that can say what a right ordinal would be, which a- // digest comparison against an arm that publishes none never could.+ // WITHOUT THE VIRTUALIZATION BOOKKEEPING: runtime/readiness.py accepts `aria-posinset` /+ // `aria-setsize` on the message or on an ancestor row, so an arm taking the first option carries+ // two attributes on every message the fully mounted arm publishes on none, and visible-region+ // parity reported a wall of differences for a DOM shape the gate permits. NOT normalised in the+ // whole-document digest, which only runs where neither arm windows, so the exclusion is passed in+ // by this caller. A windowed arm publishing a WRONG ordinal is covered by the readiness gate.const sig = signature(el, VIRTUALIZATION_ATTRS);if (Object.prototype.hasOwnProperty.call(byOrdinal, String(ord))) {ordinal_collisions += 1;@@ -683,41 +437,25 @@role: el.getAttribute("data-role") || "?",digest: hash(sig),chars: sig.length,- // Still being written at capture time, so its digest names a point in a stream rather- // than a rendering. Carried here for the same reason as in the structural capture and- // handled the same way by `compare_visible`: residue, never a difference.+ // Still being written at capture time, so its digest names a point in a stream rather than a+ // rendering: residue, never a difference.in_flight: live.has(el),};}const ever = [...VIS.ever].sort((a, b) => a - b);- // THE SAME POSITIVE CONTROL THE STRUCTURAL CAPTURE CARRIES, because this payload is scored- // by `compare_visible` on its own and never consults the structural one. The per-row- // `in_flight` above is taken from the SAME `streamingMessages()` call, so it goes quiet in- // the same way and at the same moment, and the arms of an A/B are two DIFFERENT builds --- // `--ab REF` installs the treatment under its own `UNSLOTH_STUDIO_HOME` -- so a renamed- // hook on the treatment only is the asymmetric case rather than a symmetric one that- // cancels out. Both rows then read- // `in_flight: false`, one arm's reply is mid-tail and the other's has finished, and the- // loop in `compare_visible` scores that as a rendering difference: the wall-clock false- // alarm this whole change exists to remove.- //- // READ GLOBALLY, not over the visible rows. "Is a reply being written" and "is the message- // it is being written into on screen" are different questions, and a reply streaming below- // the fold is an ordinary state that must not refuse anything.+ // THE SAME POSITIVE CONTROL THE STRUCTURAL CAPTURE CARRIES, because this payload is scored by+ // `compare_visible` alone. The per-row `in_flight` comes from the SAME `streamingMessages()`+ // call, and the two arms are two different builds, so a hook renamed on the treatment only is+ // asymmetric rather than cancelling. READ GLOBALLY, not over the visible rows: a reply streaming+ // below the fold is ordinary and must not refuse anything.+ // `--ab REF` installs the treatment under its own `UNSLOTH_STUDIO_HOME`.const generating = dom.generating? Boolean(dom.generating()): Boolean(dom.isRunning && dom.isRunning());- // WHY THE PROBE IS QUIET, which is three questions and not one. See- // `dom.statusHookPresent` / `dom.lastAssistantPublishesStatus`.- //- // the last assistant message IS publishing parts and none of them says running- // -> the hook's VOCABULARY changed. Blind, unless this arm is windowing, in which case- // the message being written may not be the last one mounted and nothing here can- // tell the two apart.- // the last assistant message publishes NOTHING- // -> it has no parts yet, which is ordinary -- unless no assistant message anywhere- // publishes anything, and then the hook itself is gone. Blind on any arm, windowed- // or not, because a settled message would still be carrying it.+ // WHY THE PROBE IS QUIET is three questions, not one: parts published with none saying running+ // means the hook's VOCABULARY changed, blind unless this arm is windowing; the last assistant+ // message publishing nothing is ordinary unless no assistant message anywhere publishes, and+ // then the hook itself is gone.const lastPublishes = dom.lastAssistantPublishesStatus? Boolean(dom.lastAssistantPublishesStatus()): false;@@ -727,30 +465,27 @@return {visible_attempted: true,streaming: generating,- // Carried so a reader can tell the two zero-in-flight readings apart in the record, not- // only in the verdict: a stream this capture could not see, versus a hook that is gone.+ // Carried so a reader can tell the two zero-in-flight readings apart in the record: a stream this+ // capture could not see, versus a hook that is gone.status_hook_present: anyPublishes,- // `hookGone` and not merely "nothing is running": see `dom.statusHookPresent`. A windowed- // arm scrolled away from the tail can UNMOUNT the message being written, and- // `streamingMessages()` can only scan mounted DOM, so `live.size` is zero on a build- // whose hooks are perfectly intact. That is the ordinary state this mode exists to- // score, and refusing it would discard the settled rows that WERE on screen.+ // `hookGone` and not merely "nothing is running": a windowed arm scrolled away from the tail can+ // UNMOUNT the message being written, and `streamingMessages()` scans only mounted DOM, so+ // `live.size` is zero on a build whose hooks are intact.+ // See `dom.statusHookPresent`.+ // And `dom.lastAssistantPublishesStatus`.in_flight_unplaced: Boolean(generating && live.size === 0 && probeBlind),- // Every ordinal the viewport ever showed during the window, INCLUDING any that have- // since been unmounted. The gap between this and `messages` is the honest measure of- // what a windowed arm could not be asked about at capture time.+ // Every ordinal the viewport ever showed, including any since unmounted; the gap between this and+ // `messages` measures what a windowed arm could not be asked about at capture time.ever_visible: ever,ever_visible_count: ever.length,mounted_ever_visible,unmounted_at_capture: ever.length - mounted_ever_visible,- // Rows observed that this instrument could not place in the thread: no published- // ordinal, and absent from the message list when they were observed. They carry no- // ordinal and so appear nowhere above, which is a hole in the compared set rather than- // agreement, and it is counted here so a reader can see it is zero.+ // Rows this instrument could not place in the thread: no published ordinal and absent from the+ // message list when observed. They appear nowhere above, which is a hole in the compared set+ // rather than agreement, so it is counted here.unplaced_rows: VIS.unplaced,- // Mounted, ever-visible rows whose ordinal a row already written had. See the note in- // the loop: the digest map and `ever_visible` both lose one of the two, so a nonzero- // count here means neither of them can be compared.+ // Mounted, ever-visible rows whose ordinal a row already written had: the digest map and+ // `ever_visible` both lose one of the two, so a nonzero count means neither can be compared.ordinal_collisions,collided_ordinals: collided.sort((a, b) => a - b),messages: byOrdinal,@@ -772,103 +507,59 @@};window.__sb.parity = {- // Exposed so the offline unit tests can drive the exact regexes that ship, rather than a- // second copy of them that is free to drift.+ // Exposed so the offline unit tests drive the exact regexes that ship rather than a second copy free to drift.normText,normUrl,signature,hash,- // One digest for the whole thread, plus a digest PER MESSAGE so a mismatch says which message- // moved rather than only that something did. A whole-page digest that differs is a fact you- // cannot act on; the per-message row is what makes it a bug report.- //- // `opts.raw` additionally returns the signature TEXT. It is large, so it is off by default and- // used only by the null-control hunt, which has to name the volatile rather than count it.+ // One digest for the whole thread plus one PER MESSAGE, so a mismatch says which message moved+ // rather than only that something did. `opts.raw` also returns the signature TEXT: large, so it+ // is off by default and used only by the null-control hunt.capture(opts) {const want_raw = Boolean(opts && opts.raw);try {const dom = D();const rootFn = dom.threadRoot;const found = rootFn ? rootFn() : null;- // WHICH root was digested travels with the digest. `threadRoot()` falls back to- // `document.body`, which digests the sidebar, the header and every relative timestamp in- // them. Two arms that silently digested different roots would still produce two- // comparable-looking hashes, and the mismatch would be read as a UI change.+ // WHICH root was digested travels with the digest: `threadRoot()` falls back to `document.body`,+ // and two arms that silently digested different roots would still produce comparable-looking+ // hashes.const isThread = Boolean(found && found !== document.body &&found.classList && found.classList.contains("aui-thread-root"));const root = found || document.body;const nodes = (dom.messages && dom.messages()) || [];- // ── THE STREAMED MESSAGE, AND WHY IT GETS ITS OWN DIGEST ────────────────────────────────- //- // A message that is still being written has no defined moment. The two arms of an A/B are- // two cells run back to back against ONE pacer: the bytes on the wire are identical by- // construction, but each arm has its own send click, its own `t0` and its own paint clock,- // and the digest is taken at a wall-clock offset in the film rather than at a character- // count in the stream. So the two arms are compared at two different points in the same- // reply, and the difference that comes back is wall clock wearing the shape of a UI change.- //- // That is the failure this whole file's neighbours were written against: measuring at a- // moment whose meaning is not stable across the things being compared. It is the same- // mistake as a census taken on a `data-state` flip, and it is worse here because the- // renderer amplifies it. Mid-stream, Unsloth does not show a PREFIX of the finished reply:- // `parseIncompleteMarkdown` runs remend over the tail and closes whatever construct is- // half-arrived, KaTeX renders the repaired formula (and, while it will not parse, writes- // the parse error and its character offset into a `title`), Shiki re-tokenises the repaired- // fence, and the trailing code block carries `data-incomplete`. None of that is monotonic- // in how much text has arrived, which is why the difference cannot be recognised by its- // size and why a same-length mismatch here is not evidence of a missed volatile.- //- // MEASURED, on the shipped corpus (unit 9, 4,238 characters) driven through remend, KaTeX- // and Shiki into the shipped `signature()`. Stepping the arrived text by the pacer's own- // 24-character chunk, 175 of the 175 adjacent pairs produce a different digest -- one chunk- // of skew is enough to fail a stable action outright. At one-character resolution the- // serialised length moves DOWNWARDS at 52 of 4,237 steps and 34 pairs of distinct stream- // positions serialise to exactly the same length with different digests, which is the- // same-length drift that reads as a normaliser gap. It also cuts the other way: 398 of the- // 4,237 one-character advances move nothing at all.- //- // WHAT IS DONE ABOUT IT. The in-flight message is named, and a SECOND whole-thread digest- // is taken with its subtree elided -- a marker in its place, so its presence, its position- // and its role are still compared and a message that vanished still moves the digest. The- // comparison layer scores the settled document on the settled digest and refuses a verdict- // on the in-flight message rather than calling it a difference. It is NOT normalised away- // and it is not folded into a pass: see `analysis/parity.compare`.- //- // THE SCAFFOLD DIGEST ELIDES EVERY MESSAGE, NOT ONLY THE STREAMING ONES, and that is not- // over-reach -- it is the only version of this that is comparable across arms. Whether a- // given message is in flight is a property of ONE arm at the moment ITS digest was taken,- // and the ordinary case is precisely that the arms disagree about it: one has finished the- // reply and the other has not. A thread digest that elided each arm's own in-flight set- // would then be two different walks, and the two would differ because of the elision- // itself. Eliding all of them makes the walk identical on both sides by construction.- //- // Nothing is given up by decomposing it this way. The whole-thread digest is the scaffold- // plus every message subtree in place; the scaffold keeps a marker carrying each message's- // tag, role and position, and every message subtree is digested on its own row below. So a- // message that moved, vanished, changed role or changed content still moves something, and- // the comparison layer can withhold ONE of those rows without losing the rest.+ // THE STREAMED MESSAGE GETS ITS OWN DIGEST. A message still being written has no defined moment:+ // the two arms share one pacer so the bytes are identical, but each has its own send click and+ // paint clock while the digest is taken at a wall-clock offset in the film, so the arms are+ // compared at two different points in the same reply.+ // The renderer amplifies it: mid-stream `parseIncompleteMarkdown` remends the tail, KaTeX renders+ // the repaired formula, Shiki re-tokenises the repaired fence, and the trailing code block+ // carries `data-incomplete`. None of that is monotonic in how much text has arrived, so a+ // same-length mismatch is not evidence of a missed volatile.+ // MEASURED on the shipped corpus (unit 9, 4,238 characters): stepping by the pacer's 24-character+ // chunk, 175 of 175 adjacent pairs produce a different digest, so one chunk of skew fails a stable+ // action outright. At one-character resolution the serialised length even moves DOWNWARDS at 52+ // of 4,237 steps.+ // SO: the in-flight message is named and a SECOND whole-thread digest is taken with its subtree+ // elided, a marker keeping its presence, position and role. The comparison layer scores the+ // settled document on the settled digest and refuses a verdict on the in-flight message.+ // THE SCAFFOLD DIGEST ELIDES EVERY MESSAGE, not only the streaming ones: whether a message is in+ // flight is a property of ONE arm at the moment ITS digest was taken, and the ordinary case is+ // that the arms disagree, so per-arm elision would make the two walks differ because of the+ // elision itself. Nothing is given up -- the scaffold plus the per-message rows IS the digest.const inFlightNodes = (dom.streamingMessages && dom.streamingMessages()) || [];const inFlight = new Set(inFlightNodes);const running = Boolean(dom.isRunning && dom.isRunning());- // The narrower reading, and the positive control below is the one place that needs it.- // `isRunning()` is true whenever the composer is refusing a fresh send, and a prompt- // waiting in the queue on an IDLE thread refuses one too. See `dom.generating`.+ // The narrower reading, needed by the positive control below: `isRunning()` is true whenever the+ // composer refuses a fresh send, including a prompt queued on an IDLE thread.+ // `dom.generating` is the reading; `analysis/parity.compare` refuses to fold it into a pass.const generating = dom.generating ? Boolean(dom.generating()) : running;- // Whether the app still publishes the status contract at all. See- // `dom.statusHookPresent`: a hook that is GONE is a blind instrument, a hook that is- // present with nothing running is an ordinary settled or not-yet-mounted thread.- // WHY THE PROBE IS QUIET, which is three questions and not one. See- // `dom.statusHookPresent` / `dom.lastAssistantPublishesStatus`.- //- // the last assistant message IS publishing parts and none of them says running- // -> the hook's VOCABULARY changed. Blind, unless this arm is windowing, in which case- // the message being written may not be the last one mounted and nothing here can- // tell the two apart.- // the last assistant message publishes NOTHING- // -> it has no parts yet, which is ordinary -- unless no assistant message anywhere- // publishes anything, and then the hook itself is gone. Blind on any arm, windowed- // or not, because a settled message would still be carrying it.+ // Whether the app still publishes the status contract at all: a hook that is GONE is a blind+ // instrument, while one present with nothing running is an ordinary settled thread.+ // WHY THE PROBE IS QUIET is three questions, not one: parts published with none saying running+ // means the hook's VOCABULARY changed; nothing published at all is ordinary unless no assistant+ // message anywhere publishes, and then the hook is gone.const lastPublishes = dom.lastAssistantPublishesStatus? Boolean(dom.lastAssistantPublishesStatus()): false;@@ -894,9 +585,9 @@if (want_raw) row.raw = sig;messages.push(row);}- // Surfaces that live OUTSIDE the thread root and are therefore invisible to the digest- // above: an open dialog, an open menu, the model picker. A perf change that alters a- // popover would otherwise pass a thread-only parity check.+ // Surfaces OUTSIDE the thread root and therefore invisible to the digest above -- an open dialog,+ // an open menu, the model picker -- so a perf change that alters a popover cannot pass a+ // thread-only parity check.const overlays = [];for (const sel of ['[data-slot="dialog-content"]', '[role="menu"]', '[role="dialog"]',".unsloth-model-selector-menu", "[data-radix-popper-content-wrapper]"]) {@@ -914,80 +605,55 @@root_kind: isThread ? "thread" : "body",digest: hash(whole),chars: whole.length,- // The thread with every message replaced by a marker: the viewport, the composer, the- // empty state, and each message's tag, role and position. `digest` = this plus the- // per-message rows below, which is what lets a single message be withheld from the- // comparison without the other two thirds of the reading going with it.+ // The thread with every message replaced by a marker carrying its tag, role and position.+ // `digest` = this plus the per-message rows below, which is what lets one message be withheld+ // from the comparison without the rest of the reading going with it.digest_scaffold: hash(scaffold),chars_scaffold: scaffold.length,- // Mounted indices of the messages that were still being written. An EMPTY list on a- // page where nothing is running is the ordinary case and means what it says.+ // Mounted indices of the messages still being written; an empty list on a page where nothing is+ // running is the ordinary case.in_flight,streaming: running,- // THE POSITIVE CONTROL, and it is not decoration. `streamingMessages()` walks selectors- // written against Unsloth's markup: rename `data-status` and it goes quiet, matches- // nothing, and every reading silently becomes "nothing was streaming" -- which is the- // strongest claim this capture can make about the stream and would be supported by no- // observation at all. The app says a reply is running through a different control (the- // Stop button), so when the two disagree the disagreement is carried out rather than- // resolved here, and `analysis/parity.comparability` refuses the pair.- //- // IT IS `generating()` AND NOT `isRunning()` that is compared against, and that- // difference is most of this field's meaning. `isRunning()` is also true while a queued- // prompt waits on a thread that is doing NOTHING, and on that thread no message- // publishes a running status because none is running -- so reading it here would refuse- // an ordinary settled pair before `compare()` ever reached its settled digests, and- // would report the instrument as broken to say it. That is this file's own failure- // mode: reading at a moment whose meaning is not stable across the things being- // compared. See `dom.generating`.- //- // AND IT REQUIRES THE HOOK TO BE GONE, not merely quiet. `streamingMessages()` scans- // MOUNTED DOM, so it also returns nothing in the gap between a send being accepted and- // the reply's first part rendering, and on an arm that has unmounted the message it is- // writing into. Neither is a broken instrument. See `dom.statusHookPresent`.+ // THE POSITIVE CONTROL: `streamingMessages()` walks selectors written against Unsloth's markup,+ // so renaming `data-status` makes every reading silently become "nothing was streaming". The app+ // also reports a running reply through the Stop button, so a disagreement is carried out rather+ // than resolved here, and `analysis/parity.comparability` refuses the pair. It compares+ // `generating()` and NOT `isRunning()`, which is also true while+ // a queued prompt waits on an idle thread. And it requires the hook to be GONE, not merely quiet:+ // `streamingMessages()` scans mounted DOM.in_flight_unplaced: Boolean(generating && inFlightNodes.length === 0 && probeBlind),status_hook_present: anyPublishes,- // The queued-idle interval itself, recorded rather than resolved away, so a reader can- // tell why a capture with `streaming: true` placed no message in flight.- // INCLUDING THE DISPATCHED WAIT. `isRunning()` matches "Stop generating" and- // "Queue message" only, so the interval where a dispatched queue entry renders- // "Stop queued message" recorded `streaming` and `queued_idle` both false -- identical- // to a settled Send arm. A pair straddling that transient then had a differing composer- // with no run-state difference to account for it, and the comparison layer is entitled- // to call that a rendering regression. It is queue timing.+ // The queued-idle interval itself, recorded rather than resolved away, so a reader can tell why a+ // capture with `streaming: true` placed no message in flight.+ // INCLUDING THE DISPATCHED WAIT: `isRunning()` matches "Stop generating" and "Queue message"+ // only, so the interval rendering "Stop queued message" recorded both flags false, identical to a+ // settled Send arm, and a pair straddling it looked like a rendering regression.queued_idle: Boolean((running || (dom.stopQueuedButton ? Boolean(dom.stopQueuedButton()) : false)) &&!generating),- // The composer's run-state slot, as a token. Carried so the comparison layer can tell a- // scaffold that differs because the two arms were at different points in one turn from a- // scaffold that differs because something was rendered differently. See- // `dom.runStateControl` and `analysis/parity.generation_disagrees`.+ // The composer's run-state slot as a token, so the comparison layer can tell a scaffold that+ // differs because the arms were at different points in one turn from one that differs because+ // something rendered differently.+ // See `dom.runStateControl` and `analysis/parity.generation_disagrees`.composer_control: dom.runStateControl ? dom.runStateControl() : null,messages,overlays,styles,- // HOW MUCH OF THE THREAD THIS DIGEST COVERS.- //- // Every per-message row above is keyed by `i`, its position in the MOUNTED list. On the- // shipped build that is also its position in the conversation, so the key is meaningful- // and two arms can be compared row by row. On an arm that mounts a window it is not:- // msg3 on one side and msg3 on the other are different messages, and comparing their- // digests produces a wall of mismatches that says nothing about either build.- //- // Carrying the two numbers means the comparison layer can REFUSE rather than report- // eighteen false differences. Identical on the shipped build, so nothing changes for a- // normal pair.+ // HOW MUCH OF THE THREAD THIS DIGEST COVERS. Every per-message row is keyed by its position in+ // the MOUNTED list, which equals its conversation position only on the shipped build; on a+ // windowed arm msg3 and msg3 are different messages and a row-by-row comparison says nothing.+ // Carrying the two numbers lets the comparison layer REFUSE instead.mounted_messages: messages.length,thread_total: (dom.threadTotal && dom.threadTotal()) || messages.length,};if (want_raw) out.raw = whole;return out;} catch (err) {- // A failure is reported as a failure. A parity check that returns an empty digest on- // error would read as "everything matched", which is the single worst thing it could do.+ // A failure is reported as a failure: a parity check that returned an empty digest on error would+ // read as "everything matched".return { parity_attempted: false, reason: String(err && err.message ? err.message : err) };}},diff --git a/tests/studio/studiobench/scene/schedule.py b/tests/studio/studiobench/scene/schedule.pyindex cc5a8b7b628..20bb0c5b3a9 100644--- a/tests/studio/studiobench/scene/schedule.py+++ b/tests/studio/studiobench/scene/schedule.py@@ -76,11 +76,9 @@ def _slots(spec: list[tuple[str, int, Optional[int]]]) -> list[Slot]:]-# The standard film. Ordered so that the actions that must happen DURING generation come first,-# then the ones that need a finished reply, then the destructive ones last -- delete and reopen-# change the thread, so anything after them would be measuring a different thread.-#-# Timings are offsets from the moment the send button was pressed.+# The standard film, ordered so actions that must happen DURING generation come first, then+# ones needing a finished reply, then the destructive ones last, since delete and reopen change+# the thread. Timings are offsets from the send button press.STANDARD = Scene(name = "standard",slots = _slots(@@ -88,10 +86,9 @@ STANDARD = Scene(# ── during generation ────────────────────────────────────────("scroll_during_generation", 3_000, 8_000),("keystroke", 12_000, 6_000),- # 12s. The tail is a JITTERED clip capped at 6,000 characters, so its drain time varies from- # about 14s to 18s across the ladder, and a during-generation slot has to open before the- # SHORTEST of those, not the longest. At 19s, then 15s, this slot ran against a finished reply- # at the top rung while still being labelled "during generation".+ # 12s: the tail is a JITTERED clip capped at 6,000 characters, so its drain varies from about+ # 14s to 18s and a during-generation slot must open before the SHORTEST of those. At 19s, then+ # 15s, this slot ran against a finished reply at the top rung.("scroll_during_generation", 12_000, 8_000),("stop_generation", 28_000, 8_000),# ── after the reply is complete ──────────────────────────────@@ -102,61 +99,48 @@ STANDARD = Scene(("copy_markdown", 86_000, 6_000),("select_text", 93_000, 6_000),("send_turn", 100_000, 12_000),- # 35s, not 10s. Selecting and copying the whole thread at the 1M rung took 27,690 ms -- two- # million characters through the clipboard -- so it blew a 10s budget by nearly three times- # and the OVERRUN pushed the next slot past its own start, which was recorded as- # `composer_fill: slot missed`. The 1M cell lost two slots that way and the cause looked like- # a slow machine rather than one under-budgeted action.- #- # This film is the one the 500K and 1M rungs use, so its budgets are sized from what those- # rungs actually cost, not from what the small ones do. At 100K the same action takes 2,476 ms.+ # 35s, not 10s: selecting and copying the whole thread at 1M took 27,690 ms, nearly three times+ # a 10s budget, and the OVERRUN pushed the next slot past its own start, recorded as+ # `composer_fill: slot missed`. This film is what 500K and 1M use, so its budgets are sized+ # from what those rungs cost; at 100K the same action takes 2,476 ms.("select_all_copy", 113_000, 35_000),("composer_fill", 149_000, 10_000),("model_change", 160_000, 10_000),("settings", 171_000, 12_000),("image_upload", 184_000, 12_000),- # ── destructive, last ────────────────────────────────────────# 22,382 ms at 1M, so 30s stands.+ # ── destructive, last ────────────────────────────────────────("thread_reopen", 197_000, 30_000),("delete_message", 228_000, 15_000),]),)-# The quick film. The SAME fifteen actions in the same order -- a tier that drops actions cannot be-# compared with one that does not -- on a shorter clock, for the small rungs where the stream is-# over in seconds.+# The quick film: the SAME fifteen actions in the same order, since a tier that drops actions+# cannot be compared with one that does not, on a shorter clock for the small rungs.QUICK = Scene(name = "quick",slots = _slots([- # BUDGETS ARE SIZED FROM MEASURED ACTION COST, not guessed. At the 100K rung -- the largest- # this film is used for -- every action finished inside 2.5 s: select_all_copy 2,476 ms,- # thread_reopen 2,234 ms, reasoning_toggle 1,788 ms, keystroke 1,041 ms, everything else under- # 500 ms. The budgets below carry roughly 2.5x headroom over those.- #- # The film previously ran 162 s while its actions used 6.4 s, so 96% of it was waiting for the- # next slot to open. That waiting is not free: it is multiplied by every cell, every arm and- # every repetition, and an A/B at four reps is sixteen films. Halving the film halves the cost- # of every comparison this tool exists to make.- #- # What CANNOT be compressed is the stream phase. The opening turn drains in 12 to 18 s, the- # during-generation slots have to open inside the shortest of those (14.1 s at 1M) and the- # after-generation slots have to open after the longest (17.8 s at 100K). The gap between- # 12 s and 20 s below is that constraint, not slack.+ # BUDGETS ARE SIZED FROM MEASURED ACTION COST: at 100K, the largest rung this film is used for,+ # every action finished inside 2.5 s (select_all_copy 2,476 ms, thread_reopen 2,234 ms,+ # reasoning_toggle 1,788 ms, keystroke 1,041 ms, the rest under 500 ms), and the budgets carry+ # roughly 2.5x headroom. The film previously ran 162 s while its actions used 6.4 s, and that+ # waiting is multiplied by every cell, arm and repetition. What CANNOT be compressed is the+ # stream phase: the opening turn drains in 12 to 18 s, during-generation slots must open inside+ # the shortest (14.1 s at 1M) and after-generation slots after the longest (17.8 s at 100K), so+ # the gap between 12 s and 20 s is that constraint, not slack.("scroll_during_generation", 1_500, 2_500),("keystroke", 5_000, 3_000),("scroll_during_generation", 9_500, 2_500),("stop_generation", 20_000, 3_000),("scroll_after", 23_500, 2_500),("reasoning_toggle", 26_500, 4_500),- # 1,500 rather than 4,000, and the reason is the gap AFTER it rather than the cost of- # the send itself. `send_turn` is a sub-100 ms action; a 4,000 ms window to start in is- # not headroom, it is permission to begin the follow-up four seconds late without- # recording a miss. The drain runs from when the send actually fires, so every one of- # those four seconds comes off `message_menu`'s window: measured from the latest legal- # send, this film left 3,500 ms for a 4,562 ms drain. The fast film had the same shape- # and CI failed on it. See test_settled_actions_open_after_the_follow_up_drains.+ # 1,500 rather than 4,000, because of the gap AFTER it rather than the cost of the send:+ # `send_turn` is a sub-100 ms action, so a 4,000 ms window is permission to begin the follow-up+ # four seconds late without recording a miss, and every one of those seconds comes off+ # `message_menu`'s window. Measured from the latest legal send, this film left 3,500 ms for a+ # 4,562 ms drain; the fast film had the same shape and CI failed on it.("send_turn", 31_500, 1_500),("message_menu", 36_000, 3_000),("copy_markdown", 39_500, 2_500),@@ -173,78 +157,51 @@ QUICK = Scene(),)-# The fast film. FOR ITERATION, NOT FOR REPORTING -- see the banner the CLI prints.-#-# Same eighteen actions in the same order as the other two films, because a tier that drops-# actions cannot tell you that your fix broke the one it dropped. What changes is the waiting.-#-# Budgets are sized from the costs this tool MEASURED at the 100K rung over eight cells of a null-# control, at roughly 1.5x rather than the 2.5x the quick film carries:-#-# thread_reopen 3,163 ms (close 2,205 + reopen 958) select_all_copy 2,454 ms-# reasoning_toggle 2,250 ms (open 1,688 + close 563) keystroke 1,002 ms-# settings 485 ms model_change 483 ms scroll 466 ms stop 335 ms copy 204 ms-#-# Everything else is under 100 ms. The eighteen actions cost about 12 s of real work; the standard-# film spends 243 s and the quick film 77.5 s to deliver them. At 1.5x headroom an action that-# overruns records `slot_missed` instead of silently pushing the next slot, which is the honest-# failure mode and is exactly what the fixed-duration design exists to provide.-#-# WHAT CANNOT BE COMPRESSED, and why this film is 47 s rather than 20 s: the opening turn streams-# a 6,000 character tail at field cadence (328.8 chars/s), so it drains in 14 to 18 s. The-# during-generation slots must open inside the SHORTEST of those and the after-generation slots-# after the LONGEST. The gap between 8 s and 19 s below is that constraint, not slack. Shrinking-# it means shrinking the streamed tail, which changes the load being measured -- and streaming-# frame cost at length is the thing most of these fixes are about.+# The fast film. FOR ITERATION, NOT FOR REPORTING. Same eighteen actions in the same order as+# the other two, because a tier that drops actions cannot tell you that your fix broke the one+# it dropped; what changes is the waiting. Budgets are sized from costs measured at 100K over+# eight null-control cells at roughly 1.5x rather than the quick film's 2.5x (thread_reopen+# 3,163 ms and select_all_copy 2,454 ms at the top, everything past model_change under 500 ms).+# At 1.5x headroom an action that overruns records `slot_missed` instead of silently pushing+# the next slot. WHAT CANNOT BE COMPRESSED, and why this film is 47 s: the opening turn streams+# a 6,000 character tail at field cadence (328.8 chars/s), draining in 14 to 18 s, so the gap+# between 8 s and 19 s below is that constraint, and shrinking it means shrinking the streamed+# tail, which changes the load being measured.FAST = Scene(name = "fast",slots = _slots([- # 18.4 s, not 8 s. `stop_generation` starts and stops its OWN turn, so opening it while the- # opening tail is still draining starts a second turn on top of the first and truncates the- # reply being measured -- which is the defect that made the seeded-vs-streamed equivalence- # check read a false 20% drift earlier in this project.- #- # KEYED TO THE DECLARED TAIL, NOT TO WHAT THE CORPUS HAPPENS TO STREAM. This said 17.8 s- # and sat at 18.2 s, which was under the ladder's own ceiling the whole time and only- # passed because the 1M rung was streaming recycled text: the manifest was sized at- # exactly the top rung's seeded target, so all three streamed turns clamped onto the last- # unit. Sizing the manifest from the ladder gave 1M its own material and the drain went- # to 18.23 s, which is what a corpus this tier can be pointed at was always allowed to- # do. STREAM_TAIL_CHARS (6,000) at field cadence is 18.25 s and every rung is held under- # it, so that ceiling is the bound to clear, and clearing it costs 200 ms here.- #- # THE SECOND PACKING CONSTRAINT, learned the expensive way. `send_turn` starts a FOLLOW-UP- # turn, and a follow-up is FOLLOW_UP_CHARS (1,500) at field cadence, so it streams for 4.6 s.- # Anything needing a SETTLED reply -- the action bar's More and Copy buttons, select-all,- # delete -- does not exist while that turn is running. The first fast film opened- # `message_menu` 1.7 s after a `send_turn` and it recorded `NOT RUN: no More button` on- # 312 of 312 attempts across a 36 job sweep, silently removing the four actions that carry- # the largest known effect in this codebase (the message menu is the 19x one). Every- # post-send slot below clears 4.6 s, and test_settled_actions_open_after_the_follow_up_drains- # now fails any film that does not.+ # 18.4 s, not 8 s: `stop_generation` starts and stops its OWN turn, so opening it while the+ # opening tail is still draining truncates the reply being measured, the defect that made the+ # seeded-vs-streamed check read a false 20% drift. KEYED TO THE DECLARED TAIL, not to what the+ # corpus happens to stream: at 17.8 s it passed only because the 1M rung was streaming recycled+ # text, and STREAM_TAIL_CHARS (6,000) at field cadence is 18.25 s, the ceiling to clear.+ # THE SECOND PACKING CONSTRAINT, learned the expensive way: `send_turn` starts a FOLLOW-UP turn+ # of FOLLOW_UP_CHARS (1,500) at field cadence, so it streams for 4.6 s, and anything needing a+ # SETTLED reply does not exist while it runs. The first fast film opened `message_menu` 1.7 s+ # after a `send_turn` and recorded `NOT RUN: no More button` on 312 of 312 attempts across a 36+ # job sweep. Every post-send slot below clears 4.6 s, and+ # test_settled_actions_open_after_the_follow_up_drains fails any film that does not.("scroll_during_generation", 1_500, 1_200),("keystroke", 3_000, 1_800),("scroll_during_generation", 6_000, 1_200),("stop_generation", 18_400, 3_000),("scroll_after", 21_500, 1_200),- # THE BUDGET IS SIZED FROM WHAT CI MEASURES, not from the null control's 2,250 ms.- # On the GitHub runner this action costs 4,415.9 ms and 4,434.4 ms on two consecutive- # jobs -- reaching the open state alone takes 2,898.8 ms there, which already exceeds- # the whole-action figure the 3,500 ms budget was derived from. At 3,500 it overran by- # 934 ms on every run, and an overrun here lands on `send_turn`, the one slot whose- # downstream gap is load-bearing. 5,000 ms covers the measured cost with room.+ # THE BUDGET IS SIZED FROM WHAT CI MEASURES, not the null control's 2,250 ms: on the GitHub+ # runner this action costs about 4,420 ms, of which reaching the open state alone is 2,898.8+ # ms. At 3,500 it overran by 934 ms on every run, and an overrun here lands on `send_turn`, the+ # one slot whose downstream gap is load-bearing.+ # 5,000 ms covers the measured cost with room.("reasoning_toggle", 23_000, 5_000),- # MOVED OUT FROM UNDER `reasoning_toggle`, which now nominally ends at 28,000. A send- # slot that opens before the action before it can finish is a slot that starts late,- # and a send that starts late shortens the drain window of everything after it while- # reporting no miss of its own.+ # MOVED OUT FROM UNDER `reasoning_toggle`: a send slot that opens before the action before it+ # can finish starts late, and a late send shortens the drain window of everything after it+ # while reporting no miss of its own.+ # Which now nominally ends at 28,000.("send_turn", 28_100, 1_500),- # THE GAP IS THE POINT, and it is measured from 29,600 -- the latest this film's send- # can fire -- not from 28,100. The follow-up drains in 4,562 ms nominal and 4,400 to- # 4,700 ms observed, so a window closing at 35,400 leaves about 1.1 s of real margin- # against the worst drain seen. The old packing left 38 ms, which is why CI failed on- # a runner no slower than the one that passed.+ # THE GAP IS THE POINT, measured from 29,600, the latest this film's send can fire, not from+ # 28,100. The follow-up drains in 4,562 ms nominal and 4,400 to 4,700 observed, so a window+ # closing at 35,400 leaves about 1.1 s of margin; the old packing left 38 ms, which is why CI+ # failed on a runner no slower than the one that passed.("message_menu", 34_400, 1_000),("copy_markdown", 35_400, 600),("select_text", 36_200, 400),@@ -254,12 +211,10 @@ FAST = Scene(("model_change", 46_900, 1_000),("settings", 48_100, 1_200),("image_upload", 49_500, 800),- # thread_reopen 6.5 s and delete 2.5 s, not 5 s and 0.6 s. Measured at 100K the pair- # costs about 3.2 s, but the ACTION overran its 5 s window and pushed the last slot: the- # machine arrived at delete_message 834 ms after its 600 ms budget had already closed, so- # on a 36 job sweep delete recorded NOT EXERCISED on the base arm of every null-control- # cell. A last slot with no slack is a slot that measures nothing, and the film's own end- # is the one place an overrun has nowhere to go.+ # thread_reopen 6.5 s and delete 2.5 s, not 5 s and 0.6 s: measured at 100K the pair costs about+ # 3.2 s, but the ACTION overran its 5 s window and pushed the last slot, so delete recorded NOT+ # EXERCISED on the base arm of every null-control cell in a 36 job sweep. The film's own end is+ # the one place an overrun has nowhere to go.("thread_reopen", 50_500, 6_500),("delete_message", 57_200, 2_500),]@@ -285,11 +240,9 @@ class SceneRunner:"""`t0` is the driver monotonic time the film started, i.e. when send was pressed."""rows: list[dict] = []for i, slot in enumerate(scene.slots):- # The GAP before this slot is itself a measured window. Without it, frame rate and- # blocked time would only ever be sampled inside actions -- and the quiet stretches- # between them, which is where the stream is doing its work unaided, would be the one- # part of the session nothing observed. Continuous coverage, no sampler thread, and- # no window ever overlaps another.+ # The GAP before this slot is itself a measured window: without it, frame rate and blocked time+ # would only be sampled inside actions, and the quiet stretches where the stream does its work+ # unaided would be the one part of the session nothing observed.self._gap_window(f"stream:gap{i}", slot.t_start_ms, t0)row = self._run_slot(slot, t0)rows.append(row)@@ -312,8 +265,8 @@ class SceneRunner:try:self.page.evaluate("() => window.__sb.parityVisible.watch()")except Exception: # noqa: BLE001- # A page that cannot install it still gets a structural digest; the visible-region- # capture reports itself absent rather than empty, which the analysis refuses.+ # A page that cannot install it still gets a structural digest; the visible-region capture+ # reports itself absent rather than empty, which the analysis refuses.passdef _visible(self) -> dict:@@ -380,32 +333,18 @@ class SceneRunner:now_ms = (time.monotonic() - t0) * 1000if until_ms - now_ms < 250:return- # `gap`, NOT `stream`. These windows were labelled `stream` and the label was read by- # people, including this project, as meaning the stream was running in them. It does not.- # A gap window is opened before EVERY slot, so on the standard film eighteen of them exist- # and only the first four contain any streaming at all; the rest are the quiet stretches- # between post-generation actions. Measured on a 100K cell, `stream:gap12` ran for 32.9 s- # at 1.6% busy with the reply finished thirty seconds earlier, and `stream:drain` -- the- # one window that really is about the stream -- was 7 ms long.- #- # Anyone filtering on `kind == "stream"` to find the streaming phase therefore selected- # mostly post-stream idle and diluted whatever they were measuring. The phase has to be- # detected from the SSE traffic (see instruments/streamcost.py), and this label no longer- # invites the mistake.- #- # The window NAME is deliberately left as `stream:gapN`. It is the join key in every- # payload this tool has already written and in the analysis scripts pointed at them;- # renaming it would break those silently, which is a worse failure than a misleading name- # that is now documented in INTERFACES.md and contradicted by the kind beside it.- # THE CENSUS IS TAKEN BEFORE THE GAP WINDOW OPENS, not at its close. Same defect as the- # action windows (workspace task #102) and the same fix, but the placement is different:- # a gap window is the QUIET stretch, and its whole purpose is to measure the page doing- # nothing but stream. Nineteen querySelectorAll passes over 195,000 elements at the end of- # it is not nothing, and it landed on the frame-rate reading for the idle phase.- #- # Before rather than after, because the gap ends the instant the next slot is due: taking- # it afterwards would push the action's start past its own slot and turn an instrument cost- # into a missed slot, which is a worse failure than the one being fixed.+ # `gap`, NOT `stream`. These windows were labelled `stream` and read as meaning the stream was+ # running in them. A gap window opens before EVERY slot, so on the standard film eighteen exist+ # and only the first four contain any streaming: on a 100K cell, `stream:gap12` ran 32.9 s at+ # 1.6% busy with the reply finished thirty seconds earlier, while `stream:drain` was 7 ms.+ # Anyone filtering on `kind == "stream"` therefore selected mostly post-stream idle. The window+ # NAME is deliberately left as `stream:gapN`, since it is the join key in every payload already+ # written.+ # THE CENSUS IS TAKEN BEFORE THE GAP WINDOW OPENS, not at its close (workspace task #102): a gap+ # window is the QUIET stretch, and nineteen querySelectorAll passes over 195,000 elements at the+ # end of it landed on the frame-rate reading for the idle phase. Before rather than after,+ # because the gap ends the instant the next slot is due, so taking it afterwards would turn an+ # instrument cost into a missed slot.census = self._census()with self.open_window(name, "gap") as window:window.note("census_before_gap", census)@@ -422,8 +361,8 @@ class SceneRunner:).row(slot.action, window_name, self.cell.cell_id)now_ms = (time.monotonic() - t0) * 1000- # Wait for the slot to open. Sleeping in small steps rather than one long sleep so a- # renderer crash is noticed within a fifth of a second rather than at the end of the wait.+ # Wait for the slot to open, in small steps rather than one long sleep, so a renderer crash is+ # noticed within a fifth of a second.while now_ms < slot.t_start_ms:time.sleep(min(0.2, (slot.t_start_ms - now_ms) / 1000))now_ms = (time.monotonic() - t0) * 1000@@ -431,8 +370,8 @@ class SceneRunner:deadline_ms = slot.t_start_ms + slot.budget_msremaining = deadline_ms - now_msif remaining <= 0:- # THE SLOT WAS MISSED. Not an error and not a slow timing: this machine could not get- # here in time, the film carries on, and the row says exactly that.+ # THE SLOT WAS MISSED. Not an error and not a slow timing: this machine could not get here in+ # time, the film carries on, and the row says exactly that.self.log(f" slot missed: {slot.action} "f"(due at {slot.t_start_ms}ms, reached at {now_ms:.0f}ms)"@@ -467,86 +406,53 @@ class SceneRunner:window.note("action", slot.action)window.note("ran", result.ran)- # THE CENSUS AND THE PARITY DIGEST ARE TAKEN OUTSIDE THE WINDOW. Workspace task #102.- #- # Both used to run inside the `with`, one line above this comment, on the strength of a- # measurement -- "0.2ms on a 1,500-element tree" -- taken on a tree three orders of- # magnitude smaller than the one this benchmark exists to study. At 100K+ tokens the- # census walks nineteen querySelectorAll passes over ~195,000 elements and the parity- # digest serialises 5.6 MB of structure, and every millisecond of it was being charged to- # the action that happened to precede it. Measured cost of the mistake:- #- # delete_message reported 14.3 fps, true 49.0 fps- # message_menu reported 17.1 fps, true 73.8 fps- #- # That is not a correction at the margin. It inverted the ranking of the actions this- # campaign is about, and it did it in the direction that makes the standing DOM look like- # a smaller problem than it is, because the instrument's own cost grows with exactly the- # quantity under investigation.- #- # They still run at the same MOMENT -- immediately after the window closes, before the- # scheduler starts waiting for the next slot -- so the digest and the occupancy still come- # from one reading of one DOM taken at the end of the action. Only the accounting moved.- #- # The consequence, stated: the emitted `window` row no longer carries them in its notes,- # because the row is emitted when the window closes. They live on the ACTION row, which is- # where every consumer already reads them from.- # THE DEADLINE IS SAMPLED HERE, BEFORE THE OBSERVATIONS. Moving the census and the digest- # out of the measured window but leaving `over_ms` to be taken after them would have left- # half the defect in place: an action that finished comfortably inside its budget would- # still be flagged `over_budget` on the strength of a multi-megabyte serialisation that is- # explicitly not its cost. Charging instrument time to the thing being measured is the- # whole of task #102, and the budget flag is not exempt from it.+ # THE CENSUS AND THE PARITY DIGEST ARE TAKEN OUTSIDE THE WINDOW (workspace task #102). Both used+ # to run inside the `with`, on the strength of "0.2ms on a 1,500-element tree" measured on a+ # tree three orders of magnitude smaller: at 100K+ the census walks nineteen querySelectorAll+ # passes over ~195,000 elements and the digest serialises 5.6 MB, and every millisecond was+ # charged to the preceding action, so delete_message reported 14.3 fps against a true 49.0. That+ # inverted the ranking of the actions this campaign is about, in the direction that makes+ # standing DOM look like a smaller problem, because the instrument's cost grows with the+ # quantity under investigation. They still run at the same MOMENT, so the digest and occupancy+ # come from one reading; only the accounting moved, and they now live on the ACTION row.+ # And message_menu 17.1 fps against a true 73.8.+ # THE DEADLINE IS SAMPLED HERE, BEFORE THE OBSERVATIONS: leaving `over_ms` to be taken after them+ # would still flag an action `over_budget` on the strength of a multi-megabyte serialisation+ # that is explicitly not its cost.window_closed_at = time.monotonic()over_ms = ((window_closed_at - t0) * 1000) - deadline_ms- # THE VISIBLE CAPTURE GOES FIRST, AND THE ORDER IS THE MEASUREMENT. The other two read a- # DOM that is sitting still; this one CLOSES an observation that has been accumulating- # since `_watch_visible`, and every millisecond it stays open is another millisecond in- # which a row can scroll into view and be counted as having been visible during the- # action. Taken after the census and the digest, that tail was as long as those two probes- # take -- and they are the one thing on this path whose cost is proportional to the arm.- #- # A fully mounted arm serialises the whole thread, a windowed arm serialises the dozen- # rows it has mounted, so the two arms' observers stayed open for materially different- # intervals: the numbers above this comment are 14.3 fps against 49.0, which is the same- # asymmetry expressed as time. With streaming and autoscroll still running at the close of- # `scroll_during_generation`, the mounted arm had the longer tail in which to admit an- # ordinal the windowed arm never saw, and `compare_visible` has no tolerance for that --- # it returns DIFFER on strict set inequality of `ever_visible`. The null control cannot- # absorb it either, being same-build against same-build, where both sides pay the same- # digest and the asymmetry does not exist.- #- # Taking it first does not make the tail zero: `capture()` still waits two frames, on- # purpose, so a quiet action does not report an empty viewport. It makes the tail the SAME- # ON BOTH ARMS, which is the property the comparison actually needs.+ # THE VISIBLE CAPTURE GOES FIRST, AND THE ORDER IS THE MEASUREMENT. The other two read a DOM+ # sitting still; this one CLOSES an observation accumulating since `_watch_visible`, so every+ # millisecond it stays open is another in which a row can scroll into view and be counted as+ # visible during the action. Taken after the census and digest, that tail was as long as those+ # two probes take, and they are the one thing here whose cost is proportional to the arm, so the+ # two arms' observers stayed open for materially different intervals (14.3 fps against 49.0 is+ # the same asymmetry expressed as time). `compare_visible` returns DIFFER on strict set+ # inequality of `ever_visible` and the null control cannot absorb it, being same-build against+ # same-build. Taking it first does not make the tail zero, since `capture()` still waits two+ # frames on purpose; it makes it the SAME ON BOTH ARMS.visible = self._visible()census = self._census()parity = self._parity()- # The observations DO consume wall clock before the next slot, so their cost is recorded- # rather than dropped -- it is simply recorded as theirs.+ # The observations DO consume wall clock before the next slot, so their cost is recorded as+ # theirs rather than dropped.observation_ms = (time.monotonic() - window_closed_at) * 1000row = result.row(slot.action, window_name, self.cell.cell_id)row["window_ms"] = window.duration_ms- # READ FROM THE LOCALS ABOVE, not from `window.notes`. The three observations were moved- # out of the measured window, so by the time this row is built they are values this method- # holds rather than notes the window carries.+ # READ FROM THE LOCALS ABOVE, not from `window.notes`: the three observations were moved out of+ # the measured window, so by now they are values this method holds.row["census"] = censusrow["parity"] = parityrow["visible"] = visible- # The observation cost itself, so it is never invisible again. `census_cost_ms` is the- # page's own timing of the walk; if this pair ever comes to dominate the film's wall clock- # the number is in the payload rather than in someone's hypothesis.+ # The observation cost itself, so it is never invisible again. `census_cost_ms` is the page's+ # own timing of the walk.row["observation_outside_window"] = Truerow["observation_ms"] = round(observation_ms, 1)- # THE SCREENSHOT IS TAKEN OUTSIDE THE WINDOW, on purpose and not as a tidiness point.- # The film runs on a wall clock and every slot has an absolute start, so an encode charged- # to the measured window eats the gap before the next slot -- which is how an action comes- # to report a MISSED SLOT on a contended runner. `--assert-liveness` counts those, so a- # camera inside the window could turn a healthy run red and it would look like the harness- # failing rather than like the instrument taxing itself. Out here it costs the gap, which- # is what the gap is for. The DOM has moved on by a few milliseconds; for a picture that- # is nothing, and for the digest, which was taken inside, it is not true at all.+ # THE SCREENSHOT IS TAKEN OUTSIDE THE WINDOW: the film runs on a wall clock with absolute slot+ # starts, so an encode charged to the measured window eats the gap before the next slot, which+ # is how an action comes to report a MISSED SLOT on a contended runner, and `--assert-liveness`+ # counts those. Out here it costs the gap, which is what the gap is for.if isinstance(row.get("parity"), dict) and row["parity"].get("parity_attempted"):row["parity"].update(self._parity_shot(slot.action))# An action that ran but overran its budget has pushed nothing (the next slot has its owndiff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py b/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.pyindex 51c0685a3f9..18b16849db5 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_action_bar_wait.py@@ -49,16 +49,15 @@ from studiobench.scene.actions import ACTION_BAR_WAIT_MS, MENU_JS # noqa: E402DOM_JS = Path(__file__).resolve().parents[1] / "dom.js"#: The shim mounts the bar this long after the run starts, standing in for a follow-up turn whose-#: last chunks land just after the slot opens. Comfortably inside `ACTION_BAR_WAIT_MS` and-#: comfortably outside anything a single sample could catch.+#: last chunks land just after the slot opens. Comfortably inside `ACTION_BAR_WAIT_MS` and outside+#: anything a single sample could catch.LATE_MOUNT_MS = 400-#: The mount deadline is armed by the action's own first look (see `HARNESS_JS`), so it can only-#: start level with or after `waitForActionButton`'s clock and `waitedMs` cannot undershoot-#: `LATE_MOUNT_MS` by any elapsed time at all. What is left is dom.js rounding `waitedMs` to a-#: tenth. This is an allowance for clock granularity, not a tolerance for load: the wait ends at-#: the first sample taken at or after the bar mounts, and a slow machine can only push that sample-#: LATER. Nothing here scales with `LATE_MOUNT_MS`.+#: The mount deadline is armed by the action's own first look, so it can only start level with or+#: after `waitForActionButton`'s clock and `waitedMs` cannot undershoot `LATE_MOUNT_MS` by any+#: elapsed time. What is left is dom.js rounding `waitedMs` to a tenth: an allowance for clock+#: granularity, not a tolerance for load.+# See `HARNESS_JS`.MOUNT_CLOCK_SLACK_MS = 1HARNESS_JS = r"""@@ -237,10 +236,9 @@ def test_a_control_that_arrives_late_is_waited_for_rather_than_reported_missing("""out = run_menu(LATE_MOUNT_MS)assert out["ran"] is True, out- # Still a liveness assertion, not a timing one: the bar was NOT there when the wait began, and- # the action stayed until it arrived. The bound is tight because the harness arms the mount- # deadline off the same clock and the same instant the action starts looking, so anything short- # of `LATE_MOUNT_MS` means the wait ended early -- a control that was there all along.+ # Still a liveness assertion, not a timing one: the bar was NOT there when the wait began and the+ # action stayed until it arrived. The bound is tight because the harness arms the mount deadline+ # off the same clock and instant, so anything short of `LATE_MOUNT_MS` means the wait ended early.assert out["waitedMs"] >= LATE_MOUNT_MS - MOUNT_CLOCK_SLACK_MS, outassert out["waitedMs"] < ACTION_BAR_WAIT_MS, out# The menu really opened and closed on the control that was waited for, so the wait produced a@@ -254,8 +252,8 @@ def test_the_wait_is_bounded_and_a_control_that_never_appears_still_reports_not_out = run_menu(float("inf"), wait_ms = 300)assert out["ran"] is False, outassert out["waitedMs"] >= 250, out- # The reason has to separate the two cases, because they are different bugs: a reply that had- # not settled, and a control that is not there at all.+ # The reason has to separate the two cases, because they are different bugs: a reply that had not+ # settled, and a control that is not there at all.assert "STILL GENERATING" in out["reason"], outassert out["running"] is True, out@@ -265,6 +263,6 @@ def test_a_settled_reply_costs_the_action_nothing():out = run_menu(0)assert out["ran"] is True, outassert out["waitedMs"] < 50, out- # Hovered before the control is read: the bar unmounts on every message that is not hovered,- # so a read without the hover is a read of a tree the control was never in.+ # Hovered before the control is read: the bar unmounts on every message that is not hovered, so a+ # read without the hover is a read of a tree the control was never in.assert out["hovers"] >= 1, outdiff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.pyindex a54c30d1bcc..20cfb4c7ea9 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py@@ -173,8 +173,8 @@ def test_a_failed_sentinel_write_does_not_re_admit_the_unchanged_clipboard(page)plausible non-empty copy of the whole thread with the 250ms settle beside it as `copy_ms`. Thatis the exact measurement the sentinel exists to refuse, re-admitted by its own fallback."""- # Whatever an earlier action in the film left behind. Non-empty and plausible, which is what- # makes it dangerous: an empty clipboard would have been caught by `clipboard_chars > 0`.+ # Whatever an earlier action in the film left behind. Non-empty and plausible, which is what makes+ # it dangerous: an empty clipboard would have been caught by `clipboard_chars > 0`.page.evaluate("async () => await navigator.clipboard.writeText('x'.repeat(2400))")_refuse_the_sentinel_write(page)_swallow_the_copy(page)@@ -224,10 +224,9 @@ def test_the_guard_is_on_the_clipboard_and_not_on_the_engine_name():src = inspect.getsource(A.select_all_copy)assert "sentinel" in src- # EXECUTABLE LINES ONLY. The engine names belong in the comment that explains which engine was- # observed failing and with what numbers; what must not exist is a BRANCH on one.+ # EXECUTABLE LINES ONLY. The engine names belong in the comment explaining which engine was+ # observed failing and with what numbers; what must not exist is a BRANCH on one. The word+ # "engine" is allowed because it appears in the refusal MESSAGE; a specific engine's NAME is not.code = "\n".join(line for line in src.splitlines() if not line.lstrip().startswith("#"))- # The word "engine" is allowed: it appears in the refusal MESSAGE, which is where it belongs.- # A specific engine's NAME is not, because that is what a branch would need.for name in ("webkit", "WebKit", "firefox", "Firefox", "browser_name", "browser_type"):assert name not in code, f"the guard branches on {name!r} rather than on the clipboard"diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.pyindex bc3377ad92b..fcec329cae4 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_follow_coverage_live.py@@ -39,14 +39,12 @@ if str(_STUDIO_TESTS) not in sys.path:_DOM_JS = _STUDIO_TESTS / "studiobench" / "scene" / "dom.js"-#: A running thread: the stop button is what `isRunning()` looks for. The viewport is scrollable so-#: "at the bottom" is a real question.-#:+#: A running thread: the stop button is what `isRunning()` looks for, and the viewport is scrollable+#: so "at the bottom" is a real question.#: The jump-to-bottom control is present and its `invisible` class is kept in sync with the scroll#: position by a listener, because that class IS the app's own answer to "are we at the bottom" and-#: `appSaysAtBottom()` reads nothing else. Omitting the control makes every sample `running_unknown`-#: and `pinned_fraction` null, which is correct behaviour on a build that renders no control and-#: useless as a fixture for scoring one.+#: `appSaysAtBottom()` reads nothing else. Omitting the control makes every sample+#: `running_unknown` and `pinned_fraction` null.FIXTURE = """<!doctype html><meta charset="utf-8"><style>@@ -230,23 +228,20 @@ def test_the_coverage_travels_with_the_verdict(page):)-# ── the run the user started is a fresh intent to be at the end ─────-#-# THE GESTURE ON THE REAL FILM NEVER ENDS AT THE BOTTOM. `scene/actions.py::SCROLL_JS` jumps to-# the bottom and then steps 14 x 420px away from it, so on any thread taller than 5,880px the-# gesture ends thousands of pixels up and the `resume()` re-attachment above cannot fire. The film-# then starts two more runs of its own -- `stop_generation` and `send_turn` both submit a turn ---# and the app pins to the bottom for them, which `runtime/session.py` already documents as-# intended rather than a violation.-#+# The run the user started is a fresh intent to be at the end. THE GESTURE ON THE REAL FILM NEVER+# ENDS AT THE BOTTOM: `SCROLL_JS` jumps to the bottom and then steps 14 x 420px away, so on any+# thread taller than 5,880px the gesture ends thousands of pixels up and the `resume()`+# re-attachment cannot fire. The film then starts two more runs of its own and the app pins to the+# bottom for them, which `runtime/session.py` documents as intended.+# `stop_generation` and `send_turn` both submit a turn.# Measured at head over every 100K payload in `outputs/`: attached_fraction_of_stream 0.07 to 0.15-# with reattachments 0, on the BASE arm as well as the treatment and on pure null controls, so the-# gate's FOLLOW_MIN_STREAM_COVERAGE of 0.50 failed every 100K cell of every run -- two copies of-# the shipped build included -- and a failed gate excludes the cell from scoring entirely. It-# passed only on the 1K smoke film, where the thread is short enough that the gesture's reversal-# lands back at the bottom by accident: a verdict about the thread's height, not about the app.+# with reattachments 0, on the BASE arm as well as the treatment and on pure null controls, so+# FOLLOW_MIN_STREAM_COVERAGE of 0.50 failed every 100K cell of every run and a failed gate excludes+# the cell from scoring. It passed only on the 1K smoke film, where the gesture's reversal lands+# back at the bottom by accident: a verdict about the thread's height, not the app.+# ── the run the user started is a fresh intent to be at the end ─────def _end_run(page) -> None:"""The reply finishes: the stop control goes, so `isRunning()` is false."""page.evaluate(diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.pyindex ebd51621f82..fc8e1f0e26e 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_hover_reveal_live.py@@ -44,9 +44,9 @@ if str(_STUDIO_TESTS) not in sys.path:from studiobench.runtime.types import ActionContext, Cell # noqa: E402from studiobench.scene import actions as A # noqa: E402-#: Unsloth's own geometry: a 20x20 action inside a 279x49 sticky header row, at the same offset the-#: live probe measured. The numbers matter -- a control large enough to be hit by chance would let-#: the test pass for the wrong reason.+#: Unsloth's own geometry: a 20x20 action inside a 279x49 sticky header row, at the offset the live+#: probe measured. The numbers matter: a control large enough to be hit by chance would let the test+#: pass for the wrong reason.FIXTURE = """<!doctype html><meta charset="utf-8"><style>@@ -78,8 +78,8 @@ FIXTURE = """SELECTOR = 'button[aria-label="New chat"]'-#: A URL that actually loads. Pointing the fallback at a dead port exercises the "navigation-#: failed too" branch instead of the navigation branch under test.+#: A URL that actually loads. Pointing the fallback at a dead port exercises the "navigation failed+#: too" branch instead of the navigation branch under test.FALLBACK_URL = "about:blank"@@ -108,8 +108,8 @@ def browser():@pytest.fixture()def page(browser):- # A PAGE PER TEST, because the mouse position is page state and a previous test's hover would- # carry the control into the next one already revealed.+ # A PAGE PER TEST, because the mouse position is page state and a previous test's hover would carry+ # the control into the next one already revealed.pg = browser.new_page(viewport = {"width": 1280, "height": 900})pg.set_content(FIXTURE)yield pg@@ -155,9 +155,9 @@ def test_hovering_reveals_the_control_and_returns_a_point_on_it(page):point = A._reveal_by_hover(_ctx(page), SELECTOR)assert point is not None, "hovering did not make the control hit-testable"x, y = point- # Against the control's OWN box, read from the page. Hard-coding the live app's (243, 319)- # asserted the fixture's layout rather than the behaviour, and failed on a fixture that was- # working correctly.+ # Against the control's OWN box, read from the page. Hard-coding the live app's (243, 319) asserted+ # the fixture's layout rather than the behaviour, and failed on a fixture that was working+ # correctly.box = page.eval_on_selector(SELECTOR, "(el) => el.getBoundingClientRect().toJSON()")assert box["left"] <= x <= box["right"], (point, box)assert box["top"] <= y <= box["bottom"], (point, box)@@ -192,10 +192,9 @@ def test_an_ordinary_control_is_not_hovered_first(page):the log rather than by asking `_reveal_by_hover` what it would have done."""page.eval_on_selector(SELECTOR,- # `transition: none` as well: `opacity` is transitioned over 150ms, so `getComputedStyle`- # sampled on the next tick still reports "0" and the control looks hidden to any check- # that reads it. That is a real hazard for the production code too, and the reason the- # reveal waits out `_REVEAL_SETTLE_MS` rather than sampling immediately.+ # `transition: none` as well: `opacity` is transitioned over 150ms, so `getComputedStyle` sampled+ # on the next tick still reports "0" and the control looks hidden to any check that reads it. A+ # real hazard for the production code too, and the reason the reveal waits out `_REVEAL_SETTLE_MS`."(el) => { el.style.transition = 'none'; el.style.opacity = '1';"" el.style.pointerEvents = 'auto'; }",)diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py b/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.pyindex 54b3d328f33..6a6a6569ab6 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_observation_cost.py@@ -156,14 +156,13 @@ class _Page:*_a,**_k,):- # The end of the thread, which `thread_reopen` reads out of the DOM before it touches- # anything so that it has a string to recognise the rebuilt thread by. It has to be a- # string here or the action refuses on that precondition and never reaches the transition- # this file is about.+ # The end of the thread, which `thread_reopen` reads out of the DOM before it touches anything so+ # it has a string to recognise the rebuilt thread by. It has to be a string here or the action+ # refuses on that precondition.if "data-role=" in script and "user" in script:return "studiobench turn 8: continue with unit 3"- # Otherwise a thread with messages in it, so `thread_reopen` gets past its own precondition- # and reaches the transition under test.+ # Otherwise a thread with messages in it, so `thread_reopen` gets past its own precondition and+ # reaches the transition under test.return 18def wait_for_timeout(self, _ms):@@ -244,9 +243,9 @@ class _RunningPage:return 18def wait_for_timeout(self, ms):- # Actually sleeps, like Playwright's. A no-op here lets the bounded wait spin thousands of- # times inside its deadline, which exhausted the scripted reply and made a test about a- # reply that never lands pass for the opposite reason.+ # Actually sleeps, like Playwright's. A no-op lets the bounded wait spin thousands of times inside+ # its deadline, which exhausted the scripted reply and made a test about a reply that never lands+ # pass for the opposite reason.self.waits += 1time.sleep(ms / 1000.0)diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py b/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.pyindex 6c73c7ec036..faceb7fd33c 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_parity_shot_hook.py@@ -75,8 +75,8 @@ def test_no_shot_is_taken_when_none_was_asked_for():def test_the_digest_call_never_takes_a_picture():- # The digest is taken INSIDE the measured window and the shot outside it, so `_parity` must- # not be a camera. If it becomes one again, the encode is charged to the film's own clock.+ # The digest is taken INSIDE the measured window and the shot outside it, so `_parity` must not be a+ # camera; if it becomes one again, the encode is charged to the film's own clock.page = StubPage()got = runner(page, parity_shots = "/nonexistent", arm_label = "base")._parity()assert page.shots == []@@ -84,8 +84,8 @@ def test_the_digest_call_never_takes_a_picture():def test_the_shot_is_named_for_its_cell_action_and_arm(tmp_path):- # The arm has to be IN the name. Both arms share a fixture, a film and a password, so a file- # that says only "settings" cannot be attributed to a side once it leaves this directory.+ # The arm has to be IN the name. Both arms share a fixture, a film and a password, so a file that+ # says only "settings" cannot be attributed to a side once it leaves this directory.page = StubPage(scroll = 940)got = runner(page, parity_shots = str(tmp_path), arm_label = "treatment")._parity_shot("settings")assert got["shot"] == "r100K.treatment.rep1__settings__treatment.png"@@ -102,8 +102,8 @@ def test_the_two_arms_do_not_collide_on_one_filename(tmp_path):def test_a_camera_failure_does_not_cost_the_measurement(tmp_path):- # The digest is the reading; the picture is evidence about it. Losing the second must not- # discard the first, or a flaky screenshot turns a green run red for no reason.+ # The digest is the reading; the picture is evidence about it. Losing the second must not discard+ # the first, or a flaky screenshot turns a green run red for no reason.page = StubPage(shot_raises = True)got = runner(page, parity_shots = str(tmp_path), arm_label = "base")._parity_shot("settings")assert "shot" not in gotdiff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.pyindex 0320ca08c28..9fe2ff7bcd6 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py@@ -67,7 +67,7 @@ _QUEUE_BUTTON_RUNNING = """</div>"""-#: The queued branch: `isQueueRunning && !thread.isRunning`, with the active item undispatched.+#:The queued branch: `isQueueRunning && !thread.isRunning`, with the active item undispatched._QUEUE_BUTTON_IDLE = """<button class="aui-composer-send ml-1.5 size-9 rounded-full" aria-label="Queue message"><span class="aui-sr-only">Queue message</span>@@ -81,10 +81,10 @@ _STOP_BUTTON = """"""#: The DISPATCHED queued branch: `isQueueRunning && !thread.isRunning` with `queueEntry.dispatched`,-#: which renders a stop control the thread does not report itself running behind. Neither-#: `stopButton()` nor `queueButton()` matches it, so on its own it reads exactly like a settled-#: composer. `getPromptQueueUIItemsForRun` drops dispatched items, so the queue surface can be gone-#: here too -- which is why this state has to be recognised from its own control.+#: rendering a stop control the thread does not report itself running behind. Neither+#: `stopButton()` nor `queueButton()` matches it, so on its own it reads like a settled composer,+#: and `getPromptQueueUIItemsForRun` drops dispatched items so the queue surface can be gone too,+#: which is why this state must be recognised from its own control._STOP_QUEUED_BUTTON = """<button class="aui-composer-cancel ml-1.5 size-9 rounded-full" aria-label="Stop queued message"><span class="aui-sr-only">Stop queued message</span>@@ -92,8 +92,8 @@ _STOP_QUEUED_BUTTON = """"""#: `PromptQueueStack`, which renders inside the composer root whenever the run has an item left to-#: show. An undispatched active item is always one of them, so this surface is up in every state-#: that renders the queued-idle Queue button.+#: show. An undispatched active item is always one, so this surface is up in every state that+#: renders the queued-idle Queue button._QUEUE_STACK = """<div aria-label="Prompt queue, 1 of 2"><div>a prompt that has not been dispatched</div>@@ -155,9 +155,9 @@ BLIND = dict(queue_stack = False,statuses = [None, None, None, None],)-#: The cost of reading the queue surface, pinned rather than left for a later reader to find: a-#: queue run with a prompt still waiting, a reply genuinely streaming, and text in the composer.-#: The surface is up and the Queue button is the only control, so the control is not armed.+#: The cost of reading the queue surface, pinned rather than left for a later reader: a queue run+#: with a prompt still waiting, a reply genuinely streaming, and text in the composer. The surface+#: is up and the Queue button is the only control, so the control is not armed.QUEUED_AND_STREAMING_BLIND = dict(control = _QUEUE_BUTTON_RUNNING,queue_stack = True,@@ -202,7 +202,7 @@ def _capture() -> dict:page.set_content(_page(tail = tail, **state))# After the content, not before it: `set_content` does not reliably run init scripts, and the- # symptom is `window.__sb` simply not existing, which reads like a broken instrument.+ # symptom is `window.__sb` simply not existing.page.add_script_tag(content = _DOM_JS.read_text(encoding = "utf-8"))page.add_script_tag(content = _PARITY_JS.read_text(encoding = "utf-8"))got = page.evaluate("() => window.__sb.parity.capture()")@@ -298,14 +298,14 @@ def test_the_shipped_composer_still_renders_the_two_queue_buttons():assert 'aria-label="Stop queued message"' in src+#: An overlay is walked from `document`, OUTSIDE `.aui-thread-root`, so its digest carries neither+#: the streamed message nor the composer, which makes it readable on a pair whose stream could not+#: be placed.# ── what the blind-probe refusal may NOT take out with it ────────────-#: An overlay is walked from `document`, OUTSIDE `.aui-thread-root`. Its digest therefore carries-#: neither the streamed message nor the composer, which is what makes it readable on a pair whose-#: stream could not be placed._MENU = '<div role="menu"><div class="item">Rename</div></div>'_MENU_CHANGED = '<div role="menu"><div class="item">Rename thread</div></div>'-#: The composer of a thread that is NOT generating. `_STOP_BUTTON` is the same composer generating.+#:The composer of a thread that is NOT generating. `_STOP_BUTTON` is the same composer generating._SEND_BUTTON = ('<button class="aui-composer-send" aria-label="Send message">''<span class="aui-sr-only">Send message</span></button>'@@ -396,12 +396,12 @@ def test_the_scaffold_is_not_an_independent_surface_and_here_is_why(page):)-# ── the composer is not a rendering difference ───────────────────────-#-# The pair this whole change is about: one arm has finished its reply, the other is still writing-# it. Its messages are withheld correctly. Its COMPOSER was not, because the dock is inside-# `.aui-thread-root` and the scaffold therefore carries Stop on one arm and Send on the other.+# The composer is not a rendering difference. The pair this change is about: one arm has finished+# its reply, the other is still writing it. Its messages are withheld correctly; its COMPOSER was+# not, because the dock is inside `.aui-thread-root` and the scaffold therefore carries Stop on+# one arm and Send on the other.+# ── the composer is not a rendering difference ───────────────────────_SETTLED_STATUSES = ["complete", "complete", "complete", "complete"]_STREAMING_STATUSES = ["complete", "complete", "complete", "running"]@@ -613,16 +613,14 @@ def test_the_queued_idle_arm_against_a_settled_one_is_still_refused(page):assert P.compare(base, treat)["verdict"] == P.NOT_COMPARABLE-# ── the style probe walks the run-state control too ──────────────────-#-# `report` now collects the style verdict BEFORE it buckets a structural refusal, because the-# computed-style probe is an independent reading and the refusal is not about it. That makes the-# probe's own reading of the composer swap visible for the first time, and the probe walks-# `button[aria-label="Send message"]` and `button[aria-label="Stop generating"]` as SEPARATE-# selectors whose names go into its signature. So the pairs the refusals above exist to withhold-# arrived at the advisory line instead.+# The style probe walks the run-state control too. `report` now collects the style verdict BEFORE+# it buckets a structural refusal, because the computed-style probe is an independent reading,+# and that makes its own reading of the composer swap visible for the first time: it walks the+# Send and Stop buttons as SEPARATE selectors whose names go into its signature, so the pairs the+# refusals exist to withhold arrived at the advisory line instead.+# ── the style probe walks the run-state control too ──────────────────def _capture_html_raw(page, html: str) -> dict:"""`_capture_html`, keeping `styles.sig` so a test can say WHY the digest moved."""page.set_content(html)diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.pyindex 7557cc10d09..9b7fca78677 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_one_live.py@@ -135,7 +135,7 @@ def test_it_opens_exactly_one_pane_and_leaves_the_rest_shut(make_page):assert got.expect_ok is True, got.reasonassert got.expect["open_after_expand"] == 1assert got.expect["open_after_collapse"] == 0- # The thread's pane count is reported as CONTEXT beside a fixed gesture. Without it the reading+ # The thread's pane count is reported as CONTEXT beside a fixed gesture; without it the reading# cannot be told apart from the thread-wide one in a payload.assert got.expect["panes"] == 8assert got.expect["panes_opened"] == 1diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.pyindex b3cbce9c544..cf01d0a4144 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_reasoning_settle_js.py@@ -208,8 +208,8 @@ def test_a_slow_state_flip_does_not_bank_quiet_frames_before_it():)assert out["spansOpen"] == SPANS_SETTLEDassert out["openCensored"] is False- # The flip and the settled read are different moments, and both are reported so a reader can- # see how much of open_ms was spent after the state arrived.+ # The flip and the settled read are different moments, and both are reported so a reader can see+ # how much of open_ms was spent after the state arrived.assert out["openStateReachedMs"] < out["openMs"]@@ -307,16 +307,16 @@ def test_the_state_reached_mark_shares_the_timing_origin():assert out["openStateReachedMs"] <= out["openMs"]-# ── the close direction ─────────────────────────────────────────────────────────────────────────-#-# The same defect the quiet streak fixes for the open direction, arriving from the other side.-# `data-state` flips on the click, and BOTH collapse mechanisms keep the children in the document-# until the exit animation ends: Radix's `Presence` until `animationend`, the grid arm's-# `UnmeasuredCollapsibleContent` until the `grid-template-rows` `transitionend` or its 250 ms-# backstop. So `pre span` is frozen at its open value for that whole window, and a streak that only-# asks whether the census has stopped moving is satisfied by a census that has not started.+# The close direction: the same defect the quiet streak fixes for the open direction, from the+# other side. `data-state` flips on the click and BOTH collapse mechanisms keep the children in+# the document until the exit animation ends (Radix's `Presence` until `animationend`, the grid+# arm until `transitionend` or its 250 ms backstop) so `pre span` is frozen at its open value and+# a streak that only asks whether the census has stopped moving is satisfied by one that has not+# started.+# The grid arm is `UnmeasuredCollapsibleContent`.+# ── the close direction ─────────────────────────────────────────────────────────────────────────def test_a_collapse_is_not_settled_while_its_panes_are_still_mounted():"""THE REGRESSION. `close_ms` must not name the state flip plus four frames.diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py b/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.pyindex 3d09ca0f419..455a38233c9 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py@@ -72,18 +72,18 @@ from studiobench.scene.actions import ( # noqa: E402)from studiobench.scene.schedule import FAST, QUICK, STANDARD # noqa: E402-#: One driver call over CDP. Measured at 43 - 130 ms across nine to thirteen `page.evaluate` calls-#: in one run of the shipped action against real chromium, and the two big ones in that total are-#: page-side waits charged separately below, so about 4 ms is the round trip itself.+#: One driver call over CDP. Measured at 43 - 130 ms across nine to thirteen `page.evaluate` calls in+#: one run of the shipped action against real chromium, and the two big ones are page-side waits+#: charged separately below, so about 4 ms is the round trip itself.ROUND_TRIP_MS = 4.0-#: Enter pressed to `isRunning()` answering true: a POST to the relay and the first SSE frame back.+#:Enter pressed to `isRunning()` answering true: a POST to the relay and the first SSE frame back.START_MS = 120.0-#: Stop clicked to `isRunning()` answering false: the abort round trip.+#:Stop clicked to `isRunning()` answering false: the abort round trip.STOP_MS = 90.0-#: The delete inside `STOP_CLEANUP_JS`: the click, the removal, and the two `__sbNextPaint` frames+#:The delete inside `STOP_CLEANUP_JS`: the click, the removal, and the two `__sbNextPaint` frames the loop waits on.#: the loop waits on.CLEANUP_MS = 90.0@@ -114,10 +114,9 @@ class _Keyboard:if not self._page.accepts_send:# `queueDisabled` in thread.tsx: the press queued nothing and the box keeps its text.return- # SENT, NOT STARTED, and the difference is the whole of the P1 above. The app takes the- # text out of the composer and puts the user turn and its reply into the thread NOW; the- # reply begins generating `start_ms` later, and the action has to poll for it -- which is- # the cost the first version of this shim handed out free.+ # SENT, NOT STARTED, and the difference is the whole of the P1 above: the app takes the text out of+ # the composer and puts the user turn and its reply into the thread NOW, while the reply begins+ # generating `start_ms` later and the action has to poll for it.self._page.composer = ""self._page.messages += 2self._page.sent_at_ms = self._page.elapsed_ms@@ -210,10 +209,9 @@ class _Page:return self.composerif "messageCount" in script:return self.messages- # The thread's length as well as the mounted count. `stop_generation` proves its own turn- # was added by `threadTotal()`, so that a windowed arm whose window refills is not read as- # a send that added nothing and left with nothing to clean up. This page models a fully- # mounted arm, where the two are the same number.+ # The thread's length as well as the mounted count: `stop_generation` proves its own turn was added+ # by `threadTotal()`, so a windowed arm whose window refills is not read as a send that added+ # nothing. This page models a fully mounted arm, where the two are the same number.if "threadTotal" in script:return self.messagesif "assistantChars" in script:@@ -294,8 +292,8 @@ def test_a_reply_that_drains_at_the_end_of_the_slot_does_not_spend_the_next_one(f"{scene.name}: stop_generation spent {page.elapsed_ms:.0f}ms of a {stop.budget_ms}ms "f"slot with only {slack_ms}ms before {nxt.action} opens")- # And it stopped by declining, not by stopping the cell's own reply, which is the whole point- # of the wait it just gave up on.+ # And it stopped by declining, not by stopping the cell's own reply, which is the whole point of+ # the wait it just gave up on.assert result.ran is Falseassert page.clicked == 0assert "one more" not in page.filled@@ -401,8 +399,8 @@ def test_a_turn_that_starts_after_the_slot_bound_is_not_left_generating(stop, _nxt, _slack = _stop_slot(scene)settled = _thread_a_measured_turn_leaves(monkeypatch, stop.budget_ms)- # Past the slot bound, which is the budget less what the rest of the turn costs, and still- # inside the time the turn is worth waiting for so it can be taken back.+ # Past the slot bound, which is the budget less what the rest of the turn costs, and still inside+ # the time the turn is worth waiting for so it can be taken back.start_ms = stop.budget_ms - 1_000 + late_by_msresult, page = _run(@@ -586,8 +584,8 @@ def test_the_reserve_is_still_the_whole_of_what_its_two_halves_reserve():assert OWN_TURN_POLL_MS == OWN_TURN_START_POLL_MS + OWN_TURN_STOP_POLL_MS# The 80 ms that settles the fill is the only fixed sleep before the send.assert OWN_TURN_FIXED_MS - OWN_TURN_FIXED_AFTER_SEND_MS == 80- # And what the turn-start wait holds back has to leave the start poll something to spend, or- # the wait is an immediate refusal and the throwaway turn is unreachable on the tightest drain.+ # And what the turn-start wait holds back has to leave the start poll something to spend, or the+ # wait is an immediate refusal and the throwaway turn is unreachable on the tightest drain.assert (OWN_TURN_RESERVE_MS - 80 - OWN_TURN_FIXED_AFTER_SEND_MS - OWN_TURN_STOP_POLL_MS== OWN_TURN_START_POLL_MSdiff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py b/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.pyindex 5d22d8749b4..6cab4b7ff25 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_thread_reopen_gate.py@@ -46,7 +46,7 @@ BASE_URL = "http://127.0.0.1:1"THREAD_URL = f"{BASE_URL}/chat?thread=t1"NEW_CHAT_URL = f"{BASE_URL}/chat?new=studiobench"-#: The length of the seeded thread, and the marker the seeder wrote into its last user turn.+#:The length of the seeded thread, and the marker the seeder wrote into its last user turn.TOTAL = 18MARKER = "studiobench turn 8: continue with unit 3"@@ -80,13 +80,13 @@ REBUILD = ()#: A rebuild that never arrives: the store keeps declaring eighteen messages and the thread stops-#: at three of them, settled, with the end of the conversation nowhere on screen. The exact shape-#: the old condition scored as a fast, successful re-open.+#: at three, settled, with the end of the conversation nowhere on screen. The exact shape the old+#: condition scored as a fast, successful re-open.STALLED = (_Frame(mounted = 3, elements = 1_200, scroll_height = 2_000, spans = 0, marker = False),)-#: A correctly windowed rebuild: a window of six rows out of eighteen, anchored at the end, with-#: the last user turn among them. `full` conditions would refuse this forever, which is why the-#: action picks its mode from the mount it left rather than assuming one.+#: A correctly windowed rebuild: six rows out of eighteen, anchored at the end, with the last user+#: turn among them. `full` conditions would refuse this forever, which is why the action picks its+#: mode from the mount it left.WINDOWED = (_Frame(mounted = 2, elements = 900, scroll_height = 11_800, spans = 0, marker = False),_Frame(mounted = 6, elements = 3_400, scroll_height = 12_400, spans = 2_100, marker = True),@@ -120,8 +120,6 @@ class _ThreadPage:self.goto_calls: list[str] = []self.probes = 0- # ── what the page would report ──────────────────────────────────-@propertydef frame(self) -> _Frame:return self.frames[min(self.step, len(self.frames) - 1)]@@ -169,8 +167,6 @@ class _ThreadPage:"pinning": False,}- # ── the Playwright surface the action uses ──────────────────────-def evaluate(self,script,@@ -186,9 +182,9 @@ class _ThreadPage:return MARKER if self.phase != "gone" else Noneif "pre span" in script:return self.frame.spans if self.phase == "rebuild" else 0- # The hit-test spread and the hover target, both of which report "no reachable point" for- # an unclickable control. Returning None here is what sends `_click_or_navigate` down the- # branch under test rather than into an off-centre click.+ # The hit-test spread and the hover target, both of which report "no reachable point" for an+ # unclickable control. Returning None sends `_click_or_navigate` down the branch under test+ # rather than into an off-centre click.return Nonedef query_selector(self, selector):@@ -208,8 +204,8 @@ class _ThreadPage:def wait_for_timeout(self, _ms):# A REAL, TINY SLEEP. The readiness wait is bounded on the monotonic clock, so a poll that- # returned instantly would spin thousands of times inside the timeout instead of walking- # the scripted frames.+ # returned instantly would spin thousands of times inside the timeout instead of walking the+ # scripted frames.time.sleep(0.002)if self.phase == "rebuild":self.step += 1@@ -425,12 +421,12 @@ def test_a_thread_whose_end_cannot_be_identified_is_refused_before_it_is_touchedassert page.phase == "thread" and page.goto_calls == []+#: How long the centre click burns before it gives up. `_click_or_navigate` passes `timeout = 2000`+#: to Playwright and the hit-target check retries for the whole of it against a control it cannot+#: hit at the centre. Scaled down here; what is asserted is that the timings do not contain it, at+#: whatever size.# ── defect three: the harness's own retry, billed to the rebuild ────-#: How long the centre click burns before it gives up. `_click_or_navigate` passes-#: `timeout = 2000` to Playwright, and the hit-target check retries for the whole of it against a-#: control it cannot hit at the centre. Scaled down here so the test is fast; what is asserted is-#: that the timings do not contain it, at whatever size it is.RETRY_MS = 400diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.pyindex 803ad9528ed..dac07a4b42b 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py@@ -70,34 +70,33 @@ def _page(rows = []for i in range(1, 5):if drop_tail and i == 4:- # The windowed case: the arm still DECLARES four messages through `aria-setsize`, and- # has unmounted the one it is writing into. That is a thread the instrument can see- # three quarters of, not an instrument that has stopped working.+ # The windowed case: the arm still DECLARES four messages through `aria-setsize` and has unmounted+ # the one it is writing into. That is a thread the instrument can see three quarters of, not an+ # instrument that has stopped working.continuerole = "user" if i % 2 else "assistant"last = i == 4if last:- # `live_role` is what the arm CALLS the row it is writing into. The shipped build says- # assistant; a build that says otherwise is the regression ordinal-role parity exists- # to catch, and it is asked here rather than assumed.+ # `live_role` is what the arm CALLS the row it is writing into. The shipped build says assistant; a+ # build that says otherwise is the regression ordinal-role parity exists to catch, and it is asked+ # here rather than assumed.role = live_rolebody = tail if last else f"message {i}"if i == 3:body = user_bodyif last and not tail_has_parts:- # The gap between the send being accepted and the reply's first part arriving. The- # assistant message is mounted and publishes nothing, because it has nothing to- # publish: thread.tsx renders "Generating..." in place of any part.+ # The gap between the send being accepted and the reply's first part arriving. The assistant+ # message is mounted and publishes nothing, because thread.tsx renders "Generating..." in place of+ # any part.rows.append(f'<div class="row" aria-posinset="{i}" aria-setsize="4">'f'<div data-role="{role}"><span>Generating...</span></div></div>')continuestatus = running_value if (last and tail_running) else "complete"- # THE HOOK IS RENAMED ON EVERY MESSAGE, not only the streamed one. `data-status` is a single- # line in markdown-text.tsx and it is rendered for `complete` parts as well as `running`- # ones, so a build that renames it renames it everywhere. A fixture that renamed it on one- # message would be describing a build that does not exist, and would then be the only+ # THE HOOK IS RENAMED ON EVERY MESSAGE, not only the streamed one: `data-status` is a single line+ # in markdown-text.tsx rendered for `complete` parts as well as `running` ones. A fixture that+ # renamed it on one message would describe a build that does not exist, and would then be the only# evidence that the control fires.attr = f'{hook}="{status}"'rows.append(@@ -167,14 +166,13 @@ def _capture(browser, **kw) -> dict:page.close()-#: The base arm: an ordinary settled thread on the shipped build.+#:The base arm: an ordinary settled thread on the shipped build._SETTLED = dict(tail = "the whole reply, arrived", tail_running = False, generating = False)-#: THE TREATMENT THAT WENT BLIND, and the shape matters. `data-status` is one line in-#: markdown-text.tsx and it is rendered for `complete` parts as well as `running` ones, so a build-#: that renames the ATTRIBUTE renames it on every message and every settled row differs too -- which-#: is a rendering difference in its own right and is reported as one. The interesting blindness is a-#: build that changed the status VOCABULARY: `complete` still reads `complete`, so every settled row-#: is byte-identical, and the only thing lost is the ability to see that a reply is being written.+#: THE TREATMENT THAT WENT BLIND, and the shape matters. A build that renames the ATTRIBUTE renames+#: it on every message, so every settled row differs too, which is a rendering difference in its+#: own right. The interesting blindness is a build that changed the status VOCABULARY: `complete`+#: still reads `complete`, so every settled row is byte-identical and the only thing lost is the+#: ability to see that a reply is being written._BLIND = dict(tail = "the whole reply, arr", running_value = "streaming", tail_running = True, generating = True)@@ -183,7 +181,7 @@ _BLIND = dict(_BLIND_ATTR = dict(tail = "the whole reply, arr", hook = "data-state", tail_running = True, generating = True)-#: The same mid-stream moment on a build whose hook IS known. Already handled, as residue.+#:The same mid-stream moment on a build whose hook IS known. Already handled, as residue._MIDSTREAM = dict(tail = "the whole reply, arr", tail_running = True, generating = True)@@ -254,16 +252,14 @@ def test_a_lost_conversation_is_still_a_finding_while_a_reply_runs(browser):assert "DIFFERENT MESSAGES on screen" in got["reason"]-# ── what the refusal may NOT take out with it ────────────────────────-#-# The blind-probe refusal is about rows whose meaning depends on where the stream had got to. Two-# kinds of row in this payload provably do not: a row both arms call the user's, because a reply is-# written into an assistant message; and a row whose ROLE changed, because a role is captured beside-# the digest and how far a reply has arrived says nothing about whose it is. Both used to leave here-# as NOT COMPARABLE with an empty `moved`, and `visible_report` buckets a refusal as blind and never-# consults it for the exit code, so the run went green on them.+# What the refusal may NOT take out with it. The blind-probe refusal is about rows whose meaning+# depends on where the stream had got to; two kinds provably do not: a row both arms call the+# user's, and a row whose ROLE changed. Both used to leave here as NOT COMPARABLE with an empty+# `moved`, and `visible_report` buckets a refusal as blind and never consults it for the exit+# code, so the run went green on them.+# ── what the refusal may NOT take out with it ────────────────────────def test_a_changed_user_row_survives_the_blind_refusal(browser):base = _capture(browser, **_SETTLED)treat = _capture(browser, **dict(_BLIND, user_body = "the user message, rewritten"))diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.pyindex f238343509d..73b84cd9579 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_capture_live.py@@ -40,9 +40,9 @@ if str(_STUDIO_TESTS) not in sys.path:_DOM_JS = _STUDIO_TESTS / "studiobench" / "scene" / "dom.js"_PARITY_JS = _STUDIO_TESTS / "studiobench" / "scene" / "parity.js"-#: A thread of tall messages in a short viewport, so only a few are ever on screen at once. Each-#: message publishes `aria-posinset`, which is how a windowed arm states thread position; the-#: capture keys on it so a window and a full mount are comparable.+#: A thread of tall messages in a short viewport, so only a few are ever on screen. Each message+#: publishes `aria-posinset`, which is how a windowed arm states thread position, and the capture+#: keys on it so a window and a full mount are comparable.FIXTURE = """<!doctype html><meta charset="utf-8"><style>@@ -108,7 +108,7 @@ def page(browser):pg.set_content(FIXTURE)# `add_script_tag` after the content, not `add_init_script` before it: Playwright's# `set_content` does not always run init scripts, and the symptom is `window.__sb` simply not- # existing, which reads like a broken instrument rather than a mis-ordered fixture.+ # existing, which reads like a broken instrument.pg.add_script_tag(content = _DOM_JS.read_text(encoding = "utf-8"))pg.add_script_tag(content = _PARITY_JS.read_text(encoding = "utf-8"))yield pg@@ -119,7 +119,7 @@ def _watch(page) -> None:got = page.evaluate("() => window.__sb.parityVisible.watch()")assert got.get("visible_attempted") is True, got# IntersectionObserver's first delivery is asynchronous, so give it a frame before anything- # scrolls. Without this the initial viewport contents are attributed to whatever came next.+ # scrolls, or the initial viewport contents are attributed to whatever came next.page.wait_for_timeout(120)@@ -306,20 +306,17 @@ def test_a_row_mounted_during_the_action_is_still_picked_up_cheaply(page):assert 3 in got["ever_visible"], got["ever_visible"]-# ── where the virtualizer publishes its ordinals is not a UI change ──-#-# `runtime/readiness.py` accepts `aria-posinset` / `aria-setsize` on the `[data-role]` message OR on-# an ancestor row wrapper -- it walks with `closest()`, because the ordinal belongs on whichever-# element is the member of the set, and refusing the first option would refuse a correctly-# implemented arm for putting the attribute in a place the gate itself calls right.-#-# The visible-region digest then read every attribute on the message, so an arm that took that-# option differed from the fully mounted arm on EVERY message -- which publishes neither attribute-# anywhere -- while the rendered content was identical. Auto-mode parity was unusable for a DOM-# shape the gate explicitly permits, and a wall of differences that are all the same non-finding-# buries anything real underneath it.+# Where the virtualizer publishes its ordinals is not a UI change. `runtime/readiness.py` accepts+# `aria-posinset` / `aria-setsize` on the `[data-role]` message OR on an ancestor row wrapper,+# walking with `closest()`, because the ordinal belongs on whichever element is the member of+# the set.+# The visible-region digest then read every attribute on the message, so an arm taking that+# option differed from the fully mounted arm on EVERY message while the rendered content was+# identical. Auto-mode parity was unusable for a DOM shape the gate explicitly permits, and a+# wall of identical non-findings buries anything real.+# ── where the virtualizer publishes its ordinals is not a UI change ──def _arm_html(ordinals: str, suffix: str = "") -> str:"""One thread of twenty messages, with the virtualization ordinals published `on_the_message`,`on_the_row` wrapper, or `nowhere` -- which is what the shipped build does."""@@ -423,20 +420,17 @@ def test_a_real_rendering_difference_is_still_caught(browser):assert verdict["verdict"] == P.DIFFER, verdict-# ── a rebuilt row is at its own position, not at the end of a lifetime count ──-#-# `thread_reopen` leaves the thread and comes back, and a FULLY MOUNTED arm answers that by-# removing every message row and creating a new one for every message inside the same document.-# Those rebuilt rows publish no `aria-posinset` -- only a windowed arm publishes one -- so their-# ordinal comes from the fallback, and the fallback used to be a LIFETIME counter of observed-# nodes. It already stood at N, so the rebuilt rows were stamped N+1..2N while the windowed arm on-# the other side of the A/B stamped its real 1..N. `compare_visible` compares the two sets of-# ordinals first, found them disjoint, and reported "the two arms put DIFFERENT MESSAGES on-# screen" -- a hard visible-difference verdict -- for a rebuild that was identical.+# A rebuilt row is at its own position, not at the end of a lifetime count. `thread_reopen`+# leaves the thread and comes back, and a FULLY MOUNTED arm removes every message row and+# creates a new one inside the same document. Those rebuilt rows publish no `aria-posinset`, so+# their ordinal came from a LIFETIME counter already standing at N: they were stamped N+1..2N+# while the windowed arm stamped its real 1..N, so `compare_visible` found the two ordinal sets+# disjoint and reported a hard visible difference for a rebuild that was identical.#: The shipped shape: every message mounted, and NOTHING publishing a virtualization ordinal.#: `__rebuild()` is the thread_reopen rebuild, in one commit, in the same document.+# ── a rebuilt row is at its own position, not at the end of a lifetime count ──REBUILD_FIXTURE = """<!doctype html><meta charset="utf-8"><style>@@ -606,15 +600,13 @@ def test_the_structural_digest_still_sees_the_ordinals(browser):assert all(numbered[i] != plain[i] for i in numbered), (numbered, plain)+#: An EMPTY viewport, so a row appended to it is on screen immediately. `__churn` mounts and+#: unmounts a row inside ONE task, which is how a row comes to be observed while detached: the+#: MutationObserver batch runs after both operations, so the node is in `addedNodes` and no+#: longer among the document's messages and has no position the instrument can honestly claim.+#: `__remount` hands THE SAME NODE back, as a virtualizer that recycles rows does.# ── a recycled node, and the ordinal it used to be denied ──────────────────--#: An EMPTY viewport, so a row appended to it is on screen immediately. `__churn` mounts a row and-#: unmounts it inside ONE task, which is how a row comes to be observed while detached: the-#: MutationObserver batch runs after both operations, the node is in `addedNodes` and is no longer-#: among the document's messages, so it has no position the instrument can honestly claim.-#: `__remount` then hands THE SAME NODE back, which is what a virtualizer that recycles its rows-#: does on the next scroll step.RECYCLE_FIXTURE = """<!doctype html><meta charset="utf-8"><style>@@ -674,9 +666,9 @@ def test_a_recycled_row_is_placed_when_it_finally_mounts(browser):assert got["ever_visible"] == [1], got-#: A row that is RENUMBERED IN PLACE. It stays connected, stays intersecting, and is handed to-#: another message -- which is what a virtualizer that recycles its row nodes does, and it is not a-#: childList mutation, so nothing about it reaches a childList-only observer.+#: A row that is RENUMBERED IN PLACE: it stays connected, stays intersecting, and is handed to+#: another message, as a virtualizer that recycles row nodes does. It is not a childList+#: mutation, so nothing about it reaches a childList-only observer.RENUMBER_FIXTURE = """<!doctype html><meta charset="utf-8"><style>@@ -728,7 +720,7 @@ def test_a_row_renumbered_in_place_is_restamped_and_reported(browser):assert "42" in got["messages"], got["messages"]assert got["unplaced_rows"] == 0, got# A LEGITIMATE RENUMBER IS NOT A COLLISION. One node holds one position at a time here, so the- # collision counter added for the two-rows-one-ordinal case must not fire on this, or a correct+ # collision counter added for the two-rows-one-ordinal case must not fire, or a correct# recycling virtualizer would be refused on every action it recycles a row in.assert got["ordinal_collisions"] == 0, got@@ -793,8 +785,8 @@ def test_two_rows_sharing_a_thread_position_are_counted_rather_than_overwritten(got = _capture_with_ghost(browser, before = before)assert got["ordinal_collisions"] == 1, (before, got)assert got["collided_ordinals"] == [1], (before, got)- # The counter is not derived from the arithmetic, and this is why: two of the three rows- # on screen are filed under one key, so the map still holds two entries.+ # The counter is not derived from the arithmetic, and this is why: two of the three rows on+ # screen are filed under one key, so the map still holds two entries.assert set(got["messages"]) == {"1", "2"}, got["messages"]@@ -834,8 +826,8 @@ def test_losing_the_thread_outranks_the_collision_refusal(browser):ghosted = _capture_with_ghost(browser, before = True)assert ghosted["ordinal_collisions"] == 1, ghosted- # The other arm ended with nothing on screen at all, which is the 100K `model_change` shape:- # 12 mounted messages to 0, never recovered.+ # The other arm ended with nothing on screen at all, the 100K `model_change` shape: 12 mounted+ # messages to 0, never recovered.empty = dict(ghosted)empty["messages"] = {}empty["ordinal_collisions"] = 0diff --git a/tests/studio/studiobench/scene/surface_sweep.py b/tests/studio/studiobench/scene/surface_sweep.pyindex 2fab49e6ba0..d925dc4d938 100644--- a/tests/studio/studiobench/scene/surface_sweep.py+++ b/tests/studio/studiobench/scene/surface_sweep.py@@ -29,14 +29,14 @@ from typing import Any, Callable, Optionalfrom . import surfaces as registry-#: How long a settle condition is given before the surface is recorded as unreached. Generous:-#: the settings panels are lazily imported chunks and the hub's first paint waits on a network-#: round trip, and a sweep that calls those failures is a sweep nobody believes.+#: How long a settle condition is given before the surface is recorded as unreached. Generous: the+#: settings panels are lazily imported chunks and the hub's first paint waits on a network round+#: trip, and a sweep that calls those failures is a sweep nobody believes.SETTLE_TIMEOUT_MS = 8000SETTLE_POLL_MS = 150-#: A per-surface ceiling covering reach, settle and capture together. A surface whose reach-#: navigates into a redirect loop would otherwise hold the sweep open indefinitely.+#: A per-surface ceiling covering reach, settle and capture together, so a reach that navigates+#: into a redirect loop cannot hold the sweep open indefinitely.SURFACE_BUDGET_MS = 25_000@@ -71,10 +71,9 @@ class _Driver:if verb == "goto":page.goto(f"{self.base_url}{args[0]}", wait_until = "domcontentloaded", timeout = 60_000)elif verb == "click":- # A REAL mouse click through the driver, not element.click(). Radix menus open on- # pointerdown, and a synthetic click never opens them -- which reads downstream as a- # menu that opened in zero milliseconds rather than as a menu that did not open. This- # is the same trap dom.js documents for the film's actions.+ # A REAL mouse click through the driver, not element.click(): Radix menus open on pointerdown and+ # a synthetic click never opens them, which reads downstream as a menu that opened in zero+ # milliseconds. The same trap dom.js documents for the film's actions.page.click(args[0], timeout = 6000)elif verb == "click_if":el = page.query_selector(args[0])@@ -144,9 +143,8 @@ def _row(surface: registry.Surface, cell_id: Optional[str]) -> dict:"reason": "the sweep did not get to this surface","conditional": surface.conditional,"also_in_film": surface.also_in_film,- # Carried on the ROW, not only in the registry, so a reader of a payload can tell a- # digest that is a parity signal from one that moves on its own without needing this- # source tree in front of them.+ # Carried on the ROW, not only in the registry, so a reader of a payload can tell a digest that is+ # a parity signal from one that moves on its own without this source tree in front of them."volatile": surface.volatile,"parity": {"parity_attempted": False, "reason": "not captured"},"settle": None,@@ -188,9 +186,9 @@ def sweep(raise registry.RegistryError(f"no such surface(s): {missing}")entries = [s for s in entries if s.id in wanted]- # Does parity.capture() honour a moved root? If it does not, every surface digest is the same- # page-wide reading and the sweep's forty passes mean nothing. Asked ONCE, up front, and the- # answer is carried on every row rather than assumed.+ # Does parity.capture() honour a moved root? If not, every surface digest is the same page-wide+ # reading and the sweep's forty passes mean nothing. Asked ONCE, up front, and the answer is+ # carried on every row.home = driver.run(registry.HOME)scoping = {"scoped": False,@@ -216,9 +214,9 @@ def sweep(started = time.monotonic()deadline = started + budget_ms / 1000- # From the KNOWN STATE, never from wherever the previous surface left the app. A reach- # that happens to work only because the last surface left a menu open is a reach that- # breaks the first time the order changes, and it would break silently.+ # From the KNOWN STATE, never from wherever the previous surface left the app: a reach that works+ # only because the last surface left a menu open breaks the first time the order changes, and+ # silently.reset = driver.run(registry.HOME)if reset is not None:row["reason"] = f"the known state could not be restored before this surface: {reset}"@@ -230,8 +228,8 @@ def sweep(failed = driver.run(surface.reach)if failed is not None:row["reason"] = f"the reach failed: {failed}"- # The facts are still recorded. Where the app ACTUALLY ended up is the first thing a- # reader of a failed reach needs, and it is gone by the time anyone looks.+ # The facts are still recorded. Where the app ACTUALLY ended up is the first thing a reader of a+ # failed reach needs, and it is gone by the time anyone looks.row["facts"] = driver.facts(surface.root)row["reach_ms"] = round((time.monotonic() - started) * 1000, 1)rows.append(row)@@ -258,9 +256,9 @@ def sweep(parity = driver.capture(surface.root)row["parity"] = parityif not parity.get("parity_attempted"):- # Reached, but no digest. Recorded as unreached FOR PARITY PURPOSES, because a- # surface with no digest contributes nothing to the parity claim, and counting it as- # covered is exactly the inflation this file is written to avoid.+ # Reached, but no digest. Recorded as unreached FOR PARITY PURPOSES, because a surface with no+ # digest contributes nothing to the parity claim and counting it as covered is the inflation this+ # file avoids.row["reason"] = f"the surface rendered but no digest was taken: {parity.get('reason')}"rows.append(row)_emit(recorder, row)@@ -285,9 +283,9 @@ def sweep(row["restore_reason"] = restored_recover(driver, log)elif (clean.get("open_dialogs") or 0) or (clean.get("open_menus") or 0):- # The declared restore ran and the app is still dirty. Said out loud: the next- # surface would otherwise be reached from a state nobody declared, and its digest- # would quietly include a leftover overlay.+ # The declared restore ran and the app is still dirty. Said out loud: the next surface would+ # otherwise be reached from a state nobody declared and its digest would include a leftover+ # overlay.row["restore_reason"] = (f"the restore ran but left {clean.get('open_dialogs')} dialog(s) and "f"{clean.get('open_menus')} menu(s) open"@@ -306,8 +304,8 @@ def _emit(recorder: Any, row: dict) -> None:try:recorder.emit(dict(row))except Exception: # noqa: BLE001- # A recorder that rejects a row must not cost the sweep the rest of its surfaces. The row- # is still in the returned list and in the manifest.+ # A recorder that rejects a row must not cost the sweep the rest of its surfaces. The row is still+ # in the returned list and in the manifest.pass@@ -323,9 +321,9 @@ def build_manifest(rows: list, entries: list, scoping: dict) -> dict:"""The coverage manifest: what was swept, what was not, and what is out of reach by design."""reached = [r for r in rows if r.get("reached")]failed = [r for r in rows if not r.get("reached")]- # A conditional surface that did not render is not a hole in the registry: it is a property of- # this installation. Kept separate so the coverage figure is not quietly deflated by a host- # that has no GPU, and not quietly inflated by counting it as covered either.+ # A conditional surface that did not render is not a hole in the registry but a property of this+ # installation. Kept separate so the coverage figure is not quietly deflated by a host with no+ # GPU, nor inflated by counting it as covered.conditional_misses = [r for r in failed if r.get("conditional")]hard_misses = [r for r in failed if not r.get("conditional")]registered = len(entries)@@ -337,8 +335,8 @@ def build_manifest(rows: list, entries: list, scoping: dict) -> dict:"not_reached": len(failed),"not_reached_conditional": len(conditional_misses),"not_reached_hard": len(hard_misses),- # Against the surfaces that COULD have rendered on this host. Reported next to the raw- # count, never instead of it.+ # Against the surfaces that COULD have rendered on this host. Reported next to the raw count,+ # never instead of it."coverage_pct": (round(100.0 * len(reached) / (registered - len(conditional_misses)), 1)if registered - len(conditional_misses) > 0@@ -363,9 +361,9 @@ def build_manifest(rows: list, entries: list, scoping: dict) -> dict:for r in rowsif r.get("restored") is False],- # Reached, digested, and NOT a parity signal. Counted separately because the number that- # matters to somebody comparing two arms is how many surfaces can carry a verdict, and- # that is `comparable`, not `reached`.+ # Reached, digested, and NOT a parity signal. Counted separately because what matters to somebody+ # comparing two arms is how many surfaces can carry a verdict, which is `comparable`, not+ # `reached`."volatile": len([r for r in reached if r.get("volatile")]),"comparable": len([r for r in reached if not r.get("volatile")]),"volatile_surfaces": [diff --git a/tests/studio/studiobench/scene/surfaces.js b/tests/studio/studiobench/scene/surfaces.jsindex 2b20bd60dc2..372a9404aed 100644--- a/tests/studio/studiobench/scene/surfaces.js+++ b/tests/studio/studiobench/scene/surfaces.js@@ -1,38 +1,31 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-//// The selector adapter for Unsloth's surfaces OUTSIDE the chat thread: the other routes, the// settings dialog's twelve tabs, the sidebar and its menus, the hub, the media pages.-//-// Separate from dom.js on purpose. dom.js is the chat thread's adapter and the eighteen film-// actions all read it; this file is read only by the surface sweep, so a selector added for a-// route cannot break an action.-//+// Separate from dom.js on purpose: dom.js is the chat thread's adapter that all eighteen film+// actions read, while this is read only by the surface sweep, so a selector added for a route+// cannot break an action.// THE TRAP THIS FILE EXISTS TO GUARD. ChatPage, ImagesPage, VideoPage and AudioPage are mounted-// PERSISTENTLY by the root layout (`shouldMountChat` and friends in app/routes/__root.tsx) so an-// in-flight generation survives leaving the tab. Off-route they are `class="hidden"` and `inert`,-// but they are still in the document. So on /hub, `document.querySelector(".aui-thread-root")`-// still returns the chat thread, and a digest taken with parity.js's default root would digest-// the HIDDEN CHAT THREAD on every route in the sweep -- forty surfaces reporting one identical-// digest, and every one of them reading as a pass. Every surface therefore names its own root and-// the digest is scoped to it.-//-// HOW THE SCOPING IS DONE, and why it is not a second digest implementation. Surface digests are+// PERSISTENTLY by the root layout so an in-flight generation survives leaving the tab; off-route+// they are `class="hidden"` and `inert` but still in the document. So on /hub,+// `document.querySelector(".aui-thread-root")` still returns the chat thread, and a digest taken+// with parity.js's default root would digest the HIDDEN CHAT THREAD on every route: forty+// surfaces reporting one identical digest, every one reading as a pass. Every surface therefore+// names its own root.+// HOW THE SCOPING IS DONE, and why it is not a second digest implementation: surface digests are// only worth taking if they are comparable with the film's action digests, which means the same-// normalisation and the same hash -- so this calls `window.__sb.parity.capture()` rather than-// walking the DOM itself. parity.js reads its root from `window.__sb.dom.threadRoot()`, so the-// root is moved for the duration of one capture and put back afterwards. `probeScoping()` below-// checks that the move is actually observed, because if a future parity.js stopped reading-// threadRoot this would silently go back to digesting the thread on every surface, which is the-// failure this file was written to prevent.+// normalisation and hash, so this calls `window.__sb.parity.capture()` rather than walking the+// DOM. parity.js reads its root from `window.__sb.dom.threadRoot()`, so the root is moved for one+// capture and put back. `probeScoping()` checks the move is observed, because a future parity.js+// that stopped reading threadRoot would silently go back to digesting the thread everywhere.(() => {if (window.__sb && window.__sb.surfaces) return;window.__sb = window.__sb || {};const q = (sel) => {- // Selector strings come from a registry, and one bad selector must cost that surface a- // reason rather than cost the whole sweep an exception.+ // Selector strings come from a registry, and one bad selector must cost that surface a reason+ // rather than cost the whole sweep an exception.try {return document.querySelector(sel);} catch (err) {@@ -48,8 +41,8 @@}};- // Rendered, not merely mounted. `hidden`, `display:none` and a zero box all mean the user- // cannot see it, and the keep-alive route containers are exactly the first case.+ // Rendered, not merely mounted: `hidden`, `display:none` and a zero box all mean the user cannot+ // see it, and the keep-alive route containers are exactly the first case.const isVisible = (el) => {if (!el) return false;if (el.hasAttribute && el.hasAttribute("hidden")) return false;@@ -59,15 +52,12 @@return style.display !== "none" && style.visibility !== "hidden";};- // The ACTIVE route's container, found by the property the root layout actually sets.- //- // The layout renders the keep-alive pages and the routed page as siblings, and marks every- // page that is NOT the current route `inert` (plus `class="hidden"`). So the active container- // is the one visible sibling of an inert one. Keyed on `inert` rather than on the class string- // because `hidden` is a Tailwind utility that a restyle can change, while `inert` is what makes- // the off-route pages unreachable and cannot be dropped without changing behaviour.- //- // When nothing is inert, no page has been kept alive yet and the inset itself is the container.+ // The ACTIVE route's container, found by the property the root layout actually sets. The layout+ // renders the keep-alive pages and the routed page as siblings and marks every non-current page+ // `inert` (plus `class="hidden"`), so the active container is the one visible sibling of an inert+ // one. Keyed on `inert` rather than the class string because `hidden` is a Tailwind utility a+ // restyle can change, while `inert` is what makes the off-route pages unreachable. When nothing+ // is inert, no page has been kept alive yet and the inset itself is the container.const routeContainer = () => {const inset = q('[data-slot="sidebar-inset"]');if (!inset) return null;@@ -75,8 +65,8 @@if (!inertNode || !inertNode.parentElement) return inset;const siblings = Array.from(inertNode.parentElement.children);const active = siblings.filter((el) => !el.hasAttribute("inert") && isVisible(el));- // Exactly one, or the assumption is wrong and the inset is the honest answer. Silently- // picking the first of several would scope the digest to an arbitrary part of the page.+ // Exactly one, or the assumption is wrong and the inset is the honest answer. Silently picking+ // the first of several would scope the digest to an arbitrary part of the page.return active.length === 1 ? active[0] : inset;};@@ -89,10 +79,8 @@const resolve = (sel) => (SPECIAL[sel] ? SPECIAL[sel]() : q(sel));const S = {- // ── roots ────────────────────────────────────────────────────- //- // `candidates` is ordered: the first one that is present AND visible wins. The auth-flow- // routes render no sidebar wrapper at all, so every list ends at a fallback that exists.+ // `candidates` is ordered: the first present AND visible wins. The auth-flow routes render no+ // sidebar wrapper at all, so every list ends at a fallback that exists.resolveRoot(candidates) {for (const sel of candidates || []) {const el = resolve(sel);@@ -100,14 +88,13 @@}for (const sel of candidates || []) {const el = resolve(sel);- // Present but not visible is still reported, with the selector, so a surface that- // rendered into a hidden container is a readable finding rather than a fallback to body.+ // Present but not visible is still reported, with the selector, so a surface that rendered into a+ // hidden container is a readable finding rather than a fallback to body.if (el) return { el, sel, visible: false };}return { el: document.body, sel: "body", fallback: true };},- // ── the scoped capture ───────────────────────────────────────capture(candidates) {const parity = (window.__sb || {}).parity;if (!parity || typeof parity.capture !== "function") {@@ -136,9 +123,8 @@},// Does parity.capture() actually honour a moved root? Pointed at a detached element with one- // short text node, an honouring capture returns a signature of a few dozen characters and a- // non-honouring one returns the whole page. The threshold is two orders of magnitude clear of- // both, so this cannot answer "yes" for a page that ignored the move.+ // short text node, an honouring capture returns a few dozen characters and a non-honouring one+ // the whole page; the threshold is two orders of magnitude clear of both.probeScoping() {const dom = (window.__sb || {}).dom;const parity = (window.__sb || {}).parity;@@ -167,8 +153,8 @@reason: "parity.capture() returned no `chars`, so scoping cannot be verified" };}if (chars > 500) {- // The digest is the whole page, not the probe. Every surface digest would then be the- // same page-wide reading and the sweep would report forty identical passes.+ // The digest is the whole page, not the probe: every surface digest would then be the same+ // page-wide reading and the sweep would report forty identical passes.return { scoped: false, scoping_attempted: true, probe_chars: chars,reason: "parity.capture() ignored the moved root (" + chars + " chars from a " +"detached probe element), so surface digests would not be scoped" };@@ -176,10 +162,8 @@return { scoped: true, scoping_attempted: true, probe_chars: chars };},- // ── settle predicates ────────────────────────────────────────- //- // Evaluated by the sweep in a poll loop. Each returns a plain boolean plus the observation it- // was made from, so a surface that never settled records WHAT it was waiting for.+ // Evaluated by the sweep in a poll loop. Each returns a plain boolean plus the observation it was+ // made from, so a surface that never settled records WHAT it was waiting for.settled(spec) {if (!spec) return { ok: true, detail: "no settle condition" };if (spec.visible) {@@ -212,8 +196,8 @@return { ok: true, detail: "unrecognised settle spec, treated as satisfied" };},- // What the sweep records alongside every surface so a digest can be read against the state it- // was taken in. A surface whose root holds three elements did not render.+ // What the sweep records alongside every surface so a digest can be read against the state it was+ // taken in. A surface whose root holds three elements did not render.facts(candidates) {const found = S.resolveRoot(candidates);return {@@ -230,8 +214,8 @@};},- // The known state the sweep returns to between surfaces. Reported rather than asserted: the- // sweep decides what to do about a dirty state, and it needs the observation to decide.+ // The known state the sweep returns to between surfaces. Reported rather than asserted: the sweep+ // decides what to do about a dirty state and needs the observation to decide.isClean() {return {clean_attempted: true,diff --git a/tests/studio/studiobench/scene/surfaces.py b/tests/studio/studiobench/scene/surfaces.pyindex 247b963cdea..720a4c3f07d 100644--- a/tests/studio/studiobench/scene/surfaces.py+++ b/tests/studio/studiobench/scene/surfaces.py@@ -40,32 +40,26 @@ import dataclassesfrom dataclasses import dataclassfrom typing import Any, Optional-#: The state every surface is reached from and restored to: a fresh, empty chat. Fresh matters.-#: The keep-alive chat container is inside `@route`'s siblings and inside `@shell`, so a sweep run-#: against a loaded thread would carry that thread's DOM into the digest of every other surface,-#: and any thread difference would flip all forty of them at once.+#: The state every surface is reached from and restored to: a fresh, empty chat. Fresh matters,+#: since the keep-alive chat container is inside `@route`'s siblings and inside `@shell`, so a+#: sweep against a loaded thread would carry that thread's DOM into every other surface's+#: digest and any thread difference would flip all forty at once.KNOWN_STATE_PATH = "/chat"-#: Steps are (verb, *args). Interpreted by surface_sweep, which owns the browser; keeping them+#: Steps are (verb, *args), interpreted by surface_sweep, which owns the browser; keeping them#: declarative is what lets the unit tests check the registry with no browser present.-#:-#: goto <path> navigate, relative to the Unsloth base url-#: click <selector> a REAL mouse click through the driver. Not element.click():-#: Radix menus open on pointerdown and a synthetic click misses-#: them entirely, which reads as a menu that opened instantly-#: click_if <selector> click only when present. For controls that exist in one of two-#: states, e.g. the sidebar is already collapsed-#: hover <selector> park the pointer over an element. REQUIRED before four of the-#: sidebar's controls: the row actions ship-#: `opacity-0 pointer-events-none` and only take-#: `group-hover:pointer-events-auto` when their row is hovered, so-#: a click on them times out against an element that is present,-#: sized and reported visible by every check except the one that-#: matters-#: press <key> a key on the focused element-#: fill <selector> <text> set an input's value through the driver-#: wait <ms> a bounded pause. Only ever ALONGSIDE a settle condition, never-#: instead of one+#: goto <path>: navigate, relative to the Unsloth base url.+#: click <selector>: a REAL mouse click through the driver, not element.click(). Radix menus+#: open on pointerdown and a synthetic click misses them, reading as a menu that opened+#: instantly.+#: click_if <selector>: click only when present, for controls that exist in one of two states+#: (e.g. an already-collapsed sidebar).+#: hover <selector>: park the pointer over an element. REQUIRED before four of the sidebar's+#: controls, whose row actions ship `opacity-0 pointer-events-none` and only take+#: `group-hover:pointer-events-auto` when hovered, so a click times out against an element+#: every check except the relevant one reports as visible.+#: press <key>: a key on the focused element. fill <selector> <text>: set an input's value+#: through the driver. wait <ms>: a bounded pause, only ever ALONGSIDE a settle condition.Step = tuple@@ -81,21 +75,19 @@ class Surface:root: tuple[str, ...]settle: Optional[dict] = None#: Set when the surface's absence is a legitimate property of THIS installation rather than a- #: broken selector: the Connected tab of the model picker needs an external provider, the hub- #: catalog needs network. A surface that fails to reach still records a reason either way;- #: this decides whether the manifest counts it against coverage.+ #: broken selector: the Connected tab needs an external provider, the hub catalog needs+ #: network. A surface that fails to reach records a reason either way; this decides whether the+ #: manifest counts it against coverage.conditional: Optional[str] = None- #: Surfaces the film already drives. Swept anyway -- the sweep runs on an EMPTY chat and the- #: film on a loaded one, so the two digests are of different states -- but flagged so the- #: coverage figure is not inflated by re-counting what was already covered.+ #: Surfaces the film already drives. Swept anyway, since the sweep runs on an EMPTY chat and the+ #: film on a loaded one, but flagged so the coverage figure is not inflated by re-counting.also_in_film: bool = False#: The mechanism by which this surface's digest differs between two runs of the SAME build.#: Measured, not assumed: three consecutive sweeps against one Unsloth agreed on 44 of 53#: surfaces, and every entry below is the mechanism behind one of the nine that did not.- #:- #: This is the same distinction `scripts/sbench_ui_parity.py` already draws for the film's- #: actions with UNSTABLE_ACTIONS, and for the same reason: a comparison that reports a live- #: memory gauge as a UI change gets ignored within a day, and then so does the rest of it.+ #: The same distinction `scripts/sbench_ui_parity.py` draws for the film's actions with+ #: UNSTABLE_ACTIONS, and for the same reason: a comparison that reports a live memory gauge as+ #: a UI change gets ignored within a day.volatile: Optional[str] = Nonenotes: str = ""@@ -115,14 +107,14 @@ class Surface:}+#: Back to the known state the hard way. The restore for anything that navigated, and the+#: recovery path when a surface's own restore left the app dirty.# ── shared fragments ────────────────────────────────────────────────-#: Back to the known state the hard way. Used as the restore for anything that navigated, and as-#: the recovery path when a surface's own restore left the app dirty.HOME: tuple[Step, ...] = (("goto", KNOWN_STATE_PATH), ("wait", 400))-#: Escape twice, not once. A settings tab that has opened a sub-view (Data -> Archived chats,-#: Voice -> Dictionary) eats the first Escape closing the sub-view and stays open on the dialog.+#: Escape twice, not once: a settings tab that opened a sub-view (Data -> Archived chats, Voice+#: -> Dictionary) eats the first Escape closing the sub-view and stays open on the dialog.ESCAPE_OUT: tuple[Step, ...] = (("press", "Escape"),("wait", 200),@@ -130,7 +122,7 @@ ESCAPE_OUT: tuple[Step, ...] = (("wait", 300),)-#: The chat composer. Its presence is what "the chat route has rendered" means, everywhere.+#:The chat composer. Its presence is what 'the chat route has rendered' means, everywhere.COMPOSER = 'textarea[aria-label="Message input"]'ROOT_ROUTE = ("@route",)@@ -138,31 +130,29 @@ ROOT_SHELL = ("@shell",)ROOT_SIDEBAR = ("@sidebar",)-#: The empty chat prints a randomly chosen greeting. `pickRandom` in+#: The empty chat prints a randomly chosen greeting: `pickRandom` in#: components/assistant-ui/thread.tsx:2019 selects from a time-of-day list, so two loads of one-#: build render different headings, and every digest whose root contains the welcome screen-#: differs run to run for that reason alone.+#: build render different headings and every digest containing the welcome screen differs.GREETING_IS_RANDOM = ("the empty chat's greeting is chosen by pickRandom ""(components/assistant-ui/thread.tsx:2019), so two loads of the same build ""render different headings")-#: The hub catalogue is fetched from Hugging Face. Download counts, trending order and the arrival-#: time of the list all move independently of the build under test.+#: The hub catalogue is fetched from Hugging Face; download counts, trending order and the+#: list's arrival time all move independently of the build under test.HUB_IS_REMOTE = ("the catalogue is fetched from Hugging Face, so its content and its arrival time ""both move independently of the build under test")-#: MEASURED TO MOVE, MECHANISM NOT ESTABLISHED. Three sweeps of one build disagreed on these, and-#: a back-to-back pair of visits produced byte-identical markup, so it is not a timer and not a-#: render race inside the surface. It is state that something earlier in the sweep, or the install,-#: carries between sweeps -- and which state has not been pinned down.-#:-#: Flagged rather than left comparable, and flagged as UNEXPLAINED rather than given a plausible-#: cause. A wrong mechanism in this field is worse than an admitted gap: it is the sentence a-#: reader uses to dismiss a real difference.+#: MEASURED TO MOVE, MECHANISM NOT ESTABLISHED. Three sweeps of one build disagreed on these+#: while a back-to-back pair of visits produced byte-identical markup, so it is not a timer and+#: not a render race inside the surface: it is state something earlier in the sweep, or the+#: install, carries between sweeps.+#: Flagged as UNEXPLAINED rather than given a plausible cause, because a wrong mechanism here+#: is worse than an admitted gap: it is the sentence a reader uses to dismiss a real+#: difference.UNEXPLAINED_DRIFT = ("measured to differ between sweeps of the SAME build while a back-to-back ""pair of visits produced byte-identical markup. The mechanism is NOT "@@ -194,9 +184,8 @@ def _route(def _settings_tab(tab_id: str, title: str) -> Surface:- # `/settings` is not a page. Its route component calls `openDialog()` and immediately- # redirects to /chat, so the dialog is the surface and the URL never stays on /settings --- # a settle condition written against the URL would never be satisfied.+ # `/settings` is not a page: its route component calls `openDialog()` and redirects to /chat,+ # so the dialog is the surface and a settle written against the URL would never be satisfied.return Surface(id = f"settings:{tab_id}",group = "settings",@@ -207,10 +196,9 @@ def _settings_tab(tab_id: str, title: str) -> Surface:("click", f'[data-testid="settings-tab-{tab_id}"]'),("wait", 400),),- # The panel is lazily imported per tab, so "the dialog is open" is not enough: the tab- # would be recorded as reached while its chunk was still in flight and the digest would- # be of the Suspense fallback. The tab's own pressed state plus a settled panel body is- # the observation that the panel arrived.+ # The panel is lazily imported per tab, so 'the dialog is open' is not enough: the tab would be+ # recorded as reached with its chunk in flight and the digest would be of the Suspense+ # fallback. The tab's pressed state plus a settled panel body is what says the panel arrived.settle = {"js": f"!!document.querySelector('[data-testid=\"settings-tab-{tab_id}\"]')"f' && !!document.querySelector(".settings-surface main")'@@ -225,7 +213,6 @@ def _settings_tab(tab_id: str, title: str) -> Surface:# ── the registry ────────────────────────────────────────────────────_SURFACES: list[Surface] = [- # ── routes ──────────────────────────────────────────────────────_route("route:chat","Chat, empty state",@@ -247,10 +234,10 @@ _SURFACES: list[Surface] = ["route:train","Train (Unsloth)","/studio",- # NOT a URL check alone, and not the nav row either: the nav row is in the sidebar and- # is there before the page is. StudioPage shows a spinner while the hardware verdict is- # unmeasured and then redirects to /chat if the host is chat-only, so the wizard's own- # last section is what says the page finished rendering.+ # NOT a URL check alone, and not the nav row either: the nav row is in the sidebar and is+ # there before the page is. StudioPage shows a spinner while the hardware verdict is+ # unmeasured and then redirects to /chat if the host is chat-only, so the wizard's last+ # section is what says the page finished rendering.{"js": 'location.pathname === "/studio" && ''(document.body.innerText || "").includes("Run preview")'@@ -280,8 +267,8 @@ _SURFACES: list[Surface] = ["/api-monitor",{"js": 'location.pathname === "/api-monitor"'},),- # The 404 is a rendered surface with its own mascot, heading and back-link, and it is exactly- # the kind of page a bundler or asset-path change breaks without anyone noticing.+ # The 404 is a rendered surface with its own mascot, heading and back-link, exactly the kind of+ # page a bundler or asset-path change breaks unnoticed._route("route:not-found","Not found",@@ -309,15 +296,14 @@ _SURFACES: list[Surface] = ["timestamps. Two visits differed by 402 lines",),_settings_tab("about", "Settings: About"),- # The dialog's own search index is a distinct rendering of every panel's labels, and it is the- # one place a settings string appears without its panel being mounted.+ # The dialog's own search index is a distinct rendering of every panel's labels, and the one+ # place a settings string appears without its panel being mounted.Surface(id = "settings:search",group = "settings",title = "Settings: search results",- # By aria-label, not `input[type="text"]`: the search box ships no `type` attribute at- # all, so a type selector matches the Embedding model field further down the General- # panel and fills THAT instead -- a reach that succeeds and lands somewhere else.+ # By aria-label, not `input[type="text"]`: the search box ships no `type` attribute, so a type+ # selector matches the Embedding model field further down the General panel and fills THAT.reach = (("goto", "/settings"),("wait", 500),@@ -331,7 +317,6 @@ _SURFACES: list[Surface] = [restore = ESCAPE_OUT + HOME,root = (".settings-surface", '[data-slot="dialog-content"]'),),- # ── the sidebar ─────────────────────────────────────────────────Surface(id = "sidebar:expanded",group = "sidebar",@@ -346,8 +331,8 @@ _SURFACES: list[Surface] = [group = "sidebar",title = "Sidebar, collapsed to the rail",reach = (("click", 'button[aria-label="Close sidebar"]'), ("wait", 500)),- # The rail is the same container in a different data-state, so presence proves- # nothing; the reopen control appearing is what says the collapse happened.+ # The rail is the same container in a different data-state, so presence proves nothing; the+ # reopen control appearing is what says the collapse happened.settle = {"visible": 'button[aria-label="Open sidebar"]'},restore = (("click_if", 'button[aria-label="Open sidebar"]'), ("wait", 400)) + HOME,root = ROOT_SIDEBAR,@@ -427,15 +412,14 @@ _SURFACES: list[Surface] = [restore = (("click_if", 'button[aria-label="Hide workflows"]'), ("wait", 300)),root = ROOT_SIDEBAR,),- # ── the chat composer's own overlays ────────────────────────────Surface(id = "chat:model-picker",group = "chat",title = "Model picker",reach = (("click", ".unsloth-model-selector-trigger"), ("wait", 600)),- # The rows, not the menu. The popover mounts before its list has been fetched, so a- # settle on the container's presence digests an empty picker roughly half the time --- # and an empty picker and a broken picker have the same digest.+ # The rows, not the menu: the popover mounts before its list has been fetched, so a settle on+ # the container's presence digests an empty picker roughly half the time, and an empty picker+ # and a broken picker have the same digest.settle = {"count_at_least": [".unsloth-model-selector-menu button", 2]},restore = ESCAPE_OUT,root = (".unsloth-model-selector-menu",),@@ -472,10 +456,10 @@ _SURFACES: list[Surface] = [reach = (("click", 'button[aria-label="Open run settings"]'), ("wait", 600)),settle = {"visible": 'button[aria-label="Close run settings"]'},restore = (("click_if", 'button[aria-label="Close run settings"]'), ("wait", 400)) + HOME,- # The panel, not a sheet. Despite the file being called chat-settings-sheet it renders- # as a docked column inside the chat layout, so it is in NO overlay list and a root of- # `[role="dialog"]` falls through to body -- which digested 301,402 characters of- # whole page and called it the run settings panel.+ # The panel, not a sheet: despite the file being called chat-settings-sheet it renders as a+ # docked column inside the chat layout, so it is in NO overlay list and a root of+ # `[role="dialog"]` falls through to body, which digested 301,402 characters of whole page and+ # called it the run settings panel.root = ('[data-slot="chat-settings-panel"]',),),Surface(@@ -491,9 +475,9 @@ _SURFACES: list[Surface] = [id = "chat:system-prompt",group = "chat",title = "System prompt editor",- # Through the run settings panel. The edit control is rendered at x=1458 on a 1440- # viewport when the panel is closed -- present, sized, and off screen -- so clicking- # it directly times out on an element every presence check reports as there.+ # Through the run settings panel: the edit control is rendered at x=1458 on a 1440 viewport+ # when the panel is closed (present, sized and off screen), so clicking it directly times out+ # on an element every presence check reports as there.reach = (("click", 'button[aria-label="Open run settings"]'),("wait", 800),@@ -519,12 +503,11 @@ _SURFACES: list[Surface] = [group = "chat",title = "Composer holding text",reach = (("fill", COMPOSER, "studiobench surface sweep"), ("wait", 400)),- # The send control is what changes, and it is the same control the film's stop action- # trips over: with text in the box the Stop button is replaced by Queue.- # ONE EXPRESSION. The settle evaluator wraps the string in `return (...)`, so a- # statement -- a `const` declaration, say -- raises SyntaxError at evaluation time,- # and the surface is recorded as never settling for a reason that is about this file- # rather than about the app.+ # The send control is what changes, and it is the control the film's stop action trips over:+ # with text in the box the Stop button is replaced by Queue.+ # ONE EXPRESSION: the settle evaluator wraps the string in `return (...)`, so a statement+ # raises SyntaxError at evaluation time and the surface is recorded as never settling for a+ # reason about this file rather than about the app.settle = {"js": "(document.querySelector(""'textarea[aria-label=\"Message input\"]') || {}).value ? true : false"@@ -534,7 +517,6 @@ _SURFACES: list[Surface] = [also_in_film = True,volatile = GREETING_IS_RANDOM,),- # ── the hub ─────────────────────────────────────────────────────Surface(id = "hub:datasets",group = "hub",@@ -545,9 +527,9 @@ _SURFACES: list[Surface] = [("click", 'button[aria-label="Datasets"]'),("wait", 800),),- # The URL flips the instant the tab is pressed, well before the list arrives. Settling- # on the URL alone digested an empty table on one visit and a populated one on the- # next, a 424-line difference that had nothing to do with any build.+ # The URL flips the instant the tab is pressed, well before the list arrives. Settling on the+ # URL alone digested an empty table on one visit and a populated one on the next, a 424-line+ # difference unrelated to any build.settle = {"js": 'location.search.includes("kind=datasets") && ''(document.body.innerText || "").includes("Downloads")'@@ -557,9 +539,9 @@ _SURFACES: list[Surface] = [conditional = "needs the catalogue, which needs network",volatile = HUB_IS_REMOTE,),- # The hub's two filter popovers are LISTBOXES, not menus. `[role="menuitem"]` counted zero on- # a popover that was open and visible on screen, and zero from a wrong selector is the failure- # mode this tool exists to refuse -- so both the settle and the root read `option`/`listbox`.+ # The hub's two filter popovers are LISTBOXES, not menus: `[role="menuitem"]` counted zero on a+ # popover open and visible on screen, and zero from a wrong selector is the failure mode this+ # tool exists to refuse, so both the settle and the root read `option`/`listbox`.Surface(id = "hub:format-filter",group = "hub",@@ -623,7 +605,6 @@ _SURFACES: list[Surface] = [conditional = "needs the catalogue, which needs network",volatile = HUB_IS_REMOTE,),- # ── the media pages ─────────────────────────────────────────────Surface(id = "images:presets",group = "media",@@ -673,7 +654,6 @@ _SURFACES: list[Surface] = [root = ('[role="menu"]', "[data-radix-popper-content-wrapper]"),conditional = "a host without video support renders the gate instead of the page",),- # ── train ───────────────────────────────────────────────────────Surface(id = "train:model-picker",group = "train",@@ -738,9 +718,9 @@ _SURFACES: list[Surface] = [("click", 'button[aria-label="Image training"]'),("wait", 1200),),- # It NAVIGATES. The control sits on the Train page but lands on /images with the- # training panel selected, so a settle written against /studio waits out its timeout- # on a page that arrived correctly.+ # It NAVIGATES: the control sits on the Train page but lands on /images with the training panel+ # selected, so a settle written against /studio waits out its timeout on a page that arrived+ # correctly.settle = {"js": 'location.pathname === "/images" && ''(document.body.innerText || "").includes("Train a LoRA")'@@ -750,7 +730,6 @@ _SURFACES: list[Surface] = [conditional = "a chat-only host redirects /studio to /chat",volatile = UNEXPLAINED_DRIFT,),- # ── the api monitor ─────────────────────────────────────────────Surface(id = "api:status-filter",group = "api",@@ -772,10 +751,9 @@ _SURFACES: list[Surface] = [#: Surfaces that exist and are NOT swept, each with the mechanism that puts them out of reach.-#:-#: This list is the honest half of the coverage number. A manifest that reports "38 of 38 reached"-#: while eleven surfaces were never registered is a worse artefact than one that reports 38 of 38-#: and names the eleven, because the first invites the reader to believe the app is fully covered.+#: This list is the honest half of the coverage number: a manifest reporting '38 of 38 reached'+#: while eleven surfaces were never registered invites the reader to believe the app is fully+#: covered.KNOWN_UNCOVERED: tuple[dict, ...] = ({"id": "route:login",@@ -902,8 +880,8 @@ def validate_registry(entries: Optional[list[Surface]] = None) -> None:raise RegistryError(f"surface {s.id!r} is missing an id, group or title")if not s.root:raise RegistryError(f"surface {s.id!r} has no digest root")- # `reach` may be empty only for a surface that IS the known state; `restore` may be empty- # only when reaching it changed nothing. Both are declared, never inferred.+ # `reach` may be empty only for a surface that IS the known state; `restore` may be empty only+ # when reaching it changed nothing. Both are declared, never inferred.for name, steps in (("reach", s.reach), ("restore", s.restore)):for step in steps:if not step:diff --git a/tests/studio/studiobench/scoring/ab.py b/tests/studio/studiobench/scoring/ab.pyindex 5a997301c0d..972da921bff 100644--- a/tests/studio/studiobench/scoring/ab.py+++ b/tests/studio/studiobench/scoring/ab.py@@ -73,8 +73,8 @@ class RunIdentity:}-#: The fields that must match. `session_id` is separate because its failure mode is different:-#: mismatched ids mean different meanings, a mismatched session means an 8% drift term.+#: The fields that must match. `session_id` is separate because its failure mode differs: mismatched+#: ids mean different meanings, a mismatched session means an 8% drift term.COMPARABILITY_FIELDS = ("bench_version", "corpus_hash", "rung_ladder_id", "weights_id")@@ -184,13 +184,13 @@ class MetricComparison:ci_high: float | Noneverdict: str = "no_reading"beyond_noise: bool = False- #: The 95% CI of the paired geometric mean contains 1.0, so the data cannot rule out "no- #: effect" however far the point estimate landed from the noise floor. Carried so the table- #: can say the size is unresolved rather than print a direction as a finding.+ #: The 95% CI of the paired geometric mean contains 1.0, so the data cannot rule out "no effect"+ #: however far the point estimate landed from the noise floor. Carried so the table can say the+ #: size is unresolved rather than print a direction as a finding.ci_spans_no_effect: bool = False- #: At least one contributing pair had a sub-floor arm, so the ratio is a BOUND: the true- #: magnitude is larger. Carried so the table can say so rather than let a number derived from- #: an instrument floor be quoted as a measurement.+ #: At least one contributing pair had a sub-floor arm, so the ratio is a BOUND: the true magnitude+ #: is larger. Carried so a number derived from an instrument floor is not quoted as a+ #: measurement.bounded: bool = False@property@@ -304,18 +304,16 @@ class AbResult:if self.regressions:return "FAIL"if self.headline_ratio is None:- # UNRESOLVED IS NOT UNMEASURED. `compare` keeps unresolved metrics out of the headline- # entirely, so a run whose every moving metric straddled 1.0 arrives here with no- # ratio at all. That is not the same as an empty payload: the data exists and says- # nothing, which is INCONCLUSIVE, while NO READING means there was nothing to read.+ # UNRESOLVED IS NOT UNMEASURED. `compare` keeps unresolved metrics out of the headline entirely, so+ # a run whose every moving metric straddled 1.0 arrives with no ratio at all: the data exists and+ # says nothing, which is INCONCLUSIVE, while NO READING means there was nothing to read.if any(m.unresolved for m in self.metrics):return "INCONCLUSIVE"return "NO READING"- # NO DIFFERENCE IS A CLAIM, AND A STRONGER ONE THAN THIS DATA SUPPORTS. Excluding an- # unresolved mover from the headline can leave only flat metrics behind, putting the- # aggregate back inside the noise floor. Reporting "no difference" there would assert the- # change did nothing, when in fact one metric moved and could not resolve its own sign.- # Only say that once some metric actually resolved a direction.+ # NO DIFFERENCE IS A CLAIM, AND A STRONGER ONE THAN THIS DATA SUPPORTS. Excluding an unresolved+ # mover can leave only flat metrics behind, putting the aggregate back inside the noise floor;+ # reporting 'no difference' there would assert the change did nothing when one metric moved and+ # could not resolve its own sign.if any(m.unresolved for m in self.metrics) and not any(m.resolves_direction for m in self.metrics):@@ -404,13 +402,11 @@ def compare(if not comparison.beyond_noise:comparison.verdict = "within noise"elif worse:- # THE ASYMMETRY IS THE POINT, ON BOTH THE LABEL AND THE HEADLINE. Withholding an- # unresolved win costs a headline; withholding an unresolved loss ships the- # regression. So the worse side keeps counting and keeps contributing to the- # aggregate, where it can only drag the number toward worse, which is the- # conservative direction. A single bounded pair over a zero base is the case that- # proves it: `_divisible` exists because that regression once vanished from the- # table, the headline and this list at once, and it must not vanish again.+ # THE ASYMMETRY IS THE POINT, ON BOTH THE LABEL AND THE HEADLINE. Withholding an unresolved win+ # costs a headline; withholding an unresolved loss ships the regression. So the worse side keeps+ # counting and keeps contributing to the aggregate, where it can only drag the number toward+ # worse. `_divisible` exists because a single bounded pair over a zero base once vanished from the+ # table, the headline and this list at once.comparison.verdict = ("regressed (unresolved)" if comparison.ci_spans_no_effect else "regressed")@@ -425,14 +421,12 @@ def compare()else:comparison.verdict = "inconclusive" if comparison.unresolved else "improved"- # AN UNRESOLVED METRIC LENDS ITS MAGNITUDE TO NOTHING. The headline is a weighted- # geometric mean of point estimates and carries no interval of its own, so a metric- # that moved a long way but could not resolve its own sign would otherwise supply most- # of the number that gets quoted. keystroke_p95_ms at 0.2, 0.2, 1.5, 1.5 (geomean- # 0.548, CI 0.200-1.500) beside a resolved menu_open_ms of 0.900 produced a headline- # of 0.631 and the word IMPROVED: a quoted 36.9% win almost entirely made of data the- # table itself labels inconclusive. Metrics still within noise keep contributing --- # they pull the headline toward 1.0, which is the conservative direction.+ # AN UNRESOLVED METRIC LENDS ITS MAGNITUDE TO NOTHING. The headline is a weighted geometric mean+ # of point estimates carrying no interval of its own, so a metric that moved a long way but could+ # not resolve its sign would supply most of the quoted number: keystroke_p95_ms at 0.2, 0.2, 1.5,+ # 1.5 beside a resolved menu_open_ms of 0.900 produced a headline of 0.631 and the word IMPROVED.+ # That metric's own mean was 0.548, CI 0.200-1.500.+ # Metrics still within noise keep contributing, pulling the headline toward 1.0.if not comparison.withheld:weight = anchor.weight if anchor else 1.0weighted_logs.append((weight, math.log(geo)))@@ -443,8 +437,8 @@ def compare(result.headline_ratio = math.exp(sum(w * lg for w, lg in weighted_logs) / total_weight)if is_null_control:- # A null control that shows a difference is the harness moving, not the build. Whatever- # it reports, no comparison run on the same machine at the same time can be believed.+ # A null control that shows a difference is the harness moving, not the build. Whatever it+ # reports, no comparison run on the same machine at the same time can be believed.offenders = [f"{m.metric_key}: {abs((m.ratio_geomean - 1.0) * 100):.1f}%"for m in result.metricsdiff --git a/tests/studio/studiobench/scoring/anchors.py b/tests/studio/studiobench/scoring/anchors.pyindex 5d1a1641052..54e7786f7e2 100644--- a/tests/studio/studiobench/scoring/anchors.py+++ b/tests/studio/studiobench/scoring/anchors.py@@ -123,23 +123,23 @@ METRIC_ANCHORS: tuple[MetricAnchor, ...] = (METRIC_BY_KEY: Mapping[str, MetricAnchor] = {m.key: m for m in METRIC_ANCHORS}-#: The rung ladder, in tokens of thread content. Log-spaced on purpose: aggregation integrates-#: over log(tokens), so evenly spaced rungs on the log axis give evenly spaced evidence.+#: The rung ladder, in tokens of thread content. Log-spaced on purpose: aggregation integrates over+#: log(tokens), so evenly spaced rungs on the log axis give evenly spaced evidence.RUNG_TOKENS: tuple[int, ...] = (1_000, 10_000, 100_000, 500_000, 1_000_000)-#: A rung counts as usable when its score clears this AND no single metric is catastrophic.-#: The onset rung -- the largest usable rung -- is the human headline, because it survives being-#: carried to a different machine in a way a 0-100 score does not.+#: A rung counts as usable when its score clears this AND no single metric is catastrophic. The+#: onset rung, the largest usable rung, is the human headline, because it survives being carried to+#: a different machine in a way a 0-100 score does not.ONSET_SCORE_THRESHOLD = 50.0ONSET_METRIC_FLOOR = 10.0#: A rung whose measured metrics do not cover at least this fraction of the declared weight is-#: INCOMPLETE, which scores 0. It does not drop out: a build that crashes at 500K must not-#: outscore one that limps through it, and dropping the rung is exactly how that happens.+#: INCOMPLETE, which scores 0. It does not drop out: a build that crashes at 500K must not outscore+#: one that limps through it.MIN_WEIGHT_COVERAGE = 0.60-#: Ratios inside this band are indistinguishable from run-to-run noise. Anything an A/B claims-#: must clear it, and the NULL calibration arm must land inside it.+#: Ratios inside this band are indistinguishable from run-to-run noise. Anything an A/B claims must+#: clear it, and the NULL calibration arm must land inside it.DEFAULT_NOISE_FLOOR_PCT = 5.0diff --git a/tests/studio/studiobench/scoring/frames.py b/tests/studio/studiobench/scoring/frames.pyindex 1747b015f08..2b5ce56cab4 100644--- a/tests/studio/studiobench/scoring/frames.py+++ b/tests/studio/studiobench/scoring/frames.py@@ -42,21 +42,21 @@ from typing import Any, Sequencefrom .schema import Measure-#: Perceptual constant. A frame longer than this reads as an interruption regardless of display.+#:Perceptual constant. A frame longer than this reads as an interruption regardless of display.JANK_FRAME_MS = 100.0-#: Detection floors. Below these the recorder cannot distinguish a reading from its own noise.+#:Detection floors. Below these the recorder cannot distinguish a reading from its own noise.FLOOR_FRAME_MS = 0.5FLOOR_TIME_IN_JANK_PCT = 0.1FLOOR_JANK_INDEX = 0.05-#: A refresh interval outside this range is not a display, it is a broken recorder.+#:A refresh interval outside this range is not a display, it is a broken recorder.REFRESH_MIN_MS = 3.0REFRESH_MAX_MS = 60.0REFRESH_FALLBACK_MS = 1000.0 / 60.0-#: Log-spaced histogram edges, in ms. Chosen so that the interesting region (one frame, a few-#: frames, a visible hitch, a freeze) each gets its own bucket rather than all landing in ">33".+#: Log-spaced histogram edges, in ms. Chosen so the interesting region (one frame, a few frames, a+#: visible hitch, a freeze) each gets its own bucket rather than all landing in ">33".HISTOGRAM_EDGES_MS: tuple[float, ...] = (0.0,4.0,@@ -179,10 +179,10 @@ def compute_frame_stats(budget_ms, refresh_source = measure_refresh_interval_ms(usable)if not usable:- # The recorder was installed and produced nothing. That is the rAF-unscheduled trap:- # a compositor that decided nothing is visible stops SCHEDULING callbacks, and the naive- # reading is "zero dropped frames". Every metric here reads as a failed measurement so- # the tri-clock gate upstream is what decides whether the window survives.+ # The recorder was installed and produced nothing: the rAF-unscheduled trap, where a compositor+ # that decided nothing is visible stops SCHEDULING callbacks and the naive reading is "zero dropped+ # frames". Every metric here reads as a failed measurement so the tri-clock gate upstream decides+ # whether the window survives.reason = "frame recorder produced no frames (rAF may be unscheduled)"return FrameStats(frames_total = 0,diff --git a/tests/studio/studiobench/scoring/from_payload.py b/tests/studio/studiobench/scoring/from_payload.pyindex e0ff367cc11..783cabe117b 100644--- a/tests/studio/studiobench/scoring/from_payload.py+++ b/tests/studio/studiobench/scoring/from_payload.py@@ -32,21 +32,21 @@ from .anchors import METRIC_BY_KEYfrom .frames import compute_frame_statsfrom .schema import Measure-# ── where each scored metric actually lives in the payload ──────────────────────────────────# (action name, timing key). Anything not listed here comes from the window frame recorder.+# ── where each scored metric actually lives in the payload ──────────────────────────────────+ACTION_SOURCES: Mapping[str, tuple[str, str]] = {"keystroke_p95_ms": ("keystroke", "p95_ms"),"menu_open_ms": ("message_menu", "open_ms"),"scroll_settle_ms": ("scroll_after", "gesture_ms"),}-# The one mapping that is NOT an identity of meaning, so it is stated rather than buried. The-# anchor was written for settle time (how long after the gesture the thread stops moving); the-# scene records the gesture itself and the per-step cost, and never measures settle separately.-# `gesture_ms` is the closest recorded quantity and is on the same scale, but a reader comparing-# this column against the anchor's 100 ms / 3000 ms rationale is comparing against a slightly-# different thing, and should be told so in the cell rather than in a changelog.+# The one mapping that is NOT an identity of meaning. The anchor was written for settle time (how+# long after the gesture the thread stops moving); the scene records the gesture and the+# per-step cost and never measures settle. `gesture_ms` is the closest recorded quantity on the+# same scale, but a reader comparing this column against the anchor's 100 ms / 3000 ms rationale+# is comparing a slightly different thing.SCROLL_SETTLE_NOTE = ("recorded as scroll_after.gesture_ms; the scene does not measure settle separately, so this ""is gesture duration, not post-gesture settle"@@ -54,21 +54,18 @@ SCROLL_SETTLE_NOTE = (FRAME_METRICS: tuple[str, ...] = ("time_in_jank_pct", "jank_index", "max_frame_ms")+# The streaming phase, separated out and normalised per character.+# The three metrics above are not blind to the stream (`_frame_measures` pools every non-`idle`+# window) but they cannot separate it: one 57.3 s film collapses eighteen action windows and the+# streaming stretch into one number the action windows dominate. On a measured 100K null control+# `reasoning_toggle` alone contributed 2,865 ms blocked at 99.3% busy with a 1,866 ms worst frame+# while the streaming stretch beside it ran at 3.6% busy with a 100 ms worst frame.+# THE WINDOW KIND CANNOT BE USED TO SEPARATE THEM, and the name is what misleads:+# `SceneRunner._gap_window` opens every inter-slot gap as `kind = "stream"`, so eighteen windows+# are named `stream:gapN` and only the first four contain streaming, while `stream:drain` is+# opened after the film (7 ms on that cell). The phase is taken from the `stream_cost`+# instrument, which detects it from SSE traffic, never from the label.# ── the streaming phase, separated out and normalised per character ─────────────────────────-#-# The three metrics above are not blind to the stream. `_frame_measures` pools every window whose-# kind is not `idle`, and the streaming windows are in that pool. What they cannot do is separate-# it: one 57.3 s film collapses eighteen action windows and the streaming stretch into a single-# number, and the action windows dominate. On a measured 100K null control, `reasoning_toggle`-# alone contributed 2,865 ms of blocked time at 99.3% busy with a 1,866 ms worst frame while the-# streaming stretch beside it ran at 3.6% busy with a 100 ms worst frame.-#-# THE WINDOW KIND CANNOT BE USED TO SEPARATE THEM, and the name is what misleads.-# `SceneRunner._gap_window` opens every inter-slot gap as `kind = "stream"`, so on the standard-# film eighteen windows are named `stream:gapN` and only the first four contain streaming;-# `stream:drain` is opened after the film has finished and measured 7 ms on that same cell. The-# phase is therefore taken from the `stream_cost` instrument, which detects it from the SSE-# traffic, and never from the label.STREAM_METRICS: tuple[str, ...] = ("stream_delta_cost_ms_per_kchar","stream_cost_ms_per_kchar",@@ -78,39 +75,33 @@ STREAM_METRICS: tuple[str, ...] = ("stream_max_frame_ms",)-# A window has to carry at least this much streamed text before its cost is divided by it. Below-# it the denominator is small enough that the ratio is dominated by whatever else shared the-# window: on a measured cell an `action:send_turn` window grew the reply by 13 characters while-# accumulating 475 ms of blocked time, which as a rate is 36,500 ms per thousand characters and is-# a statement about opening a menu, not about streaming.+# A window must carry at least this much streamed text before its cost is divided by it. Below it+# the ratio is dominated by whatever else shared the window: a measured `action:send_turn` window+# grew the reply by 13 characters while accumulating 475 ms blocked, which as a rate is 36,500 ms+# per thousand characters and is a statement about opening a menu.MIN_STREAM_CHARS_PER_WINDOW = 100# More timer ticks than the clamp says are possible means the clamp is wrong, and every blocked# figure derived from it is a subtraction against the wrong floor. `clocks_agree` would be the-# gate, but on a headless engine it is null by design (see instruments/pagejs.py: rAF has no vsync-# to be checked against and the screencast is rate-limited), so `timer_clock_ratio` is the sound-# availability signal and this is the bound it has to respect.+# gate but is null by design on a headless engine (see instruments/pagejs.py), so+# `timer_clock_ratio` is the sound availability signal and this is the bound it must respect.MAX_TIMER_CLOCK_RATIO = 1.2-# Windows that are not part of the film, and whose frames therefore say nothing about the build.-#-# `idle` is deliberately quiet: pooling it into the frame metrics would dilute every jank share-# with idle time and make a bad build look average.-#-# `setup` is the opposite and is excluded for the opposite reason. The only one is the composer-# click that starts the film, and most of it is Playwright's injected actionability script ---# selector resolution, visibility, stability and the `elementsFromPoint` hit test -- running on-# the page's own main thread, where it blocks frames indistinguishably from app work. At 500K-# that window alone runs about 11 s against a `max_frame_ms` anchor whose worst case is 2,000 ms,-# so pooling it would peg all three frame metrics on every run, including runs that never asked-# for the click probe.+# Windows that are not part of the film, whose frames say nothing about the build.+# `idle` is deliberately quiet: pooling it would dilute every jank share with idle time and make+# a bad build look average.+# `setup` is excluded for the opposite reason. The only one is the composer click that starts the+# film, mostly Playwright's injected actionability script running on the page's own main thread,+# where it blocks frames indistinguishably from app work. At 500K that window alone runs about+# 11 s against a `max_frame_ms` anchor whose worst case is 2,000 ms, so pooling it would peg all+# three frame metrics on every run.UNSCORED_WINDOW_KINDS: frozenset[str] = frozenset({"idle", "setup"})-# The window kinds in which NO SCRIPTED ACTION IS RUNNING. `gap` is the scheduler's inter-slot-# wait; `stream` is `stream:drain`, the window the session layer opens after the film to wait the-# reply out. Both are quiet by construction, which is the property `_unaided` needs -- see there-# for why the streaming numbers are taken only from these, and for what the `stream` half is worth.-# `action` is excluded, and `idle` never reaches here (UNSCORED_WINDOW_KINDS strips it first).+# The window kinds in which NO SCRIPTED ACTION IS RUNNING: `gap` is the scheduler's inter-slot+# wait and `stream` is `stream:drain`, opened after the film to wait the reply out. Both are+# quiet by construction, which is what `_unaided` needs. `action` is excluded, and `idle` never+# reaches here.+# `UNSCORED_WINDOW_KINDS` strips `idle` first.UNAIDED_WINDOW_KINDS: frozenset[str] = frozenset({"gap", "stream"})@@ -200,11 +191,9 @@ def _action_measure(metric_key: str, actions: Mapping[str, Mapping[str, Any]]) -# Attempted-and-did-not-run, which is a fact about the run, not an absent instrument.return Measure.failed(unit, f"{action_name} did not run: {reason}")if row.get("expect_ok") is False:- # RAN IS NOT DID WHAT IT CLAIMED. An action carries its own assertion -- the composer's- # value grew by the characters that were typed, the menu that was opened has items, the- # panes that were expanded are open -- and when that assertion fails the timing describes- # something other than the action. `report/payload.py` already lists these cells under- # EXCLUDED CELLS with "its timings exist and must not be quoted"; scoring them anyway let+ # RAN IS NOT DID WHAT IT CLAIMED. An action carries its own assertion, and when it fails the+ # timing describes something other than the action. `report/payload.py` already lists these cells+ # under EXCLUDED CELLS with 'its timings exist and must not be quoted'; scoring them anyway let# the same number be excluded in the report and load-bearing in the headline.reason = row.get("reason") or "no reason recorded"return Measure.failed(unit, f"{action_name} ran but its own assertion failed: {reason}")@@ -257,17 +246,12 @@ def _frame_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]:if frameless:# ONE WINDOW THAT SAW NOTHING POISONS THE POOL, IT DOES NOT DROP OUT OF IT.- #- # A window whose recorder was installed and exported no deltas at all is the rAF-- # unscheduled trap, and `compute_frame_stats` already refuses to score it: a single such- # window reads `Measure.failed` on every metric with `no_frames_recorded` set, never zero- # jank. Pooled, the same window was skipped by the `continue` above -- it contributed no- # deltas, no wall time and (its `max_frame_ms` being null) no worst frame -- so the- # REMAINING windows answered for the whole cell and a complete freeze during one action- # came back as clean numbers: a 4 s frozen window beside a smooth one scored 0.0% time in- # jank and a 40 ms worst frame, byte-identical to the cell without the freeze in it.- # An unmeasured window is not an absent one, so the cell's frame metrics fail here for- # the same reason and in the same shape as the single-window path.+ # A window whose recorder was installed and exported no deltas is the rAF-unscheduled trap, and+ # `compute_frame_stats` already refuses to score it. Pooled, such a window was skipped by the+ # `continue` above (no deltas, no wall time, no worst frame) so the REMAINING windows answered+ # for the whole cell and a complete freeze came back clean: a 4 s frozen window beside a smooth+ # one scored 0.0% time in jank and a 40 ms worst frame, byte-identical to the cell without the+ # freeze. An unmeasured window is not an absent one.reason = (f"{frameless} window(s) recorded no frames at all (rAF may be unscheduled), so the ""pooled frame metrics would describe only the windows that were measured"@@ -328,17 +312,13 @@ def _stream_windows(windows: Sequence[Mapping[str, Any]]) -> tuple[list[Mapping[str(sc.get("reply_chars_delta_reason") or "the reply's growth was not measurable"))continue- # THE INSTRUMENT'S OWN VERDICT ON ITS DENOMINATOR, and it is consulted here because this is- # the only place that can act on it. `instruments/streamcost.py` marks a window unscoreable- # when an SSE frame failed to parse inside it or an unterminated frame was still buffered- # at its close -- and per the HTML standard an event is dispatched only at the blank line- # that terminates it, so those characters have not been counted and cannot be recovered.- # The delta is therefore short by an unknown amount, and every cost-per-character divided- # by it comes out inflated. Publishing the flag and then summing the delta anyway left the- # official metrics derived from a denominator the instrument had already disowned.- #- # `is False` and not falsiness: a payload recorded before the flag existed carries no key- # at all, and those windows are admitted exactly as they were rather than voided wholesale.+ # THE INSTRUMENT'S OWN VERDICT ON ITS DENOMINATOR, consulted here because this is the only place+ # that can act on it. `instruments/streamcost.py` marks a window unscoreable when an SSE frame+ # failed to parse or an unterminated frame was still buffered at its close, and per the HTML+ # standard an event is dispatched only at the blank line that terminates it, so those characters+ # cannot be recovered. Every cost-per-character divided by that short delta comes out inflated.+ # `is False` and not falsiness: a payload recorded before the flag existed carries no key at all,+ # and those windows are admitted as they were rather than voided wholesale.if sc.get("reply_chars_scoreable") is False:reject(str(@@ -449,24 +429,18 @@ def _stream_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]for k, u in unit_by_key.items()}- # A CELL WHOSE RECORDER DIED IS A TRUNCATED CELL, not a short one. `rejected` above is read- # only when NOTHING qualified, so once one window has qualified every later rejection is- # discarded -- including the one that says the page went away. The streaming metrics are- # integrals divided by the characters that were streamed, and both halves are then missing the- # same unmeasured stretch, so the result is not a wide error bar, it is a number computed over- # whatever ran before the crash and reported as if it described the cell.- #+ # A CELL WHOSE RECORDER DIED IS A TRUNCATED CELL, not a short one. `rejected` above is read only+ # when NOTHING qualified, so once one window has qualified every later rejection is discarded,+ # including the one saying the page went away. The streaming metrics are integrals divided by+ # streamed characters, and both halves miss the same unmeasured stretch, so the result is not a+ # wide error bar but a number computed over whatever ran before the crash.# Measured on the payload corpus: 138 unaided windows record `unavailable` while a qualifying- # window precedes them, carrying `TargetClosedError: Page.evaluate: Target page, context or- # browser has been closed` and `Error: Page.evaluate: Target crashed`. They are all AFTER the- # last qualifying window, which is not evidence that they are ordinary end-of-stream windows:- # a crash is trailing by construction, because nothing can qualify once the page is gone. They- # are also long, a median of 11.0 s and up to 36.0 s, so what went unmeasured is a real- # fraction of the reply and not a rounding edge.- #- # `unavailable` was consumed nowhere in this module before this, so the crash signal the- # instrument already emits was being dropped in full. Poisoning here costs 12 cells of 1,461- # that currently publish, and every one of the 12 is a genuine page crash.+ # window precedes them, carrying `TargetClosedError` / `Page.evaluate: Target crashed`. They are+ # all AFTER the last qualifying window, which is not evidence they are ordinary end-of-stream+ # windows (a crash is trailing by construction) and they are long, median 11.0 s.+ # `unavailable` was consumed nowhere in this module before, so the crash signal was dropped in+ # full. Poisoning here costs 12 cells of 1,461 that currently publish, and every one is a+ # genuine page crash.crashed = sorted({str(sc.get("unavailable"))@@ -485,7 +459,7 @@ def _stream_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure]return {k: Measure.failed(u, reason) for k, u in unit_by_key.items()}# Every streaming quantity comes from the UNAIDED windows. See _unaided for why the targeted- # numerator is not exempt from that, which is the one thing here that measurement overturned.+ # numerator is not exempt, which is the one thing here that measurement overturned.unaided = _unaided(picked)chars = 0delta_task_ms = 0.0@@ -568,18 +542,13 @@ def _stream_measures(windows: Sequence[Mapping[str, Any]]) -> dict[str, Measure])if frameless:# THE SAME RULE AS `_frame_measures`, ON THE SAME SHAPE OF WINDOW, AND FOR THE SAME REASON.- #- # `instruments/frames.js` emits `frames_attempted: true` with `frame_gaps_ms: []` and a- # null `max_frame_ms` whenever the rAF loop was never scheduled in the window, and rAF- # going unscheduled is a rendering fact, not a quiet one: on a headless engine the loop- # runs off the rendering pipeline, so a renderer that stalls stops delivering callbacks- # while SSE keeps arriving. Skipped rather than refused, such a window contributes no- # deltas, no wall time and no worst frame, so the REMAINING unaided windows answer for the- # whole streaming stretch: one frozen window beside one smooth one reported a 16.7 ms worst- # frame and 0.0% time in jank, byte-identical to the cell without the freeze in it.- # An unmeasured window is not an absent one. Only the three FRAME metrics are poisoned --- # the cost, busy and character figures come from `stream_cost`, which measured this window- # perfectly well.+ # `instruments/frames.js` emits `frames_attempted: true` with an empty `frame_gaps_ms` and a null+ # `max_frame_ms` whenever the rAF loop was never scheduled, and that is a rendering fact: on a+ # headless engine the loop runs off the rendering pipeline, so a stalled renderer stops+ # delivering callbacks while SSE keeps arriving. Skipped rather than refused, one frozen window+ # beside one smooth one reported a 16.7 ms worst frame and 0.0% time in jank. Only the three+ # FRAME metrics are poisoned; cost, busy and characters come from `stream_cost`, which measured+ # the window fine.reason = (f"{frameless} unaided streaming window(s) recorded no frames at all (rAF may be ""unscheduled), so the pooled streaming frame metrics would describe only the windows "@@ -635,13 +604,11 @@ def measures_from_records(and str(w.get("kind") or "") not in UNSCORED_WINDOW_KINDS]frames = _frame_measures(windows)- # The streaming phase is NOT in METRIC_BY_KEY, and that is deliberate rather than an- # omission. The anchor table is hashed into `weights_id`, and a report that compares two- # runs with different `weights_id` values is refused; adding a seventh weighted metric- # would change every existing run's composite score and make it incomparable with every- # run this tool has already taken. So these are scored through the per-metric floor table,- # which judges a metric on its own floor, and the composite score is left alone. Giving- # them anchors and a weight is a separate decision that should be argued in anchors.py.+ # The streaming phase is NOT in METRIC_BY_KEY, deliberately. The anchor table is hashed into+ # `weights_id` and a report comparing two runs with different values is refused, so a seventh+ # weighted metric would make every existing run incomparable. These are scored through the+ # per-metric floor table instead and the composite score is left alone; giving them anchors and+ # a weight is a separate decision for anchors.py.stream = _stream_measures(windows)readings: dict[str, Measure] = {}@@ -657,8 +624,8 @@ def measures_from_records(METRIC_BY_KEY[key].unit, f"no source is wired for {key}")- # Repetitions of the same rung: keep the first and let the caller ask for reps- # explicitly. Silently averaging reps here would hide a bimodal rung.+ # Repetitions of the same rung: keep the first and let the caller ask for reps explicitly.+ # Silently averaging reps here would hide a bimodal rung.by_rung.setdefault(rung, readings)return by_rungdiff --git a/tests/studio/studiobench/scoring/payload_rules.py b/tests/studio/studiobench/scoring/payload_rules.pyindex fd2e7dface7..a002397b097 100644--- a/tests/studio/studiobench/scoring/payload_rules.py+++ b/tests/studio/studiobench/scoring/payload_rules.py@@ -125,8 +125,8 @@ def censored_metrics(records: Iterable[dict]) -> dict[str, set[str]]:if key.endswith("_censored") and value:metric = f"{action}.{key[:-len('_censored')]}_ms"out.setdefault(metric, set()).add(cell)- # The row still CARRIES the timings the scoring layer refuses to read, so the names are- # known here even though the values will never be pooled.+ # The row still CARRIES the timings the scoring layer refuses to read, so the names are known here+ # even though the values will never be pooled.if r.get("ran") and r.get("expect_ok") is False:for key in r.get("timings") or {}:out.setdefault(f"{action}.{key}", set()).add(cell)@@ -179,9 +179,8 @@ def refuse_partial_censoring(records: list[dict], metric: str) -> str | None:return Nonemeasured = measured_cells(records, metric)if not measured:- # Censored everywhere it was attempted. Nothing survived to be biased, and there is no- # pooled number to refuse -- refusing here would fire on every payload that simply cannot- # answer, which is a different and much noisier rule than the one being enforced.+ # Censored everywhere it was attempted: nothing survived to be biased and there is no pooled+ # number to refuse, so refusing here would fire on every payload that simply cannot answer.return Nonerungs_censored = sorted({c.split(".", 1)[0] for c in censored if c})rungs_measured = sorted({c.split(".", 1)[0] for c in measured if c})@@ -225,10 +224,9 @@ def comparability_key(run_meta: dict) -> str:import hashlibimport json- # COMPUTED OVER `comparability_fields`, not over a second copy of the same dict. The two were- # written out separately and had to be kept in step by hand, which is the drift this module- # exists to argue against: a field added to one and forgotten in the other would make the key- # and its own explanation disagree about what the key covers.+ # COMPUTED OVER `comparability_fields`, not a second copy of the same dict. The two were written+ # out separately and kept in step by hand, and a field added to one and forgotten in the other+ # would make the key and its own explanation disagree about what the key covers.blob = json.dumps(comparability_fields(run_meta), sort_keys = True, default = str).encode()return "cmp:" + hashlib.sha256(blob).hexdigest()[:10]@@ -296,62 +294,49 @@ def comparability_fields(run_meta: dict) -> dict:"cadence": run_meta.get("cadence"),"stream_tail_chars": run_meta.get("stream_tail_chars"),"corpus_dollars": run_meta.get("corpus_dollars"),- # THE THREE THAT CHANGE WHAT IS MEASURED, not merely what is measured on. The harness- # already treats all three as identity axes that `--resume` refuses to toggle, and the- # `run_meta` emission says why in its own words -- so a key that ignored them would call- # two payloads comparable that the tool itself refuses to continue as one run.- #- # `inject_stream_cost_ms` is the sharpest: an arm running it is not a measurement of the- # build at all, because the harness put the slowdown there on purpose. Nothing in the- # scoring path refuses an injected payload, so `--compare` blessing it against a clean run- # was a live route to publishing a difference the harness itself created.- #- # `click_probe` is normalised through `bool` so a payload written before the field existed- # reads as False rather than None; the other two are `None` in the ordinary case already.+ # THE THREE THAT CHANGE WHAT IS MEASURED, not merely what it is measured on. The harness already+ # treats all three as identity axes that `--resume` refuses to toggle, so a key that ignored them+ # would call two payloads comparable that the tool itself refuses to continue as one run.+ # `inject_stream_cost_ms` is the sharpest: an arm running it is not a measurement of the build at+ # all, because the harness put the slowdown there. Nothing in the scoring path refuses an+ # injected payload, so `--compare` blessing it against a clean run was a live route to publishing+ # a difference the harness created.+ # `click_probe` is normalised through `bool` so a payload written before the field existed reads+ # as False rather than None; the other two are already `None` in the ordinary case."click_probe": bool(run_meta.get("click_probe")),"probe_init_script": run_meta.get("probe_init_script"),"inject_stream_cost_ms": run_meta.get("inject_stream_cost_ms"),# THE HOST, which the engine alone does not stand in for. `run_meta` records `system` and# `machine` and the key took neither, so two payloads from different hardware and operating- # systems hashed the same and `--compare` blessed them.- #- # It needs no unusual invocation: `browser.default_engine()` returns webkit on Darwin AND- # on Linux, so a tester's Mac payload and the Linux dev box's payload, both with default- # settings, differ in `system`, `machine` and `engine_note` and were declared comparable.- # Only Windows was caught, and only incidentally, because its default engine differs.- #- # Nothing else in the tool refuses cross-host pooling: `floor_table.load` checks tier,- # corpus and probe only, and `platform` is not an identity axis. The sole existing- # statement is prose in `report/render.py` -- "machine-local; does not travel between- # machines" -- so the authors knew the hazard and the one guard meant to police a prose- # comparison was the place it was missing.+ # systems hashed the same and `--compare` blessed them. It needs no unusual invocation:+ # `browser.default_engine()` returns webkit on Darwin AND Linux, so a tester's Mac payload and+ # the Linux dev box's, both default, differ in `system`, `machine` and `engine_note` and were+ # declared comparable. Only Windows was caught, and only because its default engine differs.+ # Nothing else refuses cross-host pooling: `floor_table.load` checks tier, corpus and probe only,+ # and `platform` is not an identity axis. The sole existing statement is prose in+ # `report/render.py` ('machine-local; does not travel between machines') so the one guard meant to+ # police a prose comparison was the place it was missing."system": platform.get("system"),"machine": platform.get("machine"),- # AND WHICH PHYSICAL MACHINE, because the two fields above cannot say. `platform.machine()`- # is the ARCHITECTURE, not a machine identifier: on two ordinary Linux x86_64 hosts it- # returns `x86_64` on both while `system` returns `Linux` on both, so the pair that was- # added here to stand for the host caught only the cross-OS case -- a tester's Mac against- # the Linux dev box -- and left the commonest one, a dev box against a CI runner or a- # second dev box, hashing identically. `platform.node()` is the host's network name and is- # the field that separates them.- #- # This is the axis with the least slack in it: `floor_table.render` refuses a floor whose- # comparability fields differ from the payload's, in the words "a floor is the scatter of- # THIS measurement on THIS machine", and that refusal is computed from this dict. Without- # the host in it a null control measured on one machine certifies a result measured on- # another, which is the one comparison the report text says never travels.- #- # A payload recorded before this field existed carries None and is therefore not comparable- # with one that carries a host. That is the honest reading rather than a cost: such a- # payload does not record which machine produced it, so nothing can show it was this one.+ # AND WHICH PHYSICAL MACHINE, because the two fields above cannot say. `platform.machine()` is the+ # ARCHITECTURE: on two ordinary Linux x86_64 hosts it returns `x86_64` and `system` returns+ # `Linux` on both, so the pair added to stand for the host caught only the cross-OS case and left+ # the commonest one, a dev box against a CI runner, hashing identically. `platform.node()` is the+ # host's network name and is what separates them.+ # This is the axis with the least slack: `floor_table.render` refuses a floor whose comparability+ # fields differ from the payload's, in the words 'a floor is the scatter of THIS measurement on+ # THIS machine', and that refusal is computed from this dict. Without the host a null control+ # measured on one machine certifies a result measured on another.+ # A payload recorded before this field existed carries None and is therefore not comparable with+ # one that carries a host. That is the honest reading: such a payload does not record which+ # machine produced it."node": platform.get("node"),- # WHICH BROWSER BINARY DREW THE FRAMES, which `engine` does not settle. Since Playwright- # 1.57 headed and headless default to different executables -- `chrome` against- # `chrome-headless-shell` -- and headless falls back to software rendering for- # GPU-accelerated work while its compositor keeps its own pacing. For a tool whose output- # is frames, jank and time-to-settle, those are two different renderers under one engine- # name. Normalised through `bool` so a payload written before the field existed reads as- # the headless default rather than as None.+ # WHICH BROWSER BINARY DREW THE FRAMES, which `engine` does not settle: since Playwright 1.57+ # headed and headless default to different executables (`chrome` against+ # `chrome-headless-shell`), and headless falls back to software rendering while its compositor+ # keeps its own pacing. For a tool whose output is frames, jank and time-to-settle those are two+ # renderers under one engine name. Normalised through `bool` so a pre-field payload reads as the+ # headless default rather than None."headed": bool(run_meta.get("headed")),}diff --git a/tests/studio/studiobench/scoring/schema.py b/tests/studio/studiobench/scoring/schema.pyindex 7e67bbcd1b2..226a9fc76de 100644--- a/tests/studio/studiobench/scoring/schema.py+++ b/tests/studio/studiobench/scoring/schema.py@@ -38,36 +38,28 @@ from typing import Any, Iterable, Mapping, SequenceMEASURE_KIND = "measure"-#: Keys whose numeric zero is a genuine, meaningful zero rather than a missing measurement.-#: Everything here is an index, an identifier, a declared constant or a cardinality that is-#: interesting precisely when it is zero. Anything not on this list must be a `Measure`.+#: Keys whose numeric zero is a genuine zero rather than a missing measurement: indexes,+#: identifiers, declared constants and cardinalities that are interesting precisely when zero.+#: Anything not listed must be a `Measure`.ZERO_OK_KEYS = frozenset({"index",- # The parity capture in scene/parity.js numbers its message rows with a bare loop- # counter (`i`), not `index`, so the first message of every capture is `i: 0`. That is- # an ordinal like every other name on this list, and rejecting it made `--report` fail- # on any payload that carried parity rows at all.+ # The parity capture in scene/parity.js numbers its message rows with a bare loop counter (`i`),+ # so the first message of every capture is `i: 0`. Rejecting that ordinal made `--report` fail on+ # any payload carrying parity rows."i",+ # The visible-region capture, whose zeros are all true statements. NOT the whole `visible`+ # subtree: exempting a subtree here once defeated a test requiring a zero-length `chars` inside+ # `parity` to stay loud, and a visible message with a zero-character signature is a broken+ # capture. Only the counters are named: no message shown was unmounted, a windowed arm unmounted+ # them all (NOT COMPARABLE), or the scan observed nothing, which compare_visible refuses.+ # The keys are `unmounted_at_capture`, `mounted_ever_visible` and `ever_visible_count`.+ # The SSE wire's integrity counters, whose zeros are the good outcome, again the scalars and not+ # the `wire` subtree. 'No frame failed to parse' and 'no unterminated frame was left buffered'+ # are what make the character count trustworthy. `wireChars` is NOT listed: a zero there means+ # nothing was counted and must stay loud.# ── the visible-region capture, whose zeros are all true statements ──- #- # NOT the whole `visible` subtree, deliberately. Exempting a subtree here once already- # defeated a test that required a zero-length `chars` inside `parity` to stay a loud- # failure, and the same argument applies with more force to this one: a visible message- # whose signature is zero characters long is a broken capture and must stay loud. Only the- # counters are named, and each is a fact rather than a measurement:- # unmounted_at_capture: 0 every message the viewport showed was still mounted- # mounted_ever_visible: 0 a windowed arm had unmounted them all again, which the- # analysis reports as NOT COMPARABLE rather than as agreement- # ever_visible_count: 0 the scan observed nothing, which the positive control in- # compare_visible refuses; it is not scored anywhere# ── the SSE wire's integrity counters, whose zeros are the good outcome ──- #- # Again the scalars and not the `wire` subtree. "No frame failed to parse" and "no- # unterminated frame was left buffered" are the statements that make the character count- # trustworthy, and a run where they are zero is the run whose denominator can be scored.- # `wireChars` itself is NOT on this list: a zero there means nothing was ever counted and- # must stay loud."wire_parse_failures","wire_pending_chars","wire_parse_failures_in_window",@@ -97,9 +89,8 @@ ZERO_OK_KEYS = frozenset("records_written","frames_total", # a window with no frames is a SYMPTOM; frames.py already flags it"n_pairs",- # -- harness-row counters whose zero is the GOOD result, not a missing reading. Each of- # these is written unconditionally by the session layer for every cell, so "absent" and- # "zero" are already distinguishable: absent means the row was never written at all.+ # harness-row counters whose zero is the GOOD result, not a missing reading. Each is written+ # unconditionally per cell, so absent means the row was never written at all."slots_missed", # 0 means the film kept time, which is exactly what we want to read"expect_failures", # 0 means every action's expectation held"over_budget_ms", # 0 means the action fitted inside its slot@@ -107,26 +98,24 @@ ZERO_OK_KEYS = frozenset("seeded_messages","seed_seconds","bursts", # a pacer that never had to burst is a pacer that was never jammed- # Seeded-vs-streamed equivalence: 0 drift on a field is the PASSING result, and it is the- # single most load-bearing zero in the payload, because it is what licenses every rung- # above 10K to be seeded rather than streamed.+ # Seeded-vs-streamed equivalence: 0 drift on a field is the PASSING result, and the most+ # load-bearing zero in the payload, because it licenses every rung above 10K to be seeded rather+ # than streamed."drift",- # The two counts `drift` is computed FROM, in that same block. They are censuses of the- # DOM in each arm, so a field that is absent from both reads 0/0 and is the passing- # result: on a real quick-tier run `reasoning_spans` is 0 streamed and 0 seeded, and the- # payload marks the field `gating: false` in the same breath. Exempting the quotient but- # not the two numbers under it failed every payload that carried an equivalence block.+ # The two counts `drift` is computed FROM. They are censuses of the DOM in each arm, so a field+ # absent from both reads 0/0 and passes: on a real quick-tier run `reasoning_spans` is 0 streamed+ # and 0 seeded with `gating: false`. Exempting the quotient but not the two numbers failed every+ # payload carrying an equivalence block."streamed","seeded",})-#: Subtrees the bare-zero walker does not descend into. These hold configuration, identity,-#: histogram buckets and raw instrument bookkeeping, none of which is a measurement of the app.-#: The boundary is a contract with the harness layer: anything that is a MEASUREMENT goes under-#: a `metrics` key as a measure object, anything that is bookkeeping goes under `info`. Without-#: that split the walker either fails on every legitimate counter or is switched off entirely,-#: and switched off is how the ban stops existing.+#: Subtrees the bare-zero walker does not descend into: configuration, identity, histogram buckets+#: and raw instrument bookkeeping, none of which measures the app. The boundary is a contract with+#: the harness layer (MEASUREMENTS under `metrics` as measure objects, bookkeeping under `info`)+#: and without it the walker either fails on every legitimate counter or gets switched off, which+#: is how the ban stops existing.EXEMPT_SUBTREE_KEYS = frozenset({"info",@@ -135,21 +124,18 @@ EXEMPT_SUBTREE_KEYS = frozenset("env","identity","header",- # The comparability key's own section. `fields` is the identity block the key is hashed- # over -- `instrument_level`, `stream_tail_chars`, `corpus_dollars` -- every one of which- # is a true statement about how the run was configured rather than a measurement of the- # app, and every one of which is legitimately 0. Same rule as `identity` and `config`- # directly above.+ # The comparability key's own section. `fields` is the identity block the key is hashed over+ # (`instrument_level`, `stream_tail_chars`, `corpus_dollars`), every one a true statement about+ # how the run was configured and legitimately 0. Same rule as `identity` and `config` above."comparability","footer","record_counts","histogram","potency_counters",- # -- OBSERVATION subtrees written by the session layer. These are censuses of the DOM and- # the expectations checked against them: "0 code blocks" is a true statement about the- # page at that instant, not a failed measurement. They are never scored and never quoted- # as a cost; everything that IS scored reaches the report as a Measure via- # scoring/from_payload.py, and that path stays under the strict rule.+ # OBSERVATION subtrees written by the session layer: censuses of the DOM and the expectations+ # checked against them, where '0 code blocks' is a true statement about the page rather than a+ # failed measurement. They are never scored or quoted as a cost; everything scored reaches the+ # report as a Measure via scoring/from_payload.py, under the strict rule."census","census_before","census_after",@@ -157,11 +143,10 @@ EXEMPT_SUBTREE_KEYS = frozenset("streamed_census","seeded_census","expect",- # The readiness gate's verdict and the completeness probe's, both written once per cell by- # the session layer. Same rule as a census: `from_bottom: 0` is a true statement that the- # viewport is exactly at the bottom, `waited_ms: 0` is a thread that was ready on the- # first sample, and neither is a measurement of the app. Nothing here is scored; what the- # gate produces for the report is a `gate` row, which carries a boolean.+ # The readiness gate's verdict and the completeness probe's, written once per cell. Same rule as+ # a census: `from_bottom: 0` says the viewport is exactly at the bottom and `waited_ms: 0` a+ # thread ready on the first sample. Nothing here is scored; what the gate produces for the+ # report is a `gate` row carrying a boolean."readiness","completeness","notes",@@ -207,8 +192,6 @@ class Measure:if not self.attempted and not self.note:raise PayloadSchemaError("a not-attempted Measure must say why")- # -- constructors --------------------------------------------------------------------@classmethoddef not_attempted(cls, unit: str, reason: str) -> "Measure":return cls(value = None, attempted = False, unit = unit, note = reason)@@ -228,8 +211,6 @@ class Measure:) -> "Measure":return cls(value = float(value), attempted = True, unit = unit, floor = floor, note = note)- # -- predicates ----------------------------------------------------------------------@propertydef has_reading(self) -> bool:return self.attempted and self.value is not None@@ -251,8 +232,8 @@ class Measure:if self.value is None:return f"no reading ({self.note or 'instrument failed'})"if self.sub_floor:- # A negative reading under the floor is bounded from below, not from above; printing- # it as `< floor` would read as "small and positive" for a value that is neither.+ # A negative reading under the floor is bounded from below, not above; printing it as `< floor`+ # would read as 'small and positive' for a value that is neither.if float(self.value) < 0:return f"> -{_fmt(self.floor)} {self.unit} (instrument floor)"return f"< {_fmt(self.floor)} {self.unit} (instrument floor)"@@ -355,7 +336,7 @@ class ExcludedCell:}-#: Reasons a cell may be excluded. Free-text reasons are refused so the report can total them.+#:Reasons a cell may be excluded. Free-text reasons are refused so the report can total them.EXCLUSION_REASONS = frozenset({"clock_disagreement",@@ -382,8 +363,9 @@ def check_exclusion_reasons(cells: Iterable[ExcludedCell]) -> None:)-# ---------------------------------------------------------------------------------------# payload validation++# ---------------------------------------------------------------------------------------@@ -428,22 +410,19 @@ def _is_number_list(node: Any) -> bool:def _walk_for_bare_zeros(node: Any, path: str, problems: list[str]) -> None:if _is_measure(node):- # A measure object is self-describing; its own zero is fine because `attempted` sits- # next to it. Do not descend, the inner `value` is exactly the thing being exempted.+ # A measure object is self-describing; its own zero is fine because `attempted` sits next to it.+ # Do not descend, the inner `value` is exactly what is exempted.if node.get("attempted") is None:problems.append(f"{path}: measure object without `attempted`")returnif isinstance(node, Mapping):- # The harness layer cannot emit measure objects (its rows are JSON-safe scalars), so it- # attests with a `<name>_attempted` flag. That flag is written ONCE PER INSTRUMENT BLOCK,- # not once per counter: the frames instrument emits `frames_attempted: true` alongside- # `frames_over_33`, `long_tasks` and the rest. An exact-name-sibling rule would demand- # `frames_over_33_attempted` and fail on every real row, and a check that fails on every- # real row gets switched off, which is how the ban stops existing.- #- # So a mapping that positively attests (any `*_attempted` key that is True) covers the- # numeric zeros DIRECTLY inside it. It does not cover nested mappings, which carry their- # own attestation or none.+ # The harness layer cannot emit measure objects (its rows are JSON-safe scalars), so it attests+ # with a `<name>_attempted` flag written ONCE PER INSTRUMENT BLOCK, not per counter:+ # `frames_attempted: true` sits alongside `frames_over_33`, `long_tasks` and the rest. An+ # exact-name-sibling rule would demand `frames_over_33_attempted` and fail on every real row, and+ # a check that fails on every real row gets switched off.+ # So a mapping that positively attests covers the numeric zeros DIRECTLY inside it, not nested+ # mappings, which carry their own attestation or none.attested = any(k.endswith("_attempted") and v is True for k, v in node.items())for key, child in node.items():if key in EXEMPT_SUBTREE_KEYS:@@ -456,13 +435,10 @@ def _walk_for_bare_zeros(node: Any, path: str, problems: list[str]) -> None:and covered):continue- # The same attestation, for a plain numeric ARRAY directly inside the block. An- # instrument that attests reports samples as well as counters: `frames` writes- # `frames_attempted: true` next to `frame_gaps_ms`, whose entries are inter-frame- # gaps in whole milliseconds, so two frames in the same millisecond record a- # legitimate 0. Walking into the list dropped the attestation on the floor and every- # such run failed validation, which is the scalar case one line above with an extra- # pair of brackets around it.+ # The same attestation for a plain numeric ARRAY directly inside the block: `frames` writes+ # `frames_attempted: true` next to `frame_gaps_ms`, whose entries are whole-millisecond gaps, so+ # two frames in one millisecond record a legitimate 0. Walking into the list dropped the+ # attestation and failed every such run.if covered and _is_number_list(child):continue_walk_for_bare_zeros(child, f"{path}.{key}", problems)diff --git a/tests/studio/studiobench/scoring/score.py b/tests/studio/studiobench/scoring/score.pyindex d6833463918..015ef5799ac 100644--- a/tests/studio/studiobench/scoring/score.py+++ b/tests/studio/studiobench/scoring/score.py@@ -102,16 +102,16 @@ def score_metric(anchor: MetricAnchor, measure: Measure) -> MetricScore:)value = float(measure.value)- # A sub-floor reading is at least as good as the floor; scoring the raw value would let- # instrument noise on a fast machine invent a difference between two perfect builds.+ # A sub-floor reading is at least as good as the floor; scoring the raw value would let instrument+ # noise on a fast machine invent a difference between two perfect builds.if measure.sub_floor and measure.floor is not None and anchor.lower_is_better:value = min(abs(value), float(measure.floor))good, bad = float(anchor.good), float(anchor.bad)if value <= 0:- # Log space has no zero. A non-positive reading on a positive-only metric is either- # perfect (below every floor) or a broken instrument; the floor logic above has already- # handled the honest case, so clamp to the good anchor and say so.+ # Log space has no zero. A non-positive reading on a positive-only metric is either perfect (below+ # every floor) or a broken instrument, and the floor logic above has handled the honest case, so+ # clamp to the good anchor and say so.value = good if anchor.lower_is_better else badspan = math.log(bad) - math.log(good)@@ -202,8 +202,8 @@ def score_rung(zeroed_by = [m.key for m in scored if float(m.score) <= 0.0]if zeroed_by:- # Geometric mean of a set containing zero IS zero. Written out rather than left to- # log(0) so the reason travels with the number.+ # Geometric mean of a set containing zero IS zero. Written out rather than left to log(0) so the+ # reason travels with the number.rung_score = 0.0else:numerator = sum(m.weight * math.log(float(m.score)) for m in scored)@@ -314,9 +314,9 @@ def score_ladder(rungs: Sequence[RungScore]) -> LadderScore:"is above what this ladder measures")- # Usability is expected to be monotone in thread size. When it is not, something other than- # thread size moved -- thermal throttling, a background process, an unstable machine -- and- # the onset rung is not trustworthy on its own. Say so rather than quietly reporting the max.+ # Usability is expected to be monotone in thread size. When it is not, something other than thread+ # size moved (throttling, a background process, an unstable machine) and the onset rung is not+ # trustworthy on its own.non_monotonic = Falseseen_unusable = Falsefor rung in ordered:diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.pyindex bc1aa8fca92..eeaca27ecce 100644--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_from_payload.py@@ -258,8 +258,8 @@ def test_real_payload_shape_round_trips(tmp_path):ladder = score_payload(path, [1_000])assert [r.tokens for r in ladder.rungs] == [1_000]rung = ladder.rungs[0]- # keystroke + menu + the three frame metrics = 85% of the weight, over the 60% floor, so the- # rung scores despite scroll being legitimately unavailable at this size.+ # keystroke + menu + the three frame metrics = 85% of the weight, over the 60% floor, so the rung+ # scores despite scroll being legitimately unavailable at this size.assert rung.usable is True@@ -476,16 +476,14 @@ def _payload_with(cell):}-# ----------------------------------------------------------------------------------------# The latest attempt is the last one that WROTE anything, not the last one that finished-# ----------------------------------------------------------------------------------------#+# The latest attempt is the last one that WROTE anything, not the last one that finished.# `CellRunner.run` writes its terminal cell row in a `finally`, which a SIGKILL, an OOM kill or a-# lost machine never reaches -- while the Recorder has already flushed and fsynced every action and+# lost machine never reaches, while the Recorder has already flushed and fsynced every action and# window row before it. Keyed on cell rows alone, a resume hard-killed inside a cell left the# older, completed attempt named as the latest, and `_resume_set` skipped it.+# ---------------------------------------------------------------------------------------def _stamped(row, session):return {**row, "session_id": session}diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.pyindex f7ebe445630..5d5d0207ea5 100644--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_payload_rules.py@@ -287,10 +287,10 @@ def test_the_key_is_computed_over_the_fields_it_explains():"""corpus = "ac9d5d8e37be2a3844deed559fde6070247ad2322377295fb383b60b5eec5a0c"a = _meta(corpus)- #: Fields the key reads out of the nested `platform` dict rather than off the row itself.- #: Written as a set rather than a single special case, because the single special case is what- #: had to be edited when `system` and `machine` were added -- and a test that needs editing to- #: keep passing when a field is added is a test that can be edited into silence instead.+ #: Fields the key reads out of the nested `platform` dict rather than off the row itself. Written as+ #: a set rather than a single special case, because the special case is what had to be edited when+ #: `system` and `machine` were added, and a test that needs editing to keep passing can be edited+ #: into silence instead.nested = {"engine", "system", "machine", "node"}for field in payload_rules.comparability_fields(a):b = _meta(corpus)diff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.pyindex 58c8f8a9533..5db08304371 100644--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_scoring.py@@ -45,11 +45,10 @@ from studiobench.scoring import ( # noqa: E402from studiobench.scoring.anchors import ONSET_SCORE_THRESHOLD # noqa: E402-# ---------------------------------------------------------------------------------------# Measure: the ban on bare zeros-# ---------------------------------------------------------------------------------------+# ---------------------------------------------------------------------------------------def test_not_attempted_cannot_carry_a_value():with pytest.raises(PayloadSchemaError):Measure(value = 1.0, attempted = False, unit = "ms", note = "nope")@@ -214,11 +213,10 @@ def test_validate_payload_requires_excluded_cells():validate_payload({"excluded_cells": None})-# ---------------------------------------------------------------------------------------# frames: both directions of jank-# ---------------------------------------------------------------------------------------+# ---------------------------------------------------------------------------------------def test_uniform_mediocrity_is_caught_by_time_in_jank_and_missed_by_max():"""Every frame 120 ms. `max` says 120, which sounds survivable. It is not."""@@ -260,11 +258,10 @@ def test_histogram_is_always_present_and_totals_the_frames():assert sum(b["bucket_count"] for b in stats.histogram) == len(deltas)-# ---------------------------------------------------------------------------------------# per-metric and per-rung scoring-# ---------------------------------------------------------------------------------------+# ---------------------------------------------------------------------------------------def test_log_anchors_put_the_geometric_midpoint_at_fifty():anchor = METRIC_BY_KEY["keystroke_p95_ms"]midpoint = math.sqrt(anchor.good * anchor.bad)@@ -332,11 +329,10 @@ def test_missing_most_metrics_makes_the_rung_incomplete_rather_than_easy():assert "weight" in rung.incomplete_reason-# ---------------------------------------------------------------------------------------# aggregation, and the three ways naive AUC lies-# ---------------------------------------------------------------------------------------+# ---------------------------------------------------------------------------------------def test_log_rung_weights_do_not_let_the_top_rung_be_the_whole_score():weights = log_rung_weights([1_000, 10_000, 100_000, 500_000, 1_000_000])assert sum(weights) == pytest.approx(1.0)@@ -409,11 +405,10 @@ def test_non_monotone_usability_is_flagged_rather_than_maximised():assert len(rungs) == 3-# ---------------------------------------------------------------------------------------# A/B-# ---------------------------------------------------------------------------------------+# ---------------------------------------------------------------------------------------def _identity(session: str = "s1", **overrides) -> RunIdentity:fields = {"bench_version": "studiobench/1",@@ -537,17 +532,15 @@ def test_bootstrap_ci_brackets_the_geometric_mean():assert metric.ci_low <= metric.ratio_geomean <= metric.ci_high-# ----------------------------------------------------------------------------------------# A CI that contains 1.0 is not a result: the pairs have to agree on the sign-# ----------------------------------------------------------------------------------------#-# The noise floor is a fact about the HARNESS -- whether a difference of this size is resolvable-# here at all -- and it was the only gate a direction had to pass. Repetitions that disagree-# produce a geometric mean well clear of the floor anyway: 0.7, 0.7, 1.2, 1.2 averages to 0.917-# with a CI of 0.700-1.200, and the table said "improved" over it, in the one column anybody-# quotes. The CI was computed, printed, and never consulted.+# A CI that contains 1.0 is not a result: the pairs have to agree on the sign. The noise floor is a+# fact about the HARNESS, whether a difference of this size is resolvable at all, and it was the+# only gate a direction had to pass. Repetitions that disagree produce a geometric mean well clear+# of the floor anyway: 0.7, 0.7, 1.2, 1.2 averages to 0.917 with a CI of 0.700-1.200, and the+# table said 'improved' over it in the one column anybody quotes. The CI was computed, printed,+# and never consulted.+# ---------------------------------------------------------------------------------------def _split_pairs(metric: str, ratios: list[float]) -> list[Pair]:"""One metric, one ratio per rung, so the pairs can be made to disagree on the sign."""out = []@@ -635,22 +628,16 @@ def test_the_table_does_not_print_a_direction_it_could_not_resolve():assert "contains it" in text # the interval straddles 1.0, as opposed to being absent-# ----------------------------------------------------------------------------------------# A measured zero is a reading: sub-floor arms bound the ratio rather than voiding the pair-# ----------------------------------------------------------------------------------------#-# `Pair.usable` required both values to be strictly positive. `time_in_jank_pct` and `jank_index`-# are 0.0 on any arm smooth enough to have no over-budget frames, which is the ordinary state of a-# healthy base, so a treatment that introduced jank over a zero-jank base had its pair dropped:-# the table said `no reading` about two arms that had both read, the regression was absent from-# `regressions` and from the headline, and as a null control it neither voided nor reached the-# noise floor derived from it.-#-# The rule applied here is the one `score.py` has always applied to the same reading: a sub-floor-# value is at least as good as the floor, so the floor is what enters the ratio. What that must-# NOT do is admit a reading that was never taken, which is the distinction `frames.py` protects-# when it refuses to score an unscheduled rAF loop as zero jank.+# A measured zero is a reading: sub-floor arms bound the ratio rather than voiding the pair.+# `Pair.usable` required both values to be strictly positive, but `time_in_jank_pct` and+# `jank_index` are 0.0 on any arm smooth enough to have no over-budget frames, so a treatment that+# introduced jank over a zero-jank base had its pair dropped: the table said `no reading` about+# two arms that had both read. The rule applied here is the one `score.py` has always applied, a+# sub-floor value is at least as good as the floor so the floor enters the ratio, without+# admitting a reading that was never taken, the distinction `frames.py` protects when it refuses+# to score an unscheduled rAF loop as zero jank.+# ---------------------------------------------------------------------------------------JANK_FLOOR = 0.1SMOOTH = Measure.read(0.0, "%", floor = JANK_FLOOR)JANKY = Measure.read(5.0, "%", floor = JANK_FLOOR)@@ -665,8 +652,8 @@ def test_a_zero_jank_base_still_pairs_against_a_treatment_that_introduced_jank()assert pair.base.has_reading and pair.treatment.has_readingassert pair.usable is True- # The floor stands in for the sub-floor arm, so the ratio is a LOWER bound: the true- # magnitude is larger, never smaller.+ # The floor stands in for the sub-floor arm, so the ratio is a LOWER bound: the true magnitude is+ # larger, never smaller.assert pair.ratio == 50.0assert pair.bounded is Trueassert pair.to_json()["bounded"] is Truediff --git a/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py b/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.pyindex b5328ee2e68..fe28a10bb73 100644--- a/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py+++ b/tests/studio/studiobench/scoring/selftest/test_studiobench_stream_metrics.py@@ -368,9 +368,9 @@ def test_an_action_running_during_generation_does_not_set_the_worst_streaming_frm = _stream_measures(windows)assert m["stream_max_frame_ms"].value == pytest.approx(286.0)# And its SSE task chains are excluded from the targeted numerator too. Measured on a- # standard-tier 10K null, an `action:keystroke` chain cost 23.77 ms per burst against 1.69 ms- # in the gap windows either side: the chain runs until the event loop drains, so the typing- # lands inside it and is billed to the stream.+ # standard-tier 10K null, an `action:keystroke` chain cost 23.77 ms per burst against 1.69 ms in+ # the gap windows either side: the chain runs until the event loop drains, so the typing lands+ # inside it and is billed to the stream.assert "1 unaided streaming window(s)" in m["stream_delta_cost_ms_per_kchar"].note@@ -473,8 +473,8 @@ def test_a_recorder_that_died_partway_poisons_every_streaming_metric():m["stream_cost_ms_per_kchar"].note or "")- # The giveaway: without this the crashed cell was byte-identical to the one that never- # crashed, so nothing downstream could tell a truncated run from a complete one.+ # The giveaway: without this the crashed cell was byte-identical to the one that never crashed, so+ # nothing downstream could tell a truncated run from a complete one.assert _stream_measures([good])["stream_cost_ms_per_kchar"].value is not Nonediff --git a/tests/studio/studiobench/sweep/floor_table.py b/tests/studio/studiobench/sweep/floor_table.pyindex 34c42d2790c..3d814067dc7 100644--- a/tests/studio/studiobench/sweep/floor_table.py+++ b/tests/studio/studiobench/sweep/floor_table.py@@ -35,9 +35,8 @@ import sysfrom pathlib import Pathif __package__ in (None, ""): # pragma: no cover- # Running the file directly rather than as a module. Supported because the first thing a new- # contributor does with a script is run it by path, and failing there with an import error is a- # bad first minute.+ # Running the file directly rather than as a module, because the first thing a new contributor+ # does with a script is run it by path.sys.path.insert(0, str(Path(__file__).resolve().parents[4]))from tests.studio.studiobench.runtime.ab import failed_invalidating_gates # noqa: E402@@ -56,28 +55,16 @@ from tests.studio.studiobench.scoring import payload_rules # noqa: E402METRICS = tuple(ACTION_SOURCES) + FRAME_METRICS + STREAM_METRICS-# The metrics compared by DIFFERENCE rather than by ratio, because zero is their CLEAN reading.-#-# Everything else here is a duration, and a duration is zero only when nothing happened. Jank is-# the opposite: a stream that never dropped a frame reports exactly 0.0, and that is the reading a-# good build is supposed to produce. Measured on this repository's recorded payloads,-# `stream_time_in_jank_pct` is exactly 0.0 on 765 of 1,438 scored cells, and 349 of 668 base-vs--# treatment pairs across 117 payload files have a zero base arm.-#-# Paired as a ratio those pairs cannot survive: `t / 0.0` is undefined, so `paired` dropped them,-# and dropping them is not a conservative choice. It removes exactly the comparisons where the base-# was CLEAN -- the ones a treatment that introduces jank shows up in -- so a regression from 0.0%-# to 12.0% left no row at all, and a run where only some repetitions had a zero base kept the rest-# under the same `n`, pooling a metric over a set whose membership depends on the base arm's value.-#-# A difference is defined from zero, so these are pooled as `treatment - base` in the metric's own-# unit, for EVERY pair rather than only the zero ones: mixing the two within one mean would be a-# second unstable pool. The null control's floor is computed the same way, so the gates compare-# like with like, and the table marks these rows so the column is not read as a percentage change.-#-# Scoped to the two streaming metrics this file has just started harvesting. `time_in_jank_pct`-# has the same shape and loses 3% of its pairs, but it is already published against ratio-based-# floors, and re-basing it would silently re-score every table taken so far.+# The metrics compared by DIFFERENCE rather than ratio, because zero is their CLEAN reading:+# `stream_time_in_jank_pct` is exactly 0.0 on 765 of 1,438 scored cells. Paired as a ratio+# `t / 0.0` is undefined, so `paired` dropped exactly the comparisons where the base was+# clean, the ones a treatment that introduces jank shows up in. Pooled as `treatment - base`+# for EVERY pair rather than only the zero ones, since mixing the two in one mean would be a+# second unstable pool; the null control's floor is computed the same way and the table marks+# these rows. Scoped to the two streaming metrics: `time_in_jank_pct` has the same shape but+# is already published against ratio-based floors.+# And 349 of 668 pairs have a zero base arm.+# Across 117 payload files.DIFFERENCE_METRICS: frozenset[str] = frozenset({"stream_time_in_jank_pct", "stream_jank_index"})@@ -106,15 +93,11 @@ def _action_timings(records: list[dict], cid: str) -> dict[str, float]:for key, value in (row.get("timings") or {}).items():if isinstance(value, (int, float)) and not isinstance(value, bool):out[f"{name}.{key}"] = float(value)- # Correctness invariants, harvested alongside the timings and named apart from them.- #- # Same paired arithmetic, opposite meaning. A timing falling is the result a change is- # trying to produce; a count falling is a regression, and one that no timing can reveal.- # `select_all_copy.count.selected_chars` is the case this exists for: the selection is- # taken over the viewport's DOM, so anything that stops mounting the whole thread- # truncates the clipboard while every timing improves and the action still reports- # `expect_ok`. Reading it against the other arm needs no calibration, because both arms- # seed a byte-identical thread.+ # Correctness invariants, harvested alongside the timings and named apart from them: same+ # paired arithmetic, opposite meaning, because a count falling is a regression no timing can+ # reveal. `select_all_copy.count.selected_chars` is the case this exists for: the selection is+ # taken over the viewport's DOM, so anything that stops mounting the whole thread truncates+ # the clipboard while every timing improves and the action still reports `expect_ok`.for key, value in (row.get("counts") or {}).items():if isinstance(value, (int, float)) and not isinstance(value, bool):out[f"{name}.count.{key}"] = float(value)@@ -148,33 +131,20 @@ def refuse_collisions(records: list[dict]) -> None:if not collided:return# A SEQUENTIAL RE-RUN IS NOT A COLLISION, and refusing it rejected every resumed A/B.- #- # `ab.skippable_cells` skips a repetition only when EVERY arm of it completed, and an A/B with- # any work left re-runs EVERY pair in one new session -- deliberately, because a lone arm- # measured in a new session can never be paired and a partial table publishes a verdict over- # whichever rungs survived. So a resumed comparison legitimately writes a SECOND completed row- # under the same deterministic `cell_id`, and the writer's own docstring says what happens- # next: "The old attempt stays in the payload and `latest_attempt_rows` supersedes it."- # Nothing here superseded anything, so a resumed run could not be scored at all.- #- # WHAT IS ASKED IS WHETHER THE TWO SESSIONS WERE RUNNING AT ONCE, which is the property that- # makes the readings contended, and it is recorded: each session's `run_meta` carries- # `started_at` and every row carries `ts_ms` from that session's own clock. On the real- # two-launcher payload of defect 9 the two sessions occupy 23:50:24-00:10:03 and- # 23:53:45-00:12:43, sixteen minutes of overlap. A resume starts after its predecessor stopped.- #- # SILENCE IS NOT A LICENCE. A payload that does not carry the evidence -- no `started_at`, no- # `ts_ms` -- is refused exactly as before, because "no proof they overlapped" is not "proof- # they did not", and this is the guard that stands between a reader and a blend of two- # contending runs. Only a payload that can SHOW its sessions were sequential is superseded.+ # `ab.skippable_cells` re-runs EVERY pair of an unfinished A/B in one new session, so a+ # resumed comparison legitimately writes a SECOND completed row under the same `cell_id` for+ # `latest_attempt_rows` to supersede. What is asked is whether the two sessions were running+ # AT ONCE, which is recorded: each `run_meta` carries `started_at` and every row a `ts_ms` (on+ # the real two-launcher payload of defect 9 the sessions overlap by sixteen minutes). Silence+ # is not a licence: a payload carrying no such evidence is refused exactly as before.guilty: set[str] = set()for sessions in collided.values():guilty |= sessionsverdict, both = concurrent_sessions(records, only = guilty)if verdict == "sequential":- # The caller supersedes through `latest_attempt_rows`, which keeps the LAST attempt that- # wrote anything -- so a resume that was itself hard-killed leaves an incomplete cell and- # drops out, rather than resurrecting the older completed reading.+ # The caller supersedes through `latest_attempt_rows`, which keeps the LAST attempt that wrote+ # anything, so a resume that was itself hard-killed drops out rather than resurrecting the+ # older completed reading.returnif verdict == "overlap":why = (@@ -333,63 +303,31 @@ def cell_metrics(records: list[dict], session: str | None = None) -> dict[str, d"""if session is None:refuse_collisions(records)- # SUPERSEDED ATTEMPTS ARE DROPPED, because a resumed A/B re-runs a repetition that had- # already completed and both copies are in the file. Keyed on the LAST attempt that wrote- # anything rather than the last that FINISHED: a resume killed inside a cell has flushed- # its action rows and never reaches the cell row its `finally` would write, and treating- # the older completed attempt as current would hand a stale reading to a floor. Dropped- # here, that repetition has no completed cell and falls out of the table instead.- #- # ONLY WHEN NO SESSION WAS NAMED. `session=` is the escape hatch this module's own refusal- # points a reader at -- "pass `session=` to score one of them" -- and superseding inside it- # would empty the older of two concurrent sessions and take the hatch away. A caller who- # names a session is asking for that session, not for the payload's latest view of it.+ # SUPERSEDED ATTEMPTS ARE DROPPED, because a resumed A/B re-runs a completed repetition and+ # both copies are in the file. Keyed on the LAST attempt that wrote anything rather than the+ # last that FINISHED: a resume killed inside a cell has flushed its action rows and never+ # reaches the cell row, and treating the older completed attempt as current would hand a stale+ # reading to a floor. ONLY WHEN NO SESSION WAS NAMED: `session=` is the escape hatch this+ # module's refusal points a reader at, and superseding inside it would empty the older of two+ # concurrent sessions.records = list(latest_attempt_rows(records))# A CELL THAT FAILED AN INVALIDATING GATE IS NOT A READING, HERE EITHER. Both gates are- # advisory where they are emitted, so such a cell arrives `completed=True` with a full set of- # timings that are CHEAPER than a correct cell's, and this table is where a floor is built and- # where a result is scored against it. `paired` divides treatment by base per pair, so a- # one-sided failure is not cancelled by the null control being same-build: it lands in- # `delta_pct` and `spread_pct` directly. On the result side it is worse, because a gate-failed- # treatment cell pairs against a clean base cell and prints as `faster`.- #- # This is not hypothetical. The virtualization negative result recorded in CONTRIBUTING-perf.md- # was measured through this workflow and contains both facts at once: the treatment arm's- # census going from 12 mounted messages to 0 and never recovering, and a headline of 28.2%- # faster weighted. Nothing between the two noticed.- #- # Scoped by hand, because a gate row is not one of the row types `latest_attempt_rows` covers:- # that reduction has already dropped the superseded attempt's cells, actions and windows above,- # and a `gate` row carries no attempt of its own to supersede.+ # advisory where they are emitted, so such a cell arrives `completed=True` with timings+ # CHEAPER than a correct cell's, and `paired` divides treatment by base per pair, so a+ # one-sided failure lands in `delta_pct` and `spread_pct` directly. Worse on the result side,+ # where a gate-failed treatment cell pairs against a clean base and prints as `faster`. Not+ # hypothetical: the virtualization negative result in CONTRIBUTING-perf.md has the treatment's+ # census going from 12 mounted messages to 0 alongside a 28.2% faster headline. Scoped by+ # hand, because a `gate` row is not a row type `latest_attempt_rows` covers.gate_failures = failed_invalidating_gates(records)- # A SUPERSEDED ATTEMPT'S COMPLETION IS NOT THIS CELL'S READING, and the session scoping below- # cannot say so: it answers "which rows belong to this completed cell row", not "is this- # completed cell row still the cell". `ab.skippable_cells` re-runs an A/B pair WHOLE, so a- # resume with any work left re-attempts cells that had already completed, and a retry that- # dies leaves the older completed row as the only one carrying a full set of timings.- #- # Left raw, that row is admitted -- and the guard above it disagrees about which attempt is- # being scored. `failed_invalidating_gates` names the winner from the LAST cell row, so a- # retry that crashed and wrote its `completed=False` row supersedes the dead attempt's FAILED- # gate while this loop keeps that same attempt's numbers. A cell refused for losing its- # thread's middle before the resume was published after it, pairing its cheaper timings- # against a clean base arm and printing as `faster`: the 28.2% failure described above,- # arriving through the resume. Same rows, same rule, one lens -- as `runtime/ab.readings_by_arm`- # and `sweep/ui_parity.collect` already read them.- #- # THE REDUCTION THAT DOES THIS NOW SITS AT THE TOP OF THE FUNCTION, and it is CONDITIONAL --- # skipped when the caller named a `session`. A second unconditional call here would have been- # a no-op on the default path and a silent regression on that one: it would supersede inside- # the escape hatch, empty the older of two concurrent sessions, and take away the very thing- # this module's collision refusal tells the reader to reach for. So the guard above and this- # loop still read one lens, and which lens that is now depends on the argument, deliberately.- #- # Left open on purpose: with `session=` named, `failed_invalidating_gates` still resolves its- # winner across every session in the file, so a caller who names the OLDER of two sessions can- # have its gate discarded by the newer one while this loop admits its cells. Scoping the gate- # search to the named session is the fix, and it is a change to the escape hatch's semantics- # rather than to this merge.+ # A SUPERSEDED ATTEMPT'S COMPLETION IS NOT THIS CELL'S READING, and session scoping cannot say+ # so. Left raw, the older completed row is admitted while `failed_invalidating_gates` names+ # its winner from the LAST cell row, so a crashed retry supersedes the dead attempt's FAILED+ # gate while this loop keeps that attempt's numbers: the 28.2% failure above, arriving through+ # the resume. The reduction that fixes it sits at the top of the function and is CONDITIONAL,+ # skipped when the caller named a `session`, so it cannot empty the escape hatch. Left open on+ # purpose: with `session=` named, `failed_invalidating_gates` still resolves across the file.out: dict[str, dict[str, float]] = {}if session is not None:@@ -402,16 +340,15 @@ def cell_metrics(records: list[dict], session: str | None = None) -> dict[str, dif str(row.get("cell_id")) in gate_failures:continuecid = row["cell_id"]- # Scoped to this cell's OWN attempt: `--resume` re-runs a died cell under a new- # session_id into the same file, and pooling both attempts mixes two measurements.+ # Scoped to this cell's OWN attempt: `--resume` re-runs a died cell under a new session_id into+ # the same file, and pooling both attempts mixes two measurements.sid = row.get("session_id")own = [r for r in records if r.get("cell_id") == cid and r.get("session_id") == sid]vals: dict[str, float] = _action_timings(own, cid)- # `UNSCORED_WINDOW_KINDS` is idle plus setup, dropped here as in `measures_from_records`.- # The setup window is the composer click that starts the film, which is mostly- # Playwright's injected actionability script blocking the page's own main thread; pooled- # in, an 11 s driver stall would set this table's `max_frame_ms` floor and hide every real- # regression under it.+ # `UNSCORED_WINDOW_KINDS` is idle plus setup, dropped here as in `measures_from_records`: the+ # setup window is the composer click, mostly Playwright's injected actionability script+ # blocking the main thread, so pooled in an 11 s driver stall would set this table's+ # `max_frame_ms` floor.windows = [wfor w in own@@ -421,10 +358,9 @@ def cell_metrics(records: list[dict], session: str | None = None) -> dict[str, dfor key, m in _frame_measures(windows).items():if m.value is not None:vals[key] = float(m.value)- # The streaming phase on its own, per streamed character. Kept separate from the pooled- # frame metrics rather than replacing them: the pooled ones answer "was this film janky",- # which is still worth asking, and these answer "what did streaming one character cost",- # which nothing answered before.+ # The streaming phase on its own, per streamed character. Kept separate from the pooled frame+ # metrics rather than replacing them: those answer "was this film janky", these answer "what+ # did streaming one character cost".for key, m in _stream_measures(windows).items():if m.value is not None:vals[key] = float(m.value)@@ -475,21 +411,15 @@ def paired(records: list[dict], shard: str = "") -> dict[str, list[tuple[float,A payload recorded before session ids existed has `""` on both arms and pairs exactly as before."""- # THE SESSION IS PART OF THE KEY, for the same reason the shard is. Two sessions in one- # payload both produce `rep0`, and pairing on the repetition alone matches one session's base- # against the other's treatment -- which is a comparison between two different runs under two- # different machine loads, reported as a repetition.- #- # Looped per session because several sessions in one payload is ORDINARY -- a sharded or- # resumed run appends legitimately, and each session's repetitions must pair within themselves.- # What is never ordinary is one cell completing twice, so that is refused first, here rather- # than only inside `cell_metrics`: this loop always passes a session, so the refusal there was- # unreachable from the shipped path and a payload of two concurrent runs pooled straight- # through it.+ # THE SESSION IS PART OF THE KEY, for the same reason the shard is: two sessions in one payload+ # both produce `rep0`, and pairing on the repetition alone matches one session's base against+ # the other's treatment. Looped per session because several sessions in one payload is+ # ORDINARY; what is never ordinary is one cell completing twice, so that is refused first,+ # here rather than only inside `cell_metrics`, where the shipped path made it unreachable.refuse_collisions(records)# The same superseding `cell_metrics` applies, done once here so the session list below is the# set of sessions that still own a completed cell. Without it a finished ladder re-run pairs- # ONCE PER SESSION and one repetition is pooled twice, under two different sets of numbers.+ # ONCE PER SESSION and one repetition is pooled twice.records = list(latest_attempt_rows(records))by_key: dict[tuple[str, str, str, str], dict[str, dict[str, float]]] = collections.defaultdict(dict@@ -504,9 +434,9 @@ def paired(records: list[dict], shard: str = "") -> dict[str, list[tuple[float,continuefor metric in set(sides["base"]) & set(sides["treatment"]):b, t = sides["base"][metric], sides["treatment"][metric]- # `if b` is a ratio's precondition, not a validity test: a zero base is a real reading- # for the jank metrics and only an undefined DENOMINATOR. Those are compared by- # difference (see DIFFERENCE_METRICS), so they are kept whenever both sides are finite.+ # `if b` is a ratio's precondition, not a validity test: a zero base is a real reading for the+ # jank metrics and only an undefined DENOMINATOR, and those are compared by difference (see+ # DIFFERENCE_METRICS).if metric in DIFFERENCE_METRICS:if math.isfinite(b) and math.isfinite(t):out[metric].append((b, t))@@ -578,12 +508,10 @@ def load(paths: list[Path]) -> tuple[dict[str, list[tuple[float, float]]], set[scorpora: set[str] = set()for path in paths:records = read_rows(path)- # REFUSED HERE, not warned about. This is the same class of refusal as the tier and- # corpus checks below, one step earlier: those two say "these are different films", and- # this one says "this film was shot with the camera in the shot". There is no flag to- # override it, because the only correct response is to re-run without the probe. The- # check itself lives in the scoring layer so that the A/B table and `--report` refuse on- # exactly the same evidence rather than on a second copy of the rule.+ # REFUSED HERE, not warned about: the tier and corpus checks below say "these are different+ # films", and this one says "this film was shot with the camera in the shot". No flag+ # overrides it, because the only correct response is to re-run without the probe. The check+ # lives in the scoring layer so the A/B table and `--report` refuse on the same evidence.refuse_if_probed(records, str(path))tiers |= tiers_of(records)corpora.add(corpus_of(records))@@ -595,10 +523,9 @@ def load(paths: list[Path]) -> tuple[dict[str, list[tuple[float, float]]], set[sf"fast-tier film and a standard-tier film are different measurements of "f"the same action, not repetitions of one.")- # Same rule one level down. The tier fixes how long the film runs; the corpus hash fixes what- # is IN it, and it covers the generator's parameters as well as every unit's bytes. Corpus v2- # added math, so a v1 payload and a v2 payload measure two different documents under one name,- # and pooling them would read the corpus change as a performance change.+ # Same rule one level down: the tier fixes how long the film runs, the corpus hash fixes what+ # is IN it, covering the generator's parameters and every unit's bytes. Corpus v2 added math,+ # so pooling a v1 and a v2 payload would read the corpus change as a performance change.if len(corpora) > 1:raise SystemExit(f"refusing to pool payloads built on different corpora: "@@ -634,21 +561,12 @@ def partial_censoring(paths: list[Path]) -> dict[str, str]:everything: list[dict] = []for path in paths:# SUPERSEDED ATTEMPTS DROPPED, exactly as `paired` drops them, because this decides whether- # the number `paired` produced may be quoted. A resume that died mid-cell leaves an action- # row whose `expect_ok` is False, and `censored_metrics` counts that as a censored cell --- # so reading the raw file here would mark a metric partially censored on the strength of an- # attempt that contributed nothing to the mean. The two readers have to see one payload.- #- # NAMESPACED BY SHARD, for the same reason `paired` keys on it: sharding restarts the- # repetition counter, so `r500K.base.rep0` names a DIFFERENT cell in each shard. Merged on- # the bare id, the completed-cell filter below stops separating them -- one shard's failed- # cell and another shard's successful one become a single id that is both censored and- # completed, and a metric nothing censored anywhere is refused as partially censored on- # the strength of a cell that contributed nothing to the mean. That is the very case the- # paragraph above drops superseded attempts to avoid, arriving one file over.- #- # SUFFIXED rather than prefixed, so `refuse_partial_censoring` still reads the rung off- # `cell_id.split(".", 1)[0]` and its message names rungs rather than directories.+ # `paired`'s number may be quoted: a resume that died mid-cell leaves an action row whose+ # `expect_ok` is False, which `censored_metrics` counts as a censored cell. NAMESPACED BY+ # SHARD, for the same reason `paired` keys on it: sharding restarts the repetition counter, so+ # merged on the bare id one shard's failed cell and another's successful one become a single+ # id that is both censored and completed. SUFFIXED rather than prefixed, so+ # `refuse_partial_censoring` still reads the rung off `cell_id.split(".", 1)[0]`.shard = str(path.parent.name)for row in latest_attempt_rows(read_rows(path)):row = dict(row)@@ -674,29 +592,18 @@ def summarise(paths: list[Path]) -> dict[str, dict[str, float]]:censoring = partial_censoring(paths)for metric, rows in pooled.items():if metric in DIFFERENCE_METRICS:- # Same three quantities, same three gates, measured in the metric's own unit because a- # ratio from a clean zero base does not exist. `delta_pct` and `spread_pct` are then- # percentage POINTS (or ms), which is why `render` marks the row.+ # Same three quantities and gates, measured in the metric's own unit because a ratio from a+ # clean zero base does not exist; `delta_pct` and `spread_pct` are then percentage POINTS,+ # which is why `render` marks the row.diffs = [t - b for b, t in rows]- # A TIE IS NOT A DISAGREEMENT, AND HERE THE TIE IS THE MODAL READING.- #- # The ratio branch below asks `all(r > 1) or all(r < 1)`, and that is sound there- # because a paired ratio of exactly 1.0 is vanishingly rare for a continuous timing.- # Reused verbatim on differences it stops being sound, because the tie value is 0.0- # and 0.0 is what a CLEAN reading of these metrics is: `stream_time_in_jank_pct` is- # exactly 0.0 on 765 of 1,438 scored cells, so a repetition where neither arm dropped- # a frame contributes a difference of exactly 0.0.- #- # A group of `[0.0, 12.0]` then failed both halves of the strict test and was reported- # as VOID (pairs disagree on sign), which is not what happened: nothing moved the other- # way, some repetitions were unchanged and the rest regressed together. Measured over- # the recorded payloads, that voided 34 of 250 pooled groups, 13.6%, and every one of- # the 34 came from a tie rather than from a real disagreement. Intermittent and- # load-dependent jank is exactly the shape that produces ties, so the predicate was- # discarding the regressions it was most needed for.- #- # Direction is therefore "no pair went the other way", with at least one that moved,- # so an all-zero group stays inconsistent rather than being scored as a finding.+ # A TIE IS NOT A DISAGREEMENT, AND HERE THE TIE IS THE MODAL READING. The ratio branch below+ # asks `all(r > 1) or all(r < 1)`, which is sound for a continuous timing but not for+ # differences, where the tie value 0.0 is what a CLEAN reading is. A group of `[0.0, 12.0]`+ # then failed both halves and was reported VOID though nothing moved the other way: over the+ # recorded payloads that voided 34 of 250 pooled groups, every one from a tie rather than a+ # real disagreement, and intermittent jank is exactly the shape that produces ties. Direction+ # is therefore "no pair went the other way", with at least one that moved, so an all-zero+ # group stays inconsistent.nonzero = [d for d in diffs if d != 0.0]out[metric] = {"n": len(rows),@@ -715,21 +622,19 @@ def summarise(paths: list[Path]) -> dict[str, dict[str, float]]:"base": statistics.fmean(b for b, _ in rows),"treat": statistics.fmean(t for _, t in rows),"delta_pct": (statistics.fmean(ratios) - 1.0) * 100.0,- # GATE 2. Do the pairs even agree on the SIGN? A metric whose four paired ratios- # straddle 1.0 has no direction, however far its mean happens to land from the floor.- # This is what separates "the change moved it" from "the metric is thrashing".+ # GATE 2: do the pairs even agree on the SIGN? A metric whose four paired ratios straddle 1.0+ # has no direction however far its mean lands from the floor, which is what separates "the+ # change moved it" from "the metric is thrashing"."consistent": all(r > 1.0 for r in ratios) or all(r < 1.0 for r in ratios),- # The spread of the paired ratios. From a null control it is the detection floor; from- # a real comparison it is the effect's own scatter, which is GATE 3.+ # The spread of the paired ratios: from a null control it is the detection floor, from a real+ # comparison the effect's own scatter, which is GATE 3."spread_pct": (max(ratios) - min(ratios)) * 100.0,"difference": False,}- # LABELLED, NOT DROPPED AND NOT RAISED. Raising would be the defect-10 mistake: `open_ms`- # is censored above 100K on EVERY standard and full run, so a refusal that exited would- # abort the whole table on the normal case and the guard would be removed within a day.- # Dropping the row silently would delete a reading somebody may still want at the one rung- # that produced it. So the row survives, carries the rungs it actually covers, and is- # denied a verdict below -- it can no longer clear a gate or be counted as a survivor.+ # LABELLED, NOT DROPPED AND NOT RAISED. Raising would be the defect-10 mistake, since `open_ms`+ # is censored above 100K on EVERY standard run, so a refusal would abort the whole table on+ # the normal case; dropping silently would delete a reading somebody may still want. The row+ # survives, carries the rungs it covers, and is denied a verdict below.if metric in censoring:out[metric]["poolable"] = Falseout[metric]["censoring"] = censoring[metric]@@ -764,9 +669,8 @@ def verdict_for(if stat["n"] > 1 and stat["spread_pct"] > abs(stat["delta_pct"]):return f, "VOID (effect under its own scatter)"if is_count:- # A count is an invariant, so the sign means the opposite of what it means for a timing.- # Less of the conversation copied is not an improvement, and calling it "faster" because- # the number went down is exactly the kind of misreading this table exists to prevent.+ # A count is an invariant, so the sign means the opposite of what it means for a timing: less+ # of the conversation copied is not an improvement.return f, ("LOST (invariant fell)" if stat["delta_pct"] < 0 else "gained")return f, ("faster" if stat["delta_pct"] < 0 else "SLOWER")@@ -815,19 +719,12 @@ def render(f"different corpus is a different film. Re-run the null control.")# THE REST OF THE COMPARABILITY IDENTITY, which tier and corpus are only two axes of.- #- # `--compare` refuses two payloads whose `comparability_key`s differ, and this path -- the one- # that actually certifies a number -- checked two of the eleven fields that key covers. So an- # 0.1.0 floor could score an 0.2.0 payload on the same corpus and tier, and this commit is- # precisely why that matters: it redefines `reasoning_toggle.open_ms` to terminate on a settled- # DOM rather than on the `data-state` flip, so the older floor measures a different quantity- # under the same name. `host`, `engine`, `cadence`, `headed`, `rungs` and the injection and- # probe axes are the same class of difference, and `--compare` is a separate command nobody is- # obliged to run first.- #- # Checked through `explain_incomparable` rather than against a second list of fields, so a- # field added to `comparability_fields` is enforced here without being added here too. A guard- # kept in step by hand is the drift `payload_rules` exists to argue against.+ # `--compare` refuses two payloads whose `comparability_key`s differ, while this path, the one+ # that certifies a number, checked two of the eleven fields, so an 0.1.0 floor could score an+ # 0.2.0 payload: this commit redefines `reasoning_toggle.open_ms` to terminate on a settled+ # DOM, so the older floor measures a different quantity under the same name. Checked through+ # `explain_incomparable` rather than a second list of fields, so a field added to+ # `comparability_fields` is enforced without being added here too.if floor_meta is not None:meta, conflicts = merged_meta(paths)if meta is None:@@ -865,41 +762,31 @@ def render(marked = Falsefor metric in sorted(stats, key = lambda m: (m in METRICS, m)):s = stats[metric]- # TWO INDEPENDENT CAVEATS, BOTH CARRIED IN THE NAME COLUMN, and neither allowed to hide- # the other. They answer different questions -- `(abs)` says what the delta MEANS, `[*]`- # says which rungs it covers -- and a metric can need both at once.- #- # A metric censored at some rungs and measured at others is marked so the caveat cannot be- # separated from the number when somebody copies one row out of this table into prose.- # That is the route every withdrawn figure in this harness travelled.+ # TWO INDEPENDENT CAVEATS, BOTH CARRIED IN THE NAME COLUMN, neither allowed to hide the other:+ # `(abs)` says what the delta MEANS, `[*]` says which rungs it covers, and a metric can need+ # both. A metric censored at some rungs and measured at others is marked so the caveat cannot+ # be separated from the number when somebody copies one row into prose.partial = s.get("poolable") is False- # A difference row is named as one. Its delta is in the metric's own unit, and printing it- # under a "delta %" heading would be the same class of mistake this file exists to stop.- #- # ORTHOGONAL TO THE TWO CENSORING MARKS BELOW, and applied first so it survives either of- # them: `(abs)` says what the delta MEANS, `[*]` and `[f]` say who can judge it. A metric- # can need `(abs)` and a censoring mark at once.+ # A difference row is named as one: its delta is in the metric's own unit, so printing it under+ # a "delta %" heading would be the same class of mistake this file exists to stop. Applied+ # first so it survives either censoring mark.label = metricif s.get("difference"):label, marked = f"{metric} (abs)", True# THE FLOOR IS A MEASUREMENT TOO, and it is censored by the same rule on its own cells.- ## `summarise` marks the null control's entry `poolable = False` when the metric answered on- # some of its cells and was censored on others, and this loop then read only the RESULT's- # flag -- so a result measured in full was scored against a floor built from whichever- # repetitions of the null happened to survive. Censoring is decided against a fixed budget,- # so the repetitions it removes are the slow ones, and `spread_pct` is `max - min` over the- # survivors: removing pairs can only narrow it. Observed on a real null control at 100K- # plus 500K, `reasoning_toggle.close_ms` censored above 100K, the applied floor came out at- # 17.1% from the four surviving 100K pairs while the censored repetitions of that same null- # ran 1429-2149 ms against 498-681 ms and paired as far apart as 1.50 on identical builds.- # A real result of -24.8% printed `faster` against the 17.1% and is not distinguishable- # from the null's own scatter.+ # some cells and was censored on others, and this loop read only the RESULT's flag, so a+ # result measured in full was scored against a floor built from whichever repetitions of the+ # null survived. Censoring removes the slow ones and `spread_pct` is `max - min` over the+ # survivors, so it can only narrow: on a real null at 100K plus 500K,+ # `reasoning_toggle.close_ms` gave an applied floor of 17.1% from four surviving pairs while+ # the censored repetitions of that same null paired as far apart as 1.50 on identical builds,+ # and a real result of -24.8% printed `faster` against it.+ # They ran 1429-2149 ms against the survivors' 498-681 ms.floor_stat = None if floors is None else floors.get(metric)floor_partial = floor_stat is not None and floor_stat.get("poolable") is False- # MARKED APART FROM `[*]`, because the two say different things. `[*]` means this row's own- # number is not a ladder number; `[f]` means the number is fine and the null control cannot- # judge it. Collapsing them would tell a reader to distrust a figure that is sound.+ # MARKED APART FROM `[*]`, because the two say different things: `[*]` means this row's own+ # number is not a ladder number, `[f]` means the number is fine and the null cannot judge it.shown = f"{label} [*]" if partial else (f"{label} [f]" if floor_partial else label)line = (f" {shown:<28}{s['n']:>3}{s['base']:>11.1f}{s['treat']:>11.1f}"@@ -907,13 +794,12 @@ def render()if floors is not None:if partial:- # DENIED A VERDICT. Passing three gates on the rungs that could answer is not- # passing them on the ladder the row is labelled with.+ # DENIED A VERDICT: passing three gates on the rungs that could answer is not passing them on+ # the ladder the row is labelled with.line += f"{'--':>13} not pooled"elif floor_partial:- # DENIED A VERDICT ON THE OTHER SIDE. The three gates are all measured against- # this floor, so a floor that is itself a survivor sample cannot certify anything,- # however clean the result is.+ # DENIED A VERDICT ON THE OTHER SIDE: the three gates are all measured against this floor, so a+ # floor that is itself a survivor sample cannot certify anything.line += f"{'--':>13} no poolable floor"else:f, verdict = verdict_for(s, floor_stat, is_count_metric(metric))@@ -979,9 +865,8 @@ def main(argv: list[str] | None = None) -> int:floor_rows = read_rows(floor_paths[0])floor_tier = tier_of(floor_rows)floor_corpus = corpus_of(floor_rows)- # The floor's own comparability identity, read across every shard and every header. A null- # whose shards disagree with each other is not one null control, so it gets the same- # refusal a result payload gets rather than a token averaged over the disagreement.+ # The floor's own comparability identity, read across every shard and header: a null whose+ # shards disagree with each other is not one null control.floor_meta, floor_conflicts = merged_meta(floor_paths)if floor_conflicts:print(diff --git a/tests/studio/studiobench/sweep/parity_null_control.py b/tests/studio/studiobench/sweep/parity_null_control.pyindex c5657ee047d..7a78eef49c6 100644--- a/tests/studio/studiobench/sweep/parity_null_control.py+++ b/tests/studio/studiobench/sweep/parity_null_control.py@@ -43,7 +43,7 @@ if __package__ in (None, ""): # pragma: no coverfrom tests.studio.studiobench.analysis import parity as P # noqa: E402from tests.studio.studiobench.sweep.ui_parity import collect, shards_of # noqa: E402-#: How much context to print either side of a differing run of characters.+#:How much context to print either side of a differing run of characters.CONTEXT = 90diff --git a/tests/studio/studiobench/sweep/parity_shots.py b/tests/studio/studiobench/sweep/parity_shots.pyindex 62c37963ed6..1f3f33dee6f 100644--- a/tests/studio/studiobench/sweep/parity_shots.py+++ b/tests/studio/studiobench/sweep/parity_shots.py@@ -135,10 +135,9 @@ def differing_actions(report's -- excused by the null control, then corroborated at the verdict's own `--min-reps`."""unstable, derived, _checks = unstable_set(null_paths or None)- # THE SAME EFFECTIVE SET THE VERDICT SCORED WITH. The verdict confines the imported- # exemptions to what the scored runner reproduces, so reading the raw imported set here would- # put the artifact back out of step with the job it illustrates -- an action the verdict- # failed on would have no picture, which is the same defect as publishing none at all.+ # THE SAME EFFECTIVE SET THE VERDICT SCORED WITH. The verdict confines the imported exemptions to+ # what the scored runner reproduces, so reading the raw imported set here would put the artifact+ # out of step with the job it illustrates: an action the verdict failed on would have no picture.if derived:unstable, _dropped = confine_to_runner(unstable, *in_arm_repeatability(result_paths))out = []@@ -150,9 +149,8 @@ def differing_actions(if r["verdict"] == P.DIFFER and not is_unstable(unstable, action, cell)]# Carrying the DIRECTION as the fifth element, exactly as `report` does, so the artifact- # illustrates the same set the verdict counted. Without it a direction-reversing pair would- # be firm here and uncorroborated there, and the evidence would show a regression the job- # did not fail on.+ # illustrates the same set the verdict counted. Without it a direction-reversing pair would be+ # firm here and uncorroborated there.one_sided = [(action, shard, cell, [r.get("reason", "")], r.get("one_sided") or None)for action, shard, cell, r in results@@ -160,9 +158,9 @@ def differing_actions(and r.get("one_sided")and not P.racy_execution(action, r.get("idle_reason") or "")]- # The third way `report` returns 1: the action ran on both arms and its own assertion failed- # on one. Not filtered by the unstable set, for the same reason the verdict does not filter- # it -- that set measures digest stability, and this is not a digest.+ # The third way `report` returns 1: the action ran on both arms and its own assertion failed on+ # one. Not filtered by the unstable set, for the same reason the verdict does not filter it: that+ # set measures digest stability, and this is not a digest.expect_bad = [(action, shard, cell, [r.get("expect_reason", "")], r["expect_regressed"])for action, shard, cell, r in results@@ -263,9 +261,9 @@ def build(missing += 1print(f" {d['action']:<26} {d['cell']}: shot file absent on disk; not a pair")continue- # The shard is in the FILENAME as well as the key. Two shards produce the same- # `<action>__<rung>_<rep>` stem, so without it the second composite silently overwrites- # the first and the artifact shows one picture for two findings.+ # The shard is in the FILENAME as well as the key: two shards produce the same+ # `<action>__<rung>_<rep>` stem, so without it the second composite silently overwrites the+ # first.stem = f"{d['shard']}__{d['action']}__{rung}_{rep}"ok = composite(bp,@@ -319,8 +317,8 @@ def prune(payload_dir: Path, shots_dir: Path) -> int:for action, _shard, cell, r in results:if r["verdict"] == P.MATCH:continue- # The prune reads one arm's own output dir, but key by the shard it came from anyway so- # the index has one shape everywhere.+ # The prune reads one arm's own output dir, but key by the shard it came from anyway so the index+ # has one shape everywhere.rung, rep = cell.split(" ", 1)for arm in ("base", "treatment"):got = index.get((_shard, f"{rung}.{arm}.{rep}", action, arm))@@ -372,9 +370,9 @@ def main(argv: list[str] | None = None) -> int:Path(args.null) if args.null else None,Path(args.shots),Path(args.out),- # FORWARDED, and the omission here was the whole bug: the workflow passes --min-reps 2 to- # match the verdict, `build` defaulted it back to 1, and the artifact then carried- # composites of one-repetition flakes beside the change that actually failed the job.+ # FORWARDED, and the omission here was the whole bug: the workflow passes --min-reps 2 to match the+ # verdict, `build` defaulted it back to 1, and the artifact carried composites of one-repetition+ # flakes beside the change that failed the job.min_reps = args.min_reps,)diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.pyindex ff9676d0978..47c1987b744 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_floor_applicability.py@@ -45,7 +45,7 @@ from tests.studio.studiobench.runtime.types import Recorderfrom tests.studio.studiobench.scoring import payload_rulesfrom tests.studio.studiobench.sweep import floor_table-#: The corpus every payload here is recorded on. Real, from `outputs/rp/sbench_T_null`.+#:The corpus every payload here is recorded on. Real, from `outputs/rp/sbench_T_null`.CORPUS = "ac9d5d8e37be2a3844deed559fde6070247ad2322377295fb383b60b5eec5a0c"@@ -129,9 +129,9 @@ def _write(return out / "payload.jsonl"-#: Real 100K `close_ms` readings. Base then treatment, `outputs/rp/sbench_T_campaign`.+#:Real 100K `close_ms` readings. Base then treatment, `outputs/rp/sbench_T_campaign`.RESULT_100K = ((897.6, 631.4), (665.6, 515.5), (665.6, 498.8), (682.1, 532.3))-#: Real 100K `close_ms` readings from the null control, `outputs/rp/sbench_T_null`.+#:Real 100K `close_ms` readings from the null control, `outputs/rp/sbench_T_null`.NULL_100K = ((681.4, 564.7), (565.7, 515.7), (532.0, 515.1), (498.8, 498.8))#: Real 500K `close_ms` readings from the null control, `outputs/rp/sbench_C_null`. Every one of#: these cells is censored: the open settle blew its budget, so the action failed and the harness@@ -235,8 +235,8 @@ def test_the_result_is_not_labelled_as_the_censored_one(tmp_path, capsys):def test_the_same_result_scores_normally_against_a_whole_floor(tmp_path, capsys):"""NOT A REFUSAL OF EVERYTHING. Identical result, a null control that censored nothing."""floors = floor_table.summarise([_null(tmp_path, censored = False)])- # The whole-ladder floor is 67.5%, which this -24.8% result does not clear, so the verdict- # under it is VOID -- itself the point: the censored floor turned a VOID into a `faster`.+ # The whole-ladder floor is 67.5%, which this -24.8% result does not clear, so the verdict under it+ # is VOID, itself the point: the censored floor turned a VOID into a `faster`.floor_table.render([_result(tmp_path)], "t", floors = floors)row = next(line for line in capsys.readouterr().out.splitlines() if line.strip().startswith(METRIC)diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.pyindex 437735c580c..5e850e50467 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py@@ -28,8 +28,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[4]))from tests.studio.studiobench.analysis import parity as P # noqa: E402from tests.studio.studiobench.sweep import ui_parity as U # noqa: E402-# parents[4] is `tests/`, which is what the sibling selftests put on sys.path; the repo root is-# one further up and is where .github lives.+# parents[4] is `tests/`, which the sibling selftests put on sys.path; the repo root is one+# further up and is where .github lives.REPO_ROOT = Path(__file__).resolve().parents[5]WORKFLOW = REPO_ROOT / ".github/workflows/studiobench-ui-parity.yml"@@ -91,9 +91,9 @@ def two_reps(action: str, base: str, treat: str) -> list[tuple]:def test_a_null_control_that_decided_everything_and_found_nothing_passes(tmp_path):- # Every action reached two observations and not one of them differed. This is the best null- # control obtainable and the measured unstable set is empty BECAUSE there was nothing to- # measure. Requiring a measured entry here would fail the job on its best day.+ # Every action reached two observations and none differed: the best null control obtainable,+ # whose measured unstable set is empty BECAUSE there was nothing to measure. Requiring a+ # measured entry here would fail the job on its best day.null = null_run(tmp_path,"quiet",@@ -105,17 +105,14 @@ def test_a_null_control_that_decided_everything_and_found_nothing_passes(tmp_patassert len(report["decided"]) == 2assert report["undecided"] == []- # And the thing the naive gate keyed on really is absent, so this test would not have caught- # the bug by accident: the derived set emits no `action@rung` entry at all.unstable, _derived, _checks = U.unstable_set([null])assert not [e for e in unstable if isinstance(e, tuple)]def test_a_null_control_with_one_observation_each_is_undecided(tmp_path):- # The other direction, so the audit cannot pass by never failing. --reps 1 gives one- # observation per (rung, action); `derive_unstable` needs two, so nothing is decided, the- # measured set is empty for a completely different reason, and the result would be scored- # against the declared list.+ # The other direction, so the audit cannot pass by never failing: --reps 1 gives one+ # observation per (rung, action) where `derive_unstable` needs two, so the measured set is+ # empty for a completely different reason.null = null_run(tmp_path,"thin",@@ -131,9 +128,8 @@ def test_a_null_control_with_one_observation_each_is_undecided(tmp_path):def test_the_two_empty_measured_sets_are_told_apart(tmp_path):- # Both of the above derive an EMPTY measured set. The whole point of the audit is that one is- # the best possible reading and the other is a broken one, so assert they are distinguished- # rather than merely that each has the right code.+ # Both of the above derive an EMPTY measured set, so assert the audit distinguishes the best+ # possible reading from a broken one, not merely that each has the right code.quiet = null_run(tmp_path, "q2", two_reps("settings", "SAME", "SAME"))thin = null_run(tmp_path, "t2", [("r100K", "rep0", "settings", "SAME", "SAME")])for path in (quiet, thin):@@ -167,9 +163,9 @@ def test_a_noisy_null_control_is_decided_and_reports_what_differed(tmp_path):def test_an_action_that_never_captured_is_undecided_not_stable(tmp_path):- # A failed capture is blind. `derive_unstable` refuses to count it as an observation, because- # an action derived as stable from pairs that never rendered would be trusted forever on the- # strength of nothing. So it must read as UNDECIDED, never as a quiet pass.+ # A failed capture is blind, and `derive_unstable` refuses to count it: an action derived as+ # stable from pairs that never rendered would be trusted forever on nothing. So it must read+ # UNDECIDED, never a quiet pass.null = null_run(tmp_path, "blind", two_reps("image_upload", None, None))rc, report = U.audit_null([null])assert rc == 1@@ -177,9 +173,8 @@ def test_an_action_that_never_captured_is_undecided_not_stable(tmp_path):def test_an_excused_action_is_allowed_but_still_needs_a_decided_one(tmp_path):- # image_upload is permanently blind on this fixture, so it is excusable. What is NOT excusable- # is a run in which everything is excused: that measured nothing at all while naming a reason- # for each blank, and passing it would let the excuse list grow until the audit is vacuous.+ # image_upload is permanently blind on this fixture, so it is excusable; a run in which+ # EVERYTHING is excused measured nothing at all while naming a reason for each blank.only_excused = null_run(tmp_path, "allblind", two_reps("image_upload", None, None))rc, report = U.audit_null([only_excused], frozenset({"image_upload"}))assert rc == 1@@ -232,10 +227,9 @@ def test_the_cli_audits_a_single_repetition_as_a_failure(tmp_path, capsys):def test_the_parity_workflow_audits_the_null_with_the_tool(tmp_path):- # Twice now a guard written as a regex over this tool's own printed prose has been wrong: once- # because a real payload spells its rung `r100K` where a fixture spells it `100K`, and once- # because the literal `action@rung` appears in the explanatory text whether or not anything- # was measured. The check belongs in the tool, where it is reachable by the tests above.+ # Twice now a guard written as a regex over this tool's printed prose has been wrong: a real+ # payload spells its rung `r100K` where a fixture spells it `100K`, and the literal+ # `action@rung` appears in the explanatory text whether or not anything was measured.text = WORKFLOW.read_text(encoding = "utf-8")assert "--audit-null" in textassert "--allow-undecided image_upload" in text@@ -247,8 +241,6 @@ def test_the_parity_workflow_audits_the_null_with_the_tool(tmp_path):def test_a_difference_in_one_repetition_of_two_is_not_counted(tmp_path):- # The measured false alarm, in miniature. A build renders the same way every time, so a stable- # action that differs in rep0 and matches in rep1 is telling you about the run.mine = null_run(tmp_path,"one_rep",@@ -264,7 +256,7 @@ def test_a_difference_in_one_repetition_of_two_is_not_counted(tmp_path):def test_a_difference_in_every_repetition_still_fails(tmp_path):# The other direction, and the one that matters more: an injected element differs on every# pass, so raising the bar must not make the gate unable to fail. Measured on the real probe- # -- one <span> added inside the thread root -- which stays red at --min-reps 2.+ # (one <span> added inside the thread root), which stays red at --min-reps 2.mine = null_run(tmp_path,"both_reps",@@ -277,8 +269,6 @@ def test_a_difference_in_every_repetition_still_fails(tmp_path):def test_the_headline_count_is_the_one_the_exit_code_uses(tmp_path, capsys):- # Printing "stable actions differing: 1" above a verdict of 0 is how a reader decides the- # tool is lying to them.mine = null_run(tmp_path,"hdr",@@ -296,7 +286,7 @@ def test_the_headline_count_is_the_one_the_exit_code_uses(tmp_path, capsys):def test_one_repetition_seen_twice_is_not_two_observations(tmp_path):# Two shards can carry the same (rung, rep); corroboration counts DISTINCT repetitions, or a- # single flake recorded in two places would corroborate itself.+ # single flake recorded twice would corroborate itself.a = null_run(tmp_path, "sh1", [("r100K", "rep0", "settings", "A", "B")])b = null_run(tmp_path, "sh2", [("r100K", "rep0", "settings", "A", "B")])assert U.report([a, b], "t", frozenset(), min_reps = 2) == 0@@ -326,14 +316,11 @@ def run_with_completion(tmp_path: Path, name: str, specs: list[tuple]) -> Path:def test_a_null_control_ignores_a_cell_that_never_completed(tmp_path):- # The half-written film. Its rows look complete and pair like real ones, and counting them- # narrows the excuse set, which is how a truncated null manufactures false alarms downstream.null = run_with_completion(tmp_path,"truncated",[("rep0", "A", "B", True), ("rep1", "A", "B", False)],)- # Only the completed repetition is an observation, so one observation, so undecided.rc, report = U.audit_null([null])assert rc == 1assert report["decided"] == []@@ -351,10 +338,10 @@ def test_the_same_two_repetitions_decide_it_once_both_cells_completed(tmp_path):def test_a_result_still_reports_a_difference_from_a_cell_that_died(tmp_path):- # The OTHER sign, and the one that matters more. A cell that died is the latest attempt at- # itself and the difference it saw is real; dropping it would silence a regression, which is- # the worse direction. `test_an_attempt_that_was_never_re_run_still_carries_its_parity_- # verdict` holds the same line from the other file.+ # The OTHER sign, and the one that matters more: a cell that died is the latest attempt at+ # itself and the difference it saw is real, so dropping it would silence a regression.+ # test_an_attempt_that_was_never_re_run_still_carries_its_parity_verdict holds the same line+ # from the other file.mine = run_with_completion(tmp_path,"died",@@ -365,8 +352,6 @@ def test_a_result_still_reports_a_difference_from_a_cell_that_died(tmp_path):def test_a_payload_with_no_cell_rows_says_it_could_not_check(tmp_path, capsys):- # Falling back to the old behaviour is right; falling back silently is how a guard stops- # guarding without anybody noticing.old = null_run(tmp_path, "nocells", two_reps("settings", "A", "B"))U.report([old], "t", frozenset())assert "NOT GUARDED" in capsys.readouterr().out@@ -390,15 +375,14 @@ def corpus_run(tmp_path: Path, name: str, corpus: str, rep: str) -> Path:def test_two_corpora_are_refused_before_their_observations_pool(tmp_path):# Each film is one repetition and can decide nothing alone. Pooled they satisfy- # min_observations and report DECIDED from two different threads, and the result is then- # scored against a set never measured on the corpus it is applied to.+ # min_observations and report DECIDED from two different threads, so the result is scored+ # against a set never measured on the corpus it is applied to.a = corpus_run(tmp_path, "c1", "corpusAAAA", "rep0")b = corpus_run(tmp_path, "c2", "corpusBBBB", "rep1")with pytest.raises(SystemExit) as got:U.one_corpus([a, b], "null control")assert "more than one corpus" in str(got.value)- # And the refusal is load-bearing: without it the two DO pool into a decision.rc, report = U.audit_null([a, b])assert rc == 0 and report["differed"] == [("r100K", "settings")]@@ -425,9 +409,8 @@ def not_run_row(cid: str, action: str) -> dict:def test_a_missed_slot_on_both_arms_cannot_turn_a_clean_verdict_red(tmp_path):- # The measured claim, in miniature. Over 18 mutations of a real payload at 2 to 24 missed- # slots, the verdict never gained a red; NOT_EXERCISED is filed under coverage, never under- # the stable differences the exit code is taken from.+ # The measured claim in miniature: over 18 mutations of a real payload at 2 to 24 missed slots+ # the verdict never gained a red, because NOT_EXERCISED is filed under coverage.rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):for arm in ("base", "treatment"):@@ -443,8 +426,6 @@ def test_a_missed_slot_on_both_arms_cannot_turn_a_clean_verdict_red(tmp_path):def test_a_run_that_compared_almost_nothing_does_not_pass(tmp_path, capsys):- # The other half, and the reason the slot gate could not simply be deleted. Two blank pages- # have identical digests, so a film that ran nothing is the easiest possible pass.rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):for arm in ("base", "treatment"):@@ -466,7 +447,7 @@ def test_a_run_that_compared_almost_nothing_does_not_pass(tmp_path, capsys):def test_the_audit_ignores_an_action_the_result_never_compared(tmp_path):# `copy_markdown` is undecided in the null and nothing on the result turns on its verdict, so# demanding one turns machine speed into a red build. The result DOES need an excuse for- # `settings`, so the scope is not empty and the audit is not passing vacuously.+ # `settings`, so the audit is not passing vacuously.null = null_run(tmp_path,"n",@@ -474,15 +455,13 @@ def test_the_audit_ignores_an_action_the_result_never_compared(tmp_path):)result = null_run(tmp_path, "r", two_reps("settings", "AAA", "BBB"))- assert U.audit_null([null], frozenset())[0] == 1 # unscoped: copy_markdown undecided+ assert U.audit_null([null], frozenset())[0] == 1scope = U.actions_needing_an_excuse([result], min_reps = 2)assert scope == {("r100K", "settings")}- assert U.audit_null([null], frozenset(), scope)[0] == 0 # scoped: nobody needed it+ assert U.audit_null([null], frozenset(), scope)[0] == 0def test_the_audit_still_fails_on_an_action_the_result_did_compare(tmp_path):- # The other direction, so scoping cannot pass by excusing everything: an action the result- # put a verdict on, that the null could not decide, is exactly the unscored case.null = null_run(tmp_path,"n2",@@ -502,30 +481,24 @@ def test_the_audit_still_fails_on_an_action_the_result_did_compare(tmp_path):def test_a_null_from_another_tier_is_refused_rather_than_applied(tmp_path):- # fast and standard both walk 100K, so a fast-film race lands on exactly the rung key a- # standard-film regression would be scored under. The old code warned and scored anyway.assert U.cross_side_mismatch({"standard"}, {"fast"}, {"c1"}, {"c1"}).startswith("the null control was recorded at tier")def test_a_null_from_another_corpus_is_refused_rather_than_applied(tmp_path):- # The likelier of the two in practice: a corpus revision lands, the result is re-recorded,- # and last week's null control is still sitting in the directory the workflow globs.got = U.cross_side_mismatch({"fast"}, {"fast"}, {"c2"}, {"c1"})assert got.startswith("the null control was recorded at corpus")def test_matching_sides_and_unrecorded_sides_are_both_allowed():- # Equal is comparable, and an empty set means the recorder predates the field, so there is- # nothing to disagree about. Refusing those would reject every legacy payload.assert U.cross_side_mismatch({"fast"}, {"fast"}, {"c1"}, {"c1"}) == ""assert U.cross_side_mismatch({"fast"}, set(), {"c1"}, set()) == ""assert U.cross_side_mismatch(set(), {"fast"}, set(), {"c1"}) == ""def test_the_refusal_exits_two_not_one_through_the_cli(tmp_path, monkeypatch, capsys):- # Exit 2 is the tool declining to answer. Exit 1 would read as a parity failure and send+ # Exit 2 is the tool declining to answer; exit 1 would read as a parity failure and send# somebody hunting for a UI change that was never measured.null = null_run(tmp_path, "tier_null", two_reps("settings", "SAME", "SAME"), tier = "fast")result = null_run(tmp_path, "tier_res", two_reps("settings", "SAME", "SAME"), tier = "standard")@@ -578,11 +551,10 @@ def one_sided_payload(def test_an_action_the_head_can_no_longer_perform_fails_the_verdict(tmp_path, capsys):- # THE HOLE THIS CLOSES. A control that stops opening is the one user-visible regression that- # leaves no digest to differ: the head arm records `ran: false`, the pair is NOT_EXERCISED,- # and filing that under coverage meant a button that no longer opens shipped green with 64 of- # 68 pairs still compared -- comfortably above the workflow's --min-compared 16, so the- # coverage floor could never catch it either.+ # THE HOLE THIS CLOSES: a control that stops opening is the one user-visible regression that+ # leaves no digest to differ, and filing NOT_EXERCISED under coverage meant a button that no+ # longer opens shipped green with 64 of 68 pairs still compared, comfortably above the+ # workflow's --min-compared 16.path = one_sided_payload(tmp_path, "regressed", "settings", ("rep0", "rep1"))assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 1printed = capsys.readouterr().out@@ -591,10 +563,9 @@ def test_an_action_the_head_can_no_longer_perform_fails_the_verdict(tmp_path, cadef test_one_arms_missed_slot_in_one_repetition_is_still_only_a_warning(tmp_path, capsys):- # The other half, and the reason this is not simply failed on sight. A contended runner can- # lose one arm's slot once, and both arms are driven from one script in one session, so the- # loss is not always symmetric. Held to the SAME corroboration bar as a differing digest: a- # build that cannot open a control cannot open it on either pass.+ # The other half, and why this is not failed on sight: a contended runner can lose one arm's+ # slot once, so it is held to the SAME corroboration bar as a differing digest. A build that+ # cannot open a control cannot open it on either pass.path = one_sided_payload(tmp_path, "flake", "settings", ("rep0",))assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 0assert "UNCORROBORATED one-arm-only" in capsys.readouterr().out@@ -602,8 +573,7 @@ def test_one_arms_missed_slot_in_one_repetition_is_still_only_a_warning(tmp_pathdef test_an_action_expected_to_vary_is_not_failed_for_reaching_one_arm_only(tmp_path, capsys):# `stop_generation` runs only while a stream is live, so which arm reached it before the- # stream ended is the same race its UNSTABLE_ACTIONS entry already describes. Scoring that as- # a build difference would red the job on stream timing.+ # stream ended is the race its UNSTABLE_ACTIONS entry already describes.path = one_sided_payload(tmp_path,"racy",@@ -616,8 +586,6 @@ def test_an_action_expected_to_vary_is_not_failed_for_reaching_one_arm_only(tmp_def test_both_arms_missing_the_same_action_is_coverage_and_not_a_verdict(tmp_path):- # Unchanged, and pinned so the fix above cannot creep into the missed-slot case the mutation- # study measured: NEITHER arm running an action is lost coverage in both builds.rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):for arm in ("base", "treatment"):@@ -636,10 +604,9 @@ def test_both_arms_missing_the_same_action_is_coverage_and_not_a_verdict(tmp_patdef test_an_undecided_action_the_result_only_matched_does_not_fail_the_audit(tmp_path):- # THE CI FAILURE THIS FIXES. The null lost one arm's slot on `settings`, so it has a single- # observation and cannot decide it. The result MATCHED on `settings` in both repetitions, and- # no entry in the unstable set can turn a match into anything, so the null was never going to- # be asked. Requiring an opinion here is what made the gate unsatisfiable on a shared runner.+ # THE CI FAILURE THIS FIXES: the null lost one arm's slot on `settings` and cannot decide it,+ # while the result MATCHED on `settings` in both repetitions, so the null was never going to+ # be asked. Requiring an opinion made the gate unsatisfiable on a shared runner.result = null_run(tmp_path, "result", two_reps("settings", "SAME", "SAME"))null = null_run(tmp_path, "null", [("r100K", "rep0", "settings", "SAME", "SAME")])scope = U.actions_needing_an_excuse([result], min_reps = 2)@@ -649,10 +616,9 @@ def test_an_undecided_action_the_result_only_matched_does_not_fail_the_audit(tmpdef test_the_audit_still_fails_on_an_undecided_action_the_result_kept_differing_on(tmp_path):- # THE SAFETY DIRECTION, and the whole reason the scope is not simply dropped. `settings`- # differed in BOTH repetitions, so it is scored as a regression unless the unstable set- # excuses it -- and an undecided null excuses it from the DECLARED list, unmeasured, which is- # the fiction this job exists to replace. Narrowing the scope must not reach this case.+ # THE SAFETY DIRECTION, and why the scope is not simply dropped: `settings` differing in BOTH+ # repetitions is scored as a regression unless the unstable set excuses it, and an undecided+ # null excuses it from the DECLARED list, unmeasured.result = null_run(tmp_path, "result", two_reps("settings", "AAA", "BBB"))null = null_run(tmp_path, "null", [("r100K", "rep0", "settings", "SAME", "SAME")])scope = U.actions_needing_an_excuse([result], min_reps = 2)@@ -663,8 +629,6 @@ def test_the_audit_still_fails_on_an_undecided_action_the_result_kept_differing_def test_a_difference_in_one_repetition_only_needs_no_excuse(tmp_path):- # `corroborated` already reports this without counting it, so no excuse can move it and the- # scope is threaded with the verdict's own --min-reps rather than a number of its own.result = null_run(tmp_path,"result",@@ -677,9 +641,9 @@ def test_a_difference_in_one_repetition_only_needs_no_excuse(tmp_path):def test_a_corroborated_one_arm_only_action_still_needs_the_null_to_decide_it(tmp_path):- # The other shape the unstable set is load bearing for: `report` excuses a one-arm-only action+ # The other shape the unstable set is load-bearing for: `report` excuses a one-arm-only action# when it is in the unstable set, so an undecided null hands out that excuse on the declared- # list alone. Scoped in for exactly the same reason a corroborated difference is.+ # list alone.result = one_sided_payload(tmp_path, "result", "settings", ("rep0", "rep1"))null = null_run(tmp_path, "null", [("r100K", "rep0", "settings", "SAME", "SAME")])scope = U.actions_needing_an_excuse([result], min_reps = 2)@@ -722,30 +686,24 @@ def missed_slot_payload(tmp_path: Path, name: str, action: str, reps: tuple[str,def test_one_arm_missing_a_slot_in_every_repetition_is_still_not_a_build_difference(tmp_path):- # MEASURED, not reasoned: this is the run that went red after the one-arm-only gate landed.- # `settings` missed a 1200ms budget on the BASE arm at 45700ms in both repetitions, by 1292ms- # and 1352ms. The schedule is fixed, so a machine slow enough to miss that slot once is slow- # enough to miss it twice -- the two misses are correlated through the runner and corroboration- # cannot separate them from a build difference. `slot_missed` can, and the driver already- # records it. Note which arm it was, too: the BASE arm went idle, so reading it as a- # regression would have blamed the head build for the old one's timing.+ # MEASURED, not reasoned: `settings` missed a 1200ms budget on the BASE arm at 45700ms in both+ # repetitions, by 1292ms and 1352ms. The schedule is fixed, so the two misses are correlated+ # through the runner and corroboration cannot separate them from a build difference, while+ # `slot_missed` can. Note which arm too: reading it as a regression would have blamed the head+ # build for the old one's timing.path = missed_slot_payload(tmp_path, "slow", "settings", ("rep0", "rep1"))assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 0def test_a_control_that_cannot_open_is_still_caught_when_a_slot_is_also_missed_elsewhere(tmp_path):- # The discriminator must not become a blanket excuse: a genuine precondition failure still- # fails even though the same run also lost slots.path = one_sided_payload(tmp_path, "broken", "settings", ("rep0", "rep1"))assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 1def test_a_difference_at_one_rung_does_not_demand_the_null_decide_another(tmp_path):- # The scope is keyed by (rung, action) and not by action alone. Reduced to bare names it- # re-widened across the ladder: a corroborated difference at 1K pulled the action into scope- # at EVERY rung, so one missed observation at 100K -- where the result matched and no excuse- # was ever consulted -- failed the audit again. Instability is a property of the rung, which- # is why `derive_unstable` decides per rung and `unstable_set` measures per rung.+ # The scope is keyed by (rung, action), not action alone: reduced to bare names a corroborated+ # difference at 1K pulled the action into scope at EVERY rung, so one missed observation at+ # 100K failed the audit again. Instability is a property of the rung.result = null_run(tmp_path,"result",@@ -756,7 +714,6 @@ def test_a_difference_at_one_rung_does_not_demand_the_null_decide_another(tmp_patmp_path,"null",[("r1K", "rep0", "keystroke", "A", "B"), ("r1K", "rep1", "keystroke", "A", "B")]- # One observation only at 100K, so the null cannot decide it there.+ [("r100K", "rep0", "keystroke", "S", "S")],)scope = U.actions_needing_an_excuse([result], min_reps = 2)@@ -786,22 +743,18 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_vary" while the command exits 0."""rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]- # rep0 is a clean, uncontested observation and it MATCHES.for arm in ("base", "treatment"):cid = f"r100K.{arm}.rep0"row = action_row(cid, "settings", "SAME")row["session_id"] = "s1"rows.append(row)rows.append({"row_type": "cell", "cell_id": cid, "completed": True, "session_id": "s1"})- # rep1 completed in session s1 ...for arm in ("base", "treatment"):cid = f"r100K.{arm}.rep1"row = action_row(cid, "settings", "SAME")row["session_id"] = "s1"rows.append(row)rows.append({"row_type": "cell", "cell_id": cid, "completed": True, "session_id": "s1"})- # ... and was re-run whole in s2, where the treatment arm died before its cell row. Its- # digests DIFFER, which is what would make the action look unstable.for arm, digest in (("base", "SAME"), ("treatment", "OTHER")):row = action_row(f"r100K.{arm}.rep1", "settings", digest)row["session_id"] = "s2"@@ -819,7 +772,6 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_assert got["incomplete"] == 1, gotkeys = {(k[2], k[4]) for k in got["pairs"]}assert ("rep0", "settings") in keys- # The dead retry left an unpaired base row; it must not have become an observation.assert not any("base" in sides and "treatment" in sidesfor k, sides in got["pairs"].items()@@ -830,15 +782,13 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_assert ("r100K", "settings") not in unstable, unstable-# ── the audit has to run in the modes that ask for it ────────────────-#-# The audit block sat below the structural early return while reading nothing from the plan, so a-# payload with no structural entries returned first and the audit never ran. `--mode visible` and-# `--mode behaviour` hard-set `structural` to an empty set for every pattern, so those two could-# never audit at all: the command exited 0 out of the ordinary visible report having silently-# skipped the option whose promise is to FAIL unless the null decided what the result needs.+# The audit block sat below the structural early return while reading nothing from the plan,+# so a payload with no structural entries returned first and it never ran; and `--mode+# visible` / `--mode behaviour` hard-set `structural` to empty, so those two could never audit+# at all.+# ── the audit has to run in the modes that ask for it ────────────────def test_the_cli_audits_a_single_repetition_under_forced_visible_mode(tmp_path, capsys):"""THE DEFECT. Same payload and same expectation as the auto-mode failure test above, with`--mode visible` added: the audit must still run and still fail."""@@ -884,7 +834,6 @@ def _arm_payload(tmp_path: Path, name: str, regress: str | None, self_race: strif act == regress and arm == "treatment":digest = "HEAD_IS_DIFFERENT"if act == self_race:- # Racing against itself: side A does not even reproduce between repetitions.digest = f"RACE_{arm}_{rep}"rows.append(action_row(cid, act, digest))rows.append({"row_type": "cell", "cell_id": cid, "completed": True})@@ -904,8 +853,6 @@ def test_the_other_runners_race_does_not_excuse_a_regression_on_this_one(tmp_patit was excused solely by a race on a machine the result never touched -- and a corroboratedhead regression there would have shipped green."""- # The null raced on reasoning_toggle. The result's own runner did not: side A renders the- # same DOM in both repetitions, and head renders a different one in both.null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")result = _arm_payload(tmp_path, "result", regress = "reasoning_toggle", self_race = None)@@ -913,10 +860,8 @@ def test_the_other_runners_race_does_not_excuse_a_regression_on_this_one(tmp_patassert ("r100K", "reasoning_toggle") in imported, importedassert derived- # Scored against the imported set as it stands, the regression is excused and the job is GREEN.assert U.report([result], "imported", imported, min_reps = 2, min_compared = 16) == 0- # Confined to the runner being scored, the exemption does not survive and the job is RED.local_unstable, local_stable = U.in_arm_repeatability([result])assert ("r100K", "reasoning_toggle") in local_stableassert ("r100K", "reasoning_toggle") not in local_unstable@@ -927,8 +872,6 @@ def test_the_other_runners_race_does_not_excuse_a_regression_on_this_one(tmp_patdef test_an_exemption_this_runner_reproduces_is_kept(tmp_path):- # The other direction, and the one that keeps the gate usable. When the scored runner races on- # the same action, the exemption is doing real work and removing it would red a sound run.null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")result = _arm_payload(tmp_path, "result", regress = None, self_race = "reasoning_toggle")imported, _derived, _ = U.unstable_set([null])@@ -938,9 +881,8 @@ def test_an_exemption_this_runner_reproduces_is_kept(tmp_path):def test_an_action_this_runner_could_not_decide_keeps_its_exemption(tmp_path):- # UNDECIDED IS NOT STABLE. One repetition of side A is one observation, and reading it as- # "this runner says the action is repeatable" would turn a lost slot into a red job -- the- # exact direction the false-alarm data says actually happens.+ # UNDECIDED IS NOT STABLE: one repetition of side A is one observation, and reading it as+ # "this runner says the action is repeatable" would turn a lost slot into a red job.null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")rows = [json.loads(line)@@ -948,7 +890,6 @@ def test_an_action_this_runner_could_not_decide_keeps_its_exemption(tmp_path):.read_text(encoding = "utf-8").splitlines()]- # Drop side A's reasoning_toggle in rep1, leaving a single observation of it on this runner.rows = [rfor r in rows@@ -974,7 +915,7 @@ def test_an_action_this_runner_could_not_decide_keeps_its_exemption(tmp_path):def test_a_declared_exemption_is_never_dropped_by_a_runner_measurement(tmp_path):# The declared list is a standing claim about the app, not a measurement of a machine, so a- # machine cannot contradict it. Only the `(rung, action)` entries are runner-derived.+ # machine cannot contradict it; only the `(rung, action)` entries are runner-derived.null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")result = _arm_payload(tmp_path, "result", regress = None, self_race = None)imported, _derived, _ = U.unstable_set([null])@@ -983,9 +924,8 @@ def test_a_declared_exemption_is_never_dropped_by_a_runner_measurement(tmp_path)def test_the_verdict_confines_the_imported_set_when_driven_through_main(tmp_path, capsys):- # Driven through main() rather than report(), because the failure this guards against is not- # a wrong confinement, it is a correct one that never reaches the verdict. That exact shape --- # a value computed or parsed and then not forwarded -- has already shipped once in this file's+ # Driven through main() rather than report(), because the failure guarded against is a correct+ # confinement that never reaches the verdict: the same shape that shipped once in this file's# neighbour, where --min-reps was parsed and never passed to build().null = _arm_payload(tmp_path, "null", regress = None, self_race = "reasoning_toggle")result = _arm_payload(tmp_path, "result", regress = "reasoning_toggle", self_race = None)@@ -1053,7 +993,6 @@ def test_a_control_that_stopped_working_is_not_excused_by_its_digest_exemption(tThe assertion is not a digest, so the digest exemption does not reach it."""path = _expect_payload(tmp_path, "regressed", "stop_generation", failed_on = "treatment")- # Scored with stop_generation declared unstable, exactly as the gate scores it.assert "stop_generation" in U.UNSTABLE_ACTIONSassert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1printed = capsys.readouterr().out@@ -1062,22 +1001,20 @@ def test_a_control_that_stopped_working_is_not_excused_by_its_digest_exemption(tdef test_both_arms_failing_the_assertion_is_lost_coverage_not_a_difference(tmp_path):- # The fixture cannot reach the state on either build. That is worth knowing and it is not a- # statement about the change under review, so it must not red the job.path = _expect_payload(tmp_path, "both", "stop_generation", failed_on = None, both = True)assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 0def test_an_action_that_asserts_nothing_is_not_an_assertion_failure(tmp_path):# `expect_ok is None` is "this action makes no claim", which every payload recorded before the- # field existed also carries. Reading None as False would red every one of them.+ # field existed also carries, so reading None as False would red every one of them.path = _expect_payload(tmp_path, "none", "stop_generation", failed_on = None, expect_ok_value = None)assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 0- # Asserted on the signal as well as the exit code: read through the exit code alone this case- # is indistinguishable from the both-arms one, so a mutation that turns None into a failure- # would still pass here. `expect_regressed` is where the distinction actually lives.+ # Asserted on the signal as well as the exit code: through the exit code alone this case is+ # indistinguishable from the both-arms one, and `expect_regressed` is where the distinction+ # lives.row = {"ran": True,"expect_ok": None,@@ -1097,8 +1034,6 @@ def test_an_action_that_asserts_nothing_is_not_an_assertion_failure(tmp_path):def test_an_assertion_that_failed_in_one_repetition_of_two_is_not_counted(tmp_path, capsys):- # Held to the same corroboration bar as everything else: a build that cannot stop generation- # cannot stop it on either pass.path = _expect_payload(tmp_path, "flake", "stop_generation", failed_on = "treatment", reps = ("rep0",))@@ -1153,17 +1088,16 @@ def test_an_assertion_that_blames_a_different_arm_each_time_is_not_corroborated(def test_an_assertion_that_blames_the_same_arm_twice_still_fails_the_job(tmp_path):- # The other side of the same key, and the reason the fix is not just "require more". A build- # that consistently fails the assertion is exactly what this category exists to catch, and it- # must survive the direction keying.+ # The other side of the same key, and why the fix is not just "require more": a build that+ # consistently fails the assertion is what this category exists to catch and must survive the+ # direction keying.path = _expect_payload(tmp_path, "consistent", "stop_generation", failed_on = "treatment")assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1def test_one_arm_only_that_swaps_arms_between_repetitions_is_not_corroborated(tmp_path, capsys):- # The same defect on the one-arm-only category, which is the one the item was filed against.- # A precondition race that stops the treatment arm performing the action in rep0 and the base- # arm in rep1 says nothing about either build.+ # The same defect on the one-arm-only category: a precondition race stopping the treatment arm+ # in rep0 and the base arm in rep1 says nothing about either build.rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):for arm in ("base", "treatment"):@@ -1231,15 +1165,12 @@ def test_a_scoped_action_with_no_rows_in_the_null_is_undecided_not_absent(tmp_paassert report_["decided"] == [("r100K", "settings")]assert rc == 1, "an action the null never measured cannot license an excuse"- # The reason it matters: send_turn carries no measured entry and is excused by name anyway.unstable, _derived, _checks = U.unstable_set([null])assert ("r100K", "send_turn") not in unstableassert U.is_unstable(unstable, "send_turn", "r100K rep0")def test_a_missing_scoped_action_on_the_waived_list_is_still_waived(tmp_path):- # `image_upload` has no attachments button on this fixture, and the workflow waives it by- # name. Whether it produced rows or none at all, the waiver is the same statement.null = _scope_payload(tmp_path, "null", ("settings",))result = _scope_payload(tmp_path, "result", ("settings", "image_upload"))rc, report_ = U.audit_null(@@ -1252,8 +1183,6 @@ def test_a_missing_scoped_action_on_the_waived_list_is_still_waived(tmp_path):def test_an_action_outside_the_scope_is_not_required_to_exist(tmp_path):- # The reconciliation is against the SCOPE, not against the schedule. An action the result- # matched on needs no excuse, so the null owing it nothing is not a hole.null = _scope_payload(tmp_path, "null", ("settings",))result = _scope_payload(tmp_path, "result", ("settings",))rc, report_ = U.audit_null([null], frozenset(), U.actions_needing_an_excuse([result], 2))@@ -1281,8 +1210,8 @@ def test_a_broken_control_is_not_excused_because_its_digest_varies(tmp_path, capdef test_an_action_with_no_not_run_path_is_never_exempt_from_one_arm_only(tmp_path):- # `scroll_after` has no `not_run` in `scene/actions.py` at all, so a `ran: false` for it- # cannot be a race under any reading, yet the digest list exempted it.+ # `scroll_after` has no `not_run` in `scene/actions.py` at all, so a `ran: false` for it cannot+ # be a race under any reading, yet the digest list exempted it.path = one_sided_payload(tmp_path, "scroll", "scroll_after", ("rep0", "rep1"))assert "scroll_after" in U.UNSTABLE_ACTIONSassert "scroll_after" not in P.RACY_EXECUTION@@ -1290,15 +1219,13 @@ def test_an_action_with_no_not_run_path_is_never_exempt_from_one_arm_only(tmp_padef test_every_racy_execution_entry_states_its_mechanism():- # The same bar the digest list is held to. An exemption without a stated mechanism is how the- # nine-action version of this list survived unexamined, and each of these has to name the- # `not_run` it is excusing.+ # The same bar the digest list is held to: an exemption without a stated mechanism is how the+ # nine-action version survived unexamined, and the markers are the operative half, since an+ # entry matching nothing, or everything, is the failure this keying prevents.for action, (why, markers) in P.RACY_EXECUTION.items():assert action in P.UNSTABLE_ACTIONS, actionassert "not_run" in why or "not run" in why, actionassert len(why) > 60, action- # And the markers are the operative half: an entry that documents a mechanism but matches- # nothing, or matches everything, is the failure this keying exists to prevent.assert markers and all(len(m) > 10 for m in markers), action@@ -1319,16 +1246,14 @@ def test_a_racy_execution_action_is_not_put_in_the_audit_scope(tmp_path):reason = "nothing was generating and a new turn did not start within 8s",)assert "stop_generation" in P.RACY_EXECUTION- # The verdict does not count it.assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 0- # So it must not be in the scope either.assert ("r100K", "stop_generation") not in U.actions_needing_an_excuse([path], 2)def test_a_direction_reversing_one_sided_pair_is_not_put_in_the_audit_scope(tmp_path):- # Same rule, the other axis. `report` keys corroboration on the live arm, so a pair blaming- # opposite arms is UNCORROBORATED and cannot move the verdict; built here without the- # direction it corroborated, entered the scope, and an undecided null failed a passing job.+ # Same rule, the other axis: `report` keys corroboration on the live arm, so a pair blaming+ # opposite arms cannot move the verdict, yet built without the direction it entered the scope+ # and an undecided null failed a passing job.rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):for arm in ("base", "treatment"):@@ -1352,8 +1277,6 @@ def test_a_direction_reversing_one_sided_pair_is_not_put_in_the_audit_scope(tmp_def test_a_control_the_head_cannot_open_is_still_in_the_audit_scope(tmp_path):- # The other side, so the two fixes above cannot quietly empty the scope: an action the- # verdict DOES fail on is exactly what the null has to have decided.path = one_sided_payload(tmp_path, "real", "settings", ("rep0", "rep1"))assert U.report([path], "t", frozenset(), min_reps = 2, min_compared = 16) == 1assert ("r100K", "settings") in U.actions_needing_an_excuse([path], 2)@@ -1376,13 +1299,10 @@ def test_a_removed_stop_button_is_not_exempt_just_because_stop_generation_can_ra)assert not P.racy_execution("stop_generation", "the stop button is not present")assert U.report([path], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1- # And the null is asked about it, because the verdict now turns on it.assert ("r100K", "stop_generation") in U.actions_needing_an_excuse([path], 2)def test_a_missing_composer_is_not_exempt_just_because_send_turn_can_be_queued(tmp_path):- # The same for send_turn: "a reply was still streaming" is the previous turn overrunning,- # "no composer on the page" is the build having removed the composer.racy = one_sided_payload(tmp_path,"queued",@@ -1398,28 +1318,18 @@ def test_a_missing_composer_is_not_exempt_just_because_send_turn_can_be_queued(tassert U.report([gone], "t", U.UNSTABLE_ACTIONS, min_reps = 2, min_compared = 16) == 1-# ── the reply that was still arriving is not a blind capture ──────────-#-# WHAT TURNED RED. A wave of `studiobench UI parity` audited UNDECIDED with 13 decided, 0 differed-# and 2 undecided: `keystroke@r100K` and `send_turn@r100K`, both "never reached min_observations"-# despite `--min-reps 2` and both having run on both arms in both repetitions. They are the two-# actions whose slot lands INSIDE a live turn on the packed 100K film, and the payload says so:-# `streaming: true`, `in_flight: [19]` and `[22]`, the in-flight message 18,277 characters on one-# arm and 18,253 on the other.-#-# WHERE IT CAME FROM. Pre-#9575 that pair scored DIFFER, which is wrong but is an OBSERVATION, so-# the null decided the action and the audit passed. #9575 correctly demoted it to NOT_COMPARABLE,-# and `derive_unstable` files every NOT_COMPARABLE under `blind`, which is zero observations.-# Replayed against the real payload: keystroke scores 2 observations pre-#9575 and 0 after.-#-# So the refusal is right and the accounting behind it was not. `SETTLED_MATCH` is the narrow-# repair: that branch is reached only once the scaffold, overlays, message set and every settled-# row have agreed, so it is a complete reading of everything the pair could read.-#-# Held in both directions, because a fix that makes the audit easier to pass is worth nothing-# without the control that shows it still fails.+# WHAT TURNED RED: a wave of `studiobench UI parity` audited UNDECIDED with `keystroke@r100K`+# and `send_turn@r100K` "never reached min_observations" despite --min-reps 2 and both arms+# running both repetitions: the two actions whose slot lands INSIDE a live turn on the packed+# 100K film. Pre-#9575 that pair scored DIFFER, which is wrong but is an OBSERVATION; #9575+# correctly demoted it to NOT_COMPARABLE, which `derive_unstable` files under `blind` (zero+# observations). So the refusal is right and the accounting was not: `SETTLED_MATCH` is the+# narrow repair, reached only once the scaffold, overlays, message set and every settled row+# have agreed. Held in both directions, since a fix that makes the audit easier to pass is+# worth nothing without the control showing it still fails.+# ── the reply that was still arriving is not a blind capture ──────────def streaming_pair_rows(cid_base: str,cid_treat: str,@@ -1503,9 +1413,9 @@ def streaming_null(tmp_path: Path, name: str, cells: list[list[dict]]) -> Path:def test_a_mid_stream_tail_is_still_a_refusal_and_never_a_pass():- # FIRST, the thing that must not change: the verdict stays NOT_COMPARABLE. A treatment build- # whose live message renders wrongly sits in exactly this shape, so promoting it to MATCH- # would hide a regression. The repair is in what the NULL counts, not in what the result says.+ # FIRST, the thing that must not change: the verdict stays NOT_COMPARABLE, because a treatment+ # build whose live message renders wrongly sits in exactly this shape. The repair is in what+ # the NULL counts.rows = streaming_pair_rows("r100K.base.rep0","r100K.treatment.rep0",@@ -1517,13 +1427,10 @@ def test_a_mid_stream_tail_is_still_a_refusal_and_never_a_pass():assert got["verdict"] == P.NOT_COMPARABLEassert "STILL BEING WRITTEN" in got["reason"]assert got["moved"] == []- # And it carries the flag that says the rest of the thread WAS read and did agree.assert got[P.SETTLED_MATCH] is Truedef test_a_refusal_that_read_the_settled_thread_is_an_observation():- # The unit the audit is built on. Two settled-match refusals are two observations of- # non-difference, so the action is decided and is NOT called unstable.rows = streaming_pair_rows("r100K.base.rep0","r100K.treatment.rep0",@@ -1542,9 +1449,9 @@ def test_a_refusal_that_read_the_settled_thread_is_an_observation():def test_an_ordinary_refusal_is_still_blind():- # THE CONTROL. Without it the change reads as "count NOT_COMPARABLE as an observation", which- # would derive an action that never rendered at all as stable. A failed capture carries no- # settled reading and stays at zero.+ # THE CONTROL: without it the change reads as "count NOT_COMPARABLE as an observation", which+ # would derive an action that never rendered as stable. A failed capture carries no settled+ # reading and stays at zero.blind = {"verdict": P.NOT_COMPARABLE, "reason": "threadRoot is not a function"}row = P.derive_unstable([("keystroke", blind), ("keystroke", blind)])["keystroke"]assert row["observations"] == 0@@ -1554,8 +1461,6 @@ def test_an_ordinary_refusal_is_still_blind():def test_a_settled_message_that_moved_is_a_difference_not_an_observation():- # One layer down: the flag is only set when everything OUTSIDE the live message agreed. Move- # a settled row and the pair is a DIFFER, so real instability is recorded as before.rows = streaming_pair_rows("r100K.base.rep0","r100K.treatment.rep0",@@ -1570,9 +1475,8 @@ def test_a_settled_message_that_moved_is_a_difference_not_an_observation():def test_a_scaffold_that_moved_is_a_difference_not_an_observation():- # The scaffold half of the same claim. Both arms agree on streaming and on the composer- # control, so this is not the generation-disagreement refusal but a scaffolding change on an- # otherwise identical pair, and it stays a finding.+ # The scaffold half of the same claim: both arms agree on streaming and on the composer+ # control, so this is a scaffolding change on an otherwise identical pair and stays a finding.rows = streaming_pair_rows("r100K.base.rep0","r100K.treatment.rep0",@@ -1587,11 +1491,10 @@ def test_a_scaffold_that_moved_is_a_difference_not_an_observation():def test_a_live_row_whose_role_changed_carries_no_positive_reading():- # The control `_any_moved` cannot reach, which is what makes it worth asserting. The role- # sits beside the digest and an in-flight digest is withheld, so `assistant` on one arm and- # `user` on the other passes the digest comparison untouched and the pair still reads as- # "everything settled agreed". `settled_messages_moved` calls that a structural change even- # in flight, so the flag must not fire here.+ # The control `_any_moved` cannot reach: the role sits beside the digest and an in-flight+ # digest is withheld, so `assistant` on one arm and `user` on the other passes the digest+ # comparison untouched, and `settled_messages_moved` calls that a structural change even in+ # flight.rows = streaming_pair_rows("r100K.base.rep0","r100K.treatment.rep0",@@ -1605,7 +1508,6 @@ def test_a_live_row_whose_role_changed_carries_no_positive_reading():got = P.compare(base, treat)assert got["verdict"] == P.NOT_COMPARABLEassert P.SETTLED_MATCH not in got- # And so it stays blind: two of them decide nothing and the audit still has to say undecided.row = P.derive_unstable([("keystroke", got), ("keystroke", got)])["keystroke"]assert row["observations"] == 0assert row["not_comparable"] == 2@@ -1614,10 +1516,9 @@ def test_a_live_row_whose_role_changed_carries_no_positive_reading():def test_a_settled_match_cannot_supply_the_observation_that_mints_an_exemption():# DECIDING an action and CLASSIFYING it unstable are different claims, and only the first is- # this flag's to make. One DIFFER beside one settled-match refusal is one differing comparison,- # so letting the refusal supply the second observation would derive `(rung, action)` into the- # MEASURED exemption set from a reading that never saw the live subtree -- the excuse set- # growing, which `SETTLED_MATCH` promises it cannot do.+ # this flag's to make: one DIFFER beside one settled-match refusal is one differing+ # comparison, so letting the refusal supply the second would grow the MEASURED exemption set+ # from a reading that never saw the live subtree.rows = streaming_pair_rows("r100K.base.rep0","r100K.treatment.rep0",@@ -1632,21 +1533,18 @@ def test_a_settled_match_cannot_supply_the_observation_that_mints_an_exemption()assert row["observations"] == 2assert row["differed"] == 1assert row[P.SETTLED_MATCH] == 1- # NOT unstable, because only one of the two readings compared anything, and not "decided and- # stable" either: one differing comparison is the single flake `min_observations` exists for.- # On the raw total this reached `cross_check` as `declared_stable_in_practice`, whose stated- # meaning is that the null never saw this action differ.+ # NOT unstable, because only one reading compared anything, and not "decided and stable"+ # either: one differing comparison is the single flake `min_observations` exists for. On the+ # raw total this reached `cross_check` as `declared_stable_in_practice`.assert row["unstable"] is Falseassert row["undetermined"] is True- # Two complete comparisons still classify exactly as before.both = P.derive_unstable([("keystroke", differ), ("keystroke", differ)])["keystroke"]assert both["unstable"] is Truedef test_the_audit_decides_an_action_whose_only_leftover_was_the_live_reply(tmp_path):# END TO END, the case that turned the job red: two repetitions of `keystroke` inside a live- # turn, the settled thread identical on every arm and only the in-flight tail a chunk apart.- # Nothing structural moved on this build against itself, and the audit must be able to say so.+ # turn, the settled thread identical and only the in-flight tail a chunk apart.null = streaming_null(tmp_path,"inflight",@@ -1672,14 +1570,13 @@ def test_the_audit_decides_an_action_whose_only_leftover_was_the_live_reply(tmp_assert report["decided"] == [("r100K", "keystroke")]assert report["undecided"] == []assert report["differed"] == []- # Reported as the narrower reading it is, rather than folded in with a plain MATCH.assert report["decided_on_settled_thread"] == [("r100K", "keystroke")]def test_the_audit_still_fails_when_the_capture_itself_was_blind(tmp_path):- # THE PAIRED FAILURE. Same scope and repetitions, but the arms could not place the stream at- # all (`in_flight_unplaced`: the `data-status` hook went quiet). No settled reading sits- # behind that refusal, so the audit stays red.+ # THE PAIRED FAILURE: same scope and repetitions, but the arms could not place the stream at+ # all (`in_flight_unplaced`), so no settled reading sits behind the refusal and the audit+ # stays red.cells = []for rep, tails in (("rep0", ("T1", "T2")), ("rep1", ("T3", "T4"))):cell = streaming_pair_rows(@@ -1700,9 +1597,8 @@ def test_the_audit_still_fails_when_the_capture_itself_was_blind(tmp_path):def test_a_settled_match_can_never_grow_the_excuse_set(tmp_path):- # THE SAFETY DIRECTION, asserted rather than argued. The observation is a NON-difference, so- # the derived excuse set can only narrow: a null made entirely of settled-match refusals- # derives an EMPTY measured set, so a corroborated difference gets no excuse from it.+ # THE SAFETY DIRECTION, asserted rather than argued: the observation is a NON-difference, so a+ # null made entirely of settled-match refusals derives an EMPTY measured set.null = streaming_null(tmp_path,"narrow",diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.pyindex 50ddc99b165..801e3c54c2b 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py@@ -104,8 +104,8 @@ def quiet_null(tmp: Path, name: str, actions: list[str]) -> Path:def test_only_the_stable_differences_are_illustrated(tmp_path, capsys):shots = tmp_path / "shots"- # `settings` differs and is stable here; `stop_generation` differs and is DECLARED unstable,- # so it is noise the reader must not be handed.+ # `settings` differs and is stable here; `stop_generation` differs and is DECLARED unstable, so+ # it is noise the reader must not be handed.result = result_with(tmp_path,"result",@@ -205,8 +205,8 @@ def test_the_two_halves_are_labelled_in_the_image_not_only_the_filename(tmp_path},)got = Image.open(out).convert("RGB")- # The banner strip must carry ink on both halves. A pair whose sides are distinguished only- # by filename is one rename away from being misattributed.+ # The banner strip must carry ink on both halves: a pair whose sides are distinguished only by+ # filename is one rename away from being misattributed.left = got.crop((0, 0, 320, S.BANNER)).getcolors(maxcolors = 100000)right = got.crop((320 + S.GUTTER, 0, 640 + S.GUTTER, S.BANNER)).getcolors(maxcolors = 100000)assert len(left) > 1, "the BEFORE banner drew nothing"@@ -227,9 +227,8 @@ def test_shot_index_reads_the_payload_rather_than_globbing(tmp_path):def test_a_one_repetition_flake_is_not_illustrated_at_the_verdicts_threshold(tmp_path, capsys):- # `settings` differs on both passes and is what fails the job; `thread_reopen` differs on one- # and is explicitly uncorroborated. Shipping both leaves the reader unable to tell which is- # which, which buries the finding the artifact exists to show.+ # `settings` differs on both passes and fails the job; `thread_reopen` differs on one and is+ # explicitly uncorroborated. Shipping both leaves the reader unable to tell which is which.shots = tmp_path / "shots"rows = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):@@ -249,9 +248,8 @@ def test_a_one_repetition_flake_is_not_illustrated_at_the_verdicts_threshold(tmpdef test_the_workflow_illustrates_at_the_same_threshold_it_scores_at():- # Two numbers in two steps of one job that must agree, and nothing at runtime would notice- # if they drifted: the verdict would fail on one set of actions and the artifact would show- # another.+ # Two numbers in two steps of one job that must agree, with nothing at runtime to notice if they+ # drifted: the verdict would fail on one set of actions and the artifact would show another.text = (Path(__file__).resolve().parents[5] / ".github/workflows/studiobench-ui-parity.yml").read_text(encoding = "utf-8")@@ -262,10 +260,10 @@ def test_the_workflow_illustrates_at_the_same_threshold_it_scores_at():def test_the_cli_forwards_the_threshold_it_was_given(tmp_path, capsys):- # `build` takes min_reps and `main` parses --min-reps, and between the two the value was- # dropped: the workflow asked for 2, `build` defaulted back to 1, and the artifact carried a- # one-repetition flake beside the change that actually failed the job. Driven through main()- # rather than build(), because build() was never the half that was wrong.+ # `build` takes min_reps and `main` parses --min-reps, and the value was dropped between them:+ # the workflow asked for 2, `build` defaulted back to 1, and the artifact carried a+ # one-repetition flake beside the change that failed the job. Driven through main() because+ # build() was never the half that was wrong.shots = tmp_path / "shots"rows = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):@@ -301,10 +299,10 @@ def test_the_cli_forwards_the_threshold_it_was_given(tmp_path, capsys):def test_the_workflow_actually_runs_the_prune_it_documents():- # The images are the only large thing this job produces and the payload upload is- # `if: always()`, so a green run shipped every one of them -- 18 actions x 2 arms x 2- # repetitions per job -- while the step below it claimed the arms had already deleted the- # matching ones. A command nobody invokes cannot be the reason a claim is true.+ # The images are the only large thing this job produces and the payload upload is `if: always()`,+ # so a green run shipped every one of them (18 actions x 2 arms x 2 repetitions per job) while+ # the step below claimed the arms had already deleted the matching ones. A command nobody invokes+ # cannot be the reason a claim is true.text = (Path(__file__).resolve().parents[5] / ".github/workflows/studiobench-ui-parity.yml").read_text(encoding = "utf-8")@@ -333,9 +331,9 @@ def _one_sided_rows(action_name: str, reps: tuple[str, ...], shots: Path) -> lisdef test_a_control_that_only_one_arm_can_operate_is_illustrated(tmp_path, capsys):# The regression shape that leaves NO digest to differ: a control that stops opening records- # `ran: false`, not a different hash. `ui_parity.report` exits 1 on it, and the arm-side prune- # keeps its shots for exactly this step -- but the selector only accepted DIFFER, so a red- # verdict caused by it published "no STABLE differences" and no composite at all.+ # `ran: false`, not a different hash. `ui_parity.report` exits 1 on it and the arm-side prune+ # keeps its shots for this step, but the selector only accepted DIFFER, so a red verdict+ # published 'no STABLE differences' and no composite at all.shots = tmp_path / "shots"result = write(tmp_path, "result", _one_sided_rows("settings", ("rep0", "rep1"), shots))null = quiet_null(tmp_path, "null", ["settings"])@@ -347,9 +345,8 @@ def test_a_control_that_only_one_arm_can_operate_is_illustrated(tmp_path, capsysdef test_a_one_arm_miss_in_one_repetition_of_two_is_not_illustrated(tmp_path):- # The same bar the verdict uses. A contended runner can lose one arm's slot once, and that- # does not red the job -- so putting a picture of it beside the change that did would bury- # the finding the artifact exists to show.+ # The same bar the verdict uses. A contended runner can lose one arm's slot once without redding+ # the job, so a picture of it beside the change that did would bury the finding.shots = tmp_path / "shots"result = write(tmp_path, "result", _one_sided_rows("settings", ("rep0",), shots))null = quiet_null(tmp_path, "null", ["settings"])@@ -359,12 +356,10 @@ def test_a_one_arm_miss_in_one_repetition_of_two_is_not_illustrated(tmp_path):def test_a_superseded_attempts_screenshot_is_not_shown_for_the_retrys_verdict(tmp_path):- # `--resume` re-runs an A/B pair WHOLE, so an arm that already succeeded is re-run under the- # same deterministic cell id. If the retry captures the differing digest but its screenshot- # fails, a raw scan of the append-only payload keeps the DEAD attempt's `shot` -- because the- # newer row carries none -- and the artifact pairs the retry's verdict with a picture of a- # page that is not the one that turned the gate red. A missing half is a caption `build`- # already prints; a stale half is a lie.+ # `--resume` re-runs an A/B pair WHOLE under the same deterministic cell id, so if the retry+ # captures the differing digest but its screenshot fails, a raw scan of the append-only payload+ # keeps the DEAD attempt's `shot` and pairs the retry's verdict with a picture of a different+ # page. A missing half is a caption `build` already prints; a stale half is a lie.shots = tmp_path / "shots"png(shots / "old.png", 200, 120, (10, 60, 10))png(shots / "base.png", 200, 120, (10, 60, 10))@@ -409,17 +404,17 @@ def test_a_superseded_attempts_screenshot_is_not_shown_for_the_retrys_verdict(tmdef test_the_workflow_runs_the_gate_on_the_routes_the_measurement_depends_on():# A gate that does not fire on a change that can move what it measures is not measuring it.- # chat-api.ts sends the scripted turn to /api/inference/chat/completions, and lifecycle.py- # logs each arm in through /api/auth/login and /api/auth/change-password before any scene- # renders -- so both can alter, empty, or block the measured DOM on a PR with no frontend file.+ # chat-api.ts sends the scripted turn to /api/inference/chat/completions, and lifecycle.py logs+ # each arm in through /api/auth/login and /api/auth/change-password before any scene renders, so+ # both can alter, empty or block the measured DOM on a PR with no frontend file.text = (Path(__file__).resolve().parents[5] / ".github/workflows/studiobench-ui-parity.yml").read_text(encoding = "utf-8")for route in ("inference.py", "auth.py", "chat_history.py", "providers.py"):assert f"studio/backend/routes/{route}" in text, route- # A route is not one file. Its response models, its provider lookups and its storage can each- # change the measured picker without the route module being touched, which is how the same- # item arrived three rounds running, one file further down each time.+ # A route is not one file: its response models, provider lookups and storage can each change the+ # measured picker without the route module being touched, which is how the same item arrived+ # three rounds running, one file further down each time.for dep in ("studio/backend/storage/studio_db.py","studio/backend/models/providers.py",@@ -440,8 +435,8 @@ def test_the_evidence_uses_the_same_confined_set_the_verdict_scored_with(tmp_patas publishing no evidence at all, one round after it was fixed the first time."""shots = tmp_path / "shots"- # The null raced on `settings`. This runner did not: side A is identical across both- # repetitions, and head differs in both.+ # The null raced on `settings`. This runner did not: side A is identical across both repetitions,+ # and head differs in both.null_rows = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):for arm in ("base", "treatment"):@@ -468,9 +463,9 @@ def test_the_evidence_uses_the_same_confined_set_the_verdict_scored_with(tmp_patdef test_a_direction_reversing_pair_is_not_illustrated_either(tmp_path):- # The verdict does not fail on a pair whose two repetitions blame opposite arms, so the- # artifact must not present one as the change that turned the job red. The two selections are- # keyed the same way for the same reason they are corroborated the same way.+ # The verdict does not fail on a pair whose two repetitions blame opposite arms, so the artifact+ # must not present one as the change that turned the job red. The two selections are keyed the+ # same way for the same reason they are corroborated the same way.shots = tmp_path / "shots"rows = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):@@ -539,14 +534,12 @@ def test_two_shards_do_not_share_one_screenshot_identity(tmp_path):# Direct first-party imports of a measured route that are deliberately NOT gated on, each with-# the reason. This is the other half of the filter: a module is either in the workflow or it is-# here, and adding an import to a measured route fails the test below until someone decides which.-#-# The drip this ends: routes/chat_history.py was listed, then its storage/studio_db.py arrived a-# round later, then models/providers.py and core/inference/providers.py, then external_provider.py-# and sse_control_frames.py, then provider_credentials.py. Every one was correct and every one was-# found by reading the imports by hand. The boundary is a judgement, but it should be a RECORDED-# judgement rather than one rediscovered each round.+# the reason. A module is either in the workflow or here, and adding an import to a measured+# route fails the test below until someone decides which.+# The drip this ends: routes/chat_history.py was listed, then storage/studio_db.py a round later,+# then models/providers.py and core/inference/providers.py, then external_provider.py and+# sse_control_frames.py, then provider_credentials.py, every one correct and every one found by+# reading imports by hand. The boundary is a judgement, but a RECORDED one.NOT_GATED: dict[str, str] = {"studio/backend/auth/authentication.py": "session plumbing shared by every route in the app; ""routes/auth.py already gates the endpoints studiobench actually calls",@@ -594,7 +587,7 @@ def test_every_import_of_a_measured_route_is_gated_or_explicitly_waived():def test_every_waiver_names_a_file_that_exists_and_gives_a_reason():# A waiver for a file that has been moved or deleted is a stale exemption that reads as a- # decision. Same bar the RACY_EXECUTION entries are held to.+ # decision. Same bar as the RACY_EXECUTION entries.repo = Path(__file__).resolve().parents[5]for path, why in NOT_GATED.items():assert (repo / path).exists(), pathdiff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.pyindex ecf718ed183..95c73d69e81 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_partial_censoring.py@@ -56,8 +56,7 @@ def _payload(tmp_path: Path) -> Path:"completed": True,})- # A control that is measured at BOTH rungs, so the refusal cannot pass by- # rejecting everything.+ # A control that is measured at BOTH rungs, so the refusal cannot pass by rejecting everything.rows.append({"row_type": "action",diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.pyindex 01c346802c4..0249e432048 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_resumed_collision.py@@ -48,8 +48,8 @@ METRIC = "message_menu.open_close_ms"#: Wall clock per session. A resume starts after the run it continues has stopped; the real-#: two-launcher payload of defect 9 overlaps by sixteen minutes, which is what `_at` reproduces-#: for the concurrent fixtures.+#: two-launcher payload of defect 9 overlaps by sixteen minutes, which is what `_at` reproduces for+#: the concurrent fixtures._STARTED = {"s1": "2026-08-23T00:00:00","t2": "2026-08-23T02:00:00",diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.pyindex dfdab1e0406..4fb5571cfb3 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_session_isolation.py@@ -64,8 +64,8 @@ def test_cell_metrics_refuses_to_collapse_two_sessions():"cell_metrics keyed on cell_id alone and returned the last writer's values. That is how a ""payload holding two concurrent runs reported a 149.8% regression that does not exist.")- # The colliding cell is NAMED, not just counted. A refusal that says only "two sessions" sends- # the reader back to the payload to find out which reading it was protecting them from.+ # The colliding cell is NAMED, not just counted: a refusal that says only "two sessions" sends the+ # reader back to the payload to find out which reading it was protecting them from.assert "r1M.base.rep0" in messageassert "91c4d6d94da8" in message and "430f0b831dda" in messagediff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.pyindex baff730dad0..e9ef3c0d7ea 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py@@ -88,7 +88,7 @@ def test_a_large_consistent_effect_over_a_tight_floor_passes(tmp_path):def test_an_effect_under_the_floor_is_void(tmp_path):- # 3% claimed, against a null control whose own two identical builds land 20% apart.+ # 3% claimed, against a null control whose two identical builds land 20% apart.assert (verdict(tmp_path,@@ -100,9 +100,8 @@ def test_an_effect_under_the_floor_is_void(tmp_path):def test_the_floor_clears_the_null_controls_bias_not_only_its_spread(tmp_path):- # A null control that is TIGHT but systematically offset: identical builds, yet the treatment- # label reads 10% faster on every repetition. Spread is nearly zero, so a floor built from- # spread alone would admit any effect at all. The bar is max(|bias|, spread).+ # A null control that is tight but systematically offset: spread alone would admit any+ # effect, so the bar is max(|bias|, spread).floor_pairs = [(1000.0, 900.0)] * 4assert verdict(tmp_path, [(1000.0, 950.0)] * 4, floor_pairs) == "VOID (under floor)"assert verdict(tmp_path, [(1000.0, 700.0)] * 4, floor_pairs) == "faster"@@ -112,7 +111,7 @@ def test_the_floor_clears_the_null_controls_bias_not_only_its_spread(tmp_path):def test_pairs_that_disagree_on_sign_are_void_however_large_the_mean(tmp_path):- # Mean lands well past the floor; two repetitions say faster and two say slower.+ # Mean past the floor; two repetitions say faster and two say slower.assert (verdict(tmp_path,@@ -127,8 +126,7 @@ def test_pairs_that_disagree_on_sign_are_void_however_large_the_mean(tmp_path):def test_an_effect_smaller_than_its_own_scatter_is_void(tmp_path):- # Every repetition agrees on the direction and the mean clears the floor, but the readings- # range from 2% to 60% faster. Twelve rows in a 40-comparison audit looked exactly like this.+ # Every repetition agrees and the mean clears the floor, but readings range from 2% to 60% faster.assert (verdict(tmp_path,@@ -151,8 +149,7 @@ def test_gate_three_does_not_fire_on_a_tight_large_effect(tmp_path):def test_gate_three_cannot_fire_on_a_single_pair(tmp_path):- # With n=1 the spread is 0 by construction, so the gate has nothing to say. It must not- # silently pass a single reading off as having survived a scatter check.+ # With n=1 the spread is 0 by construction, so the gate must not pass a single reading off as scatter-checked.result = F.summarise([payload(tmp_path, "result", [(1000.0, 500.0)])])assert result["message_menu.open_close_ms"]["n"] == 1_f, v = F.verdict_for(@@ -203,14 +200,13 @@ def test_a_count_is_harvested_under_a_name_that_marks_it_as_an_invariant(tmp_patdef test_a_count_that_fell_reads_as_a_loss_and_never_as_faster(tmp_path):- # The regression this exists to catch: virtualization truncates select-all from 400k characters- # to 3k. Every timing improves, `expect_ok` stays true because chars > 0, and the only thing- # that can say so is this row. Scored as a timing it would read "faster".+ # The regression this catches: virtualization truncates select-all from 400k chars to 3k.+ # Every timing improves, expect_ok stays true, and only this row can say so.stats = F.summarise([count_payload(tmp_path, "r", [(400000.0, 3000.0)] * 4)])floor = {"delta_pct": 0.0, "spread_pct": 1.0}_f, v = F.verdict_for(stats[COUNT_METRIC], floor, F.is_count_metric(COUNT_METRIC))assert v == "LOST (invariant fell)"- # Same numbers, scored as a timing, would have been reported as an improvement.+ # Same numbers scored as a timing would read as an improvement.assert F.verdict_for(stats[COUNT_METRIC], floor, False)[1] == "faster"@@ -232,7 +228,7 @@ def test_a_lost_invariant_is_printed_and_counted_by_the_table(tmp_path, capsys):survivors = F.render([result], "t", floors = floors)out = capsys.readouterr().outassert "LOST (invariant fell)" in out- # It has to be counted as a finding, not dropped for not being one of the two timing verdicts.+ # Counted as a finding, not dropped for not being one of the two timing verdicts.assert survivors >= 1@@ -303,9 +299,8 @@ def test_scoring_against_a_floor_from_another_tier_is_refused(tmp_path):def test_pooling_across_corpora_is_refused(tmp_path):- # The tier fixes how long the film runs; the corpus hash fixes what is IN it. Corpus v2 added- # math, so a v1 payload and a v2 payload measure two different documents under one name, and- # pooling them would read the corpus change as a performance change.+ # The tier fixes how long the film runs, the corpus hash fixes what is IN it: pooling v1 and+ # v2 payloads reads a corpus change as a performance change.one = payload(tmp_path, "one", [(1000.0, 900.0)], corpus = "aaaa1111")two = payload(tmp_path, "two", [(1000.0, 900.0)], corpus = "bbbb2222")with pytest.raises(SystemExit) as exc:@@ -328,8 +323,7 @@ def test_the_same_corpus_on_both_sides_pools_normally(tmp_path):def test_a_payload_with_no_corpus_hash_is_not_silently_pooled_with_one_that_has_it(tmp_path):- # An older payload predating the field reads "?", which is a different value, not a wildcard.- # Treating it as compatible is how a v1 run would end up scored against a v2 floor.+ # An older payload predating the field reads '?', a different value, not a wildcard.old = payload(tmp_path, "old", [(1000.0, 900.0)], corpus = None)new = payload(tmp_path, "new", [(1000.0, 900.0)], corpus = "bbbb2222")with pytest.raises(SystemExit) as exc:@@ -338,11 +332,8 @@ def test_a_payload_with_no_corpus_hash_is_not_silently_pooled_with_one_that_has_def test_a_resumed_payload_carrying_two_corpora_is_refused(tmp_path):- # The recorder appends, so `--resume` into the same --out leaves the first run's completed- # cells next to a SECOND run_meta. If the corpus changed in between, that one file holds a- # base recorded on the old film and a treatment recorded on the new one, and `paired` matches- # them on (shard, rung, rep) without noticing. Reading only the first header would pass this- # payload and print the corpus change as a performance change.+ # `--resume` into the same --out leaves a second run_meta; one file can then hold a base on+ # the old film and a treatment on the new, and `paired` matches them regardless.out = tmp_path / "resumed"out.mkdir(parents = True, exist_ok = True)rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "corpus_hash": "aaaa1111"}]@@ -356,14 +347,14 @@ def test_a_resumed_payload_carrying_two_corpora_is_refused(tmp_path):with pytest.raises(SystemExit) as exc:F.load([path])assert "more than one corpus" in str(exc.value)- # And the floor-vs-result path refuses it too, rather than scoring it under the first hash.+ # And the floor-vs-result path refuses it too, rather than scoring under the first hash.with pytest.raises(SystemExit) as exc:F.render([path], "t", floors = {}, floor_corpus = "aaaa1111")assert "more than one corpus" in str(exc.value)def test_a_payload_with_repeated_headers_on_one_corpus_still_loads(tmp_path):- # A plain resume, nothing changed in between: two headers, one hash, no refusal.+ # A plain resume, nothing changed: two headers, one hash, no refusal.out = tmp_path / "plain"out.mkdir(parents = True, exist_ok = True)meta = {"row_type": "run_meta", "tier": "standard", "corpus_hash": "aaaa1111"}@@ -421,8 +412,7 @@ def test_an_incomplete_cell_is_not_measured(tmp_path):def test_a_shard_pairs_within_itself_and_never_across(tmp_path):- # Two shards both number their repetitions from rep0. Pairing on the repetition alone would- # cross them over and silently compare one session's base with another's treatment.+ # Both shards number repetitions from rep0; pairing on the repetition alone would cross them over.a = payload(tmp_path / "a", "s0", [(1000.0, 500.0)])b = payload(tmp_path / "b", "s0", [(2000.0, 1000.0)])pooled, _ = F.load([a, b])@@ -431,9 +421,8 @@ def test_a_shard_pairs_within_itself_and_never_across(tmp_path):def test_an_action_whose_own_assertion_failed_contributes_no_timing(tmp_path):- # `ran = True, expect_ok = False` is the action that happened and did not do its job. Its p95- # is lower BECAUSE it failed, so pairing it prints the failure as an improvement. The payload- # notes layer already says these timings must not be quoted.+ # `ran=True, expect_ok=False` is the action that happened and failed; its p95 is lower BECAUSE+ # it failed, so pairing prints the failure as an improvement.rows = [{"row_type": "run_meta", "tier": "standard"},{"row_type": "cell", "cell_id": "100K.base.rep0", "completed": True},@@ -456,8 +445,8 @@ def test_an_action_whose_own_assertion_failed_contributes_no_timing(tmp_path):def test_an_action_with_no_expectation_recorded_is_still_harvested(tmp_path):- # `expect_ok` is None on an action that asserts nothing, and on every payload recorded before- # the field existed. Only an explicit False is a failed assertion.+ # `expect_ok` is None on an action that asserts nothing and on pre-field payloads; only+ # explicit False is a failed assertion.rows = [{"row_type": "run_meta", "tier": "standard"},{"row_type": "cell", "cell_id": "100K.base.rep0", "completed": True},@@ -570,7 +559,7 @@ def test_a_clean_zero_base_arm_still_pairs_so_a_jank_regression_is_not_lost(tmp_stats, floors = F.summarise([result]), F.summarise([null])s = stats["stream_time_in_jank_pct"]assert s["n"] == 4- # Compared by DIFFERENCE, in the metric's own unit: 0.0% to 12.0% is +12.0 points.+ # Compared by DIFFERENCE in the metric's own unit: 0.0% to 12.0% is +12.0 points.assert s["difference"] is Trueassert s["delta_pct"] == pytest.approx(12.0)assert "stream_time_in_jank_pct" in floors@@ -594,7 +583,7 @@ def test_an_unchanged_repetition_does_not_read_as_a_pair_disagreeing_on_sign(tmpsomething that did not happen is reporting a wrong answer in the right vocabulary, which isworth three lines on its own; it is not worth relaxing gate 3 to chase."""- # Two repetitions clean on both arms, two that regress. Nothing regresses the other way.+ # Two repetitions clean on both arms, two that regress, nothing the other way.result = stream_payload(tmp_path, "result", [(SMOOTH, SMOOTH), (SMOOTH, JANKY)] * 2)null = stream_payload(tmp_path, "null", [(SMOOTH, SMOOTH)] * 4)@@ -607,7 +596,7 @@ def test_an_unchanged_repetition_does_not_read_as_a_pair_disagreeing_on_sign(tmp]assert s["consistent"] is True, "an unchanged repetition is a tie, not a disagreement"_f, v = F.verdict_for(s, F.summarise([null])["stream_time_in_jank_pct"])- # Still refused, and that is right, but no longer refused for a reason that did not occur.+ # Still refused, and rightly, but no longer for a reason that did not occur.assert v == "VOID (effect under its own scatter)"assert v != "VOID (pairs disagree on sign)"@@ -632,9 +621,7 @@ def test_a_ratio_metric_still_drops_a_zero_base_rather_than_dividing_by_it(tmp_pdef test_the_enforced_idle_window_is_not_pooled_into_the_frame_metrics(tmp_path):- # Every cell records a 1.5 s `idle:calibrate` window with the frame recorder running. Pooling- # its quiet into the film halves the jank share, so the column would not be the quantity the- # rest of the tool prints under that name.+ # Pooling each cell's 1.5s `idle:calibrate` quiet into the film halves the jank share.cid = "r100K.base.rep0"rows = [{"row_type": "run_meta", "tier": "standard"},@@ -648,9 +635,8 @@ def test_the_enforced_idle_window_is_not_pooled_into_the_frame_metrics(tmp_path)def test_a_resumed_cell_is_measured_from_its_own_attempt_only(tmp_path):- # `--resume` re-runs a cell that died, and the payload is append-only, so the dead attempt's- # windows sit in the file under the SAME cell id. Pooling them into the retry reports a number- # that no single run of the film ever produced.+ # `--resume` re-runs a dead cell into an append-only payload under the SAME cell id; pooling+ # both attempts reports a number no single run produced.cid = "r100K.base.rep0"rows = [{"row_type": "run_meta", "tier": "standard", "session_id": "s1"},@@ -666,8 +652,8 @@ def test_a_resumed_cell_is_measured_from_its_own_attempt_only(tmp_path):def test_one_file_holding_two_tiers_is_refused_like_two_files(tmp_path):- # The recorder appends, so a second run into the same --out leaves both films in one payload.- # Reading only the first run_meta let that file through the refusal it was written for.+ # A second run into the same --out leaves both films in one payload, and reading only the+ # first run_meta let that through.rows = [{"row_type": "run_meta", "tier": "fast"},*cell("100K", "base", "rep0", {"open_close_ms": 1000.0}),@@ -736,10 +722,9 @@ def resumed_payload(tmp_path: Path, name: str, retry_session: str) -> Path:def test_an_arm_resumed_into_a_new_session_is_not_paired_with_the_old_one(tmp_path):- # `--resume` skips the arm that completed and re-runs the one that died, under a NEW session id- # in the SAME shard directory. Keyed on the repetition alone the two pair, and the whole 8%- # session drift is charged to whichever arm was re-run. `scoring/ab.py` refuses exactly this- # comparison; the sweep's own pairing has to refuse it too.+ # `--resume` re-runs the dead arm under a NEW session id in the same shard; keyed on the+ # repetition alone the two pair and the 8% drift is charged to the re-run arm.+ # `scoring/ab.py` refuses exactly this.path = resumed_payload(tmp_path, "resumed", "s2")assert F.cell_metrics(F.read_rows(path))["r100K.treatment.rep0"] == {"message_menu.open_close_ms": 108.0@@ -748,8 +733,8 @@ def test_an_arm_resumed_into_a_new_session_is_not_paired_with_the_old_one(tmp_padef test_an_arm_resumed_inside_the_same_session_still_pairs(tmp_path):- # The other direction, so the refusal above cannot pass by rejecting every resumed run. Two- # attempts in ONE session are still one session, and the retry's own numbers pair normally.+ # The other direction, so the refusal cannot pass by rejecting every resumed run: two attempts+ # in ONE session still pair normally.path = resumed_payload(tmp_path, "same", "s1")assert F.paired(F.read_rows(path), shard = "same") == {"message_menu.open_close_ms": [(100.0, 108.0)]@@ -757,8 +742,8 @@ def test_an_arm_resumed_inside_the_same_session_still_pairs(tmp_path):def test_a_payload_with_no_session_ids_pairs_exactly_as_before(tmp_path):- # Payloads recorded before session ids existed carry none, so both arms resolve to "" and the- # new key term is inert. A refusal that also rejected these would delete every old reading.+ # Pre-session-id payloads resolve both arms to '', so the new key term is inert; refusing them+ # would delete every old reading.path = payload(tmp_path, "legacy", [(1000.0, 900.0)])assert F.paired(F.read_rows(path), shard = "legacy") == {"message_menu.open_close_ms": [(1000.0, 900.0)]@@ -789,8 +774,8 @@ def parity_action(cid: str, action: str, digest: str) -> dict:def test_a_mismatch_at_a_smaller_rung_survives_the_later_rungs(tmp_path):- # A standard-tier repetition walks 1K, 10K and 100K. Keyed on the repetition alone, the 100K- # rows overwrite the 1K ones and a real difference at 1K is reported as a pass.+ # A standard-tier repetition walks 1K, 10K and 100K; keyed on the repetition alone the 100K+ # rows overwrite the 1K ones.rows = [{"row_type": "run_meta", "tier": "standard"}]for rung, base_digest, treat_digest in (("r1K", "AAA", "BBB"), ("r100K", "CCC", "CCC")):rows.append(parity_action(f"{rung}.A0.rep0", "settings", base_digest))@@ -825,12 +810,9 @@ def parity_run(tmp_path: Path, name: str, cells: list[tuple[str, str, str, str]]def test_instability_measured_at_one_rung_does_not_silence_a_stable_rung(tmp_path):- # A standard-tier null control at the default --reps 1 walks 1K, 10K and 100K. `settings`- # races only at 100K, where the mounted thread is largest. Derived over the pooled action- # name, that single differing observation borrows the other two rungs' observation COUNT,- # clears min_observations, and marks `settings` unstable everywhere -- so a real DOM- # regression at 1K, a rung the null control measured clean, prints as expected variation and- # the command exits 0.+ # `settings` races only at 100K, but derived over the pooled action name that one observation+ # borrows the other rungs' COUNT, clears min_observations, and marks it unstable everywhere,+ # so a real 1K regression prints as expected variation and exits 0.null = parity_run(tmp_path,"null",@@ -853,10 +835,8 @@ def test_instability_measured_at_one_rung_does_not_silence_a_stable_rung(tmp_patdef test_instability_measured_at_a_rung_still_silences_that_rung(tmp_path):- # The other direction, so the scoping above cannot pass by never silencing anything. With two- # repetitions the null control has the observations to MEAN it at 100K, and a mismatch there- # is expected variation -- while the same action at 1K, measured clean, still carries a- # verdict.+ # The other direction, so the scoping cannot pass by never silencing anything: with two+ # repetitions the null control can mean it at 100K while 1K still carries a verdict.null = parity_run(tmp_path,"null2",@@ -878,13 +858,11 @@ def test_instability_measured_at_a_rung_still_silences_that_rung(tmp_path):def test_a_declared_unstable_action_still_holds_at_every_rung(tmp_path):- # A declared entry carries a MECHANISM that is a property of the action, so it is not scoped- # to a rung and a null control is not needed to honour it.- #- # NOT 1 is what this test is about: the difference is filed as expected variation rather than- # as a stable difference. It is 2 and not 0 because the only pair in this payload landed in- # that bucket, so nothing was compared at all -- see `report`'s `matched == 0` guard, which- # refuses to exit 0 over a run where the digest never produced a verdict.+ # A declared entry carries a MECHANISM that is a property of the action, so it is not+ # rung-scoped and needs no null control.+ # NOT 1: the difference is filed as expected variation rather than a stable difference. It is+ # 2 because the only pair landed in that bucket, so `report`'s `matched == 0` guard refuses to+ # exit 0 over a run that produced no verdict.rows = [{"row_type": "run_meta", "tier": "standard"}]rows.append(parity_action("r1K.base.rep0", "stop_generation", "A"))rows.append(parity_action("r1K.treatment.rep0", "stop_generation", "B"))@@ -927,11 +905,9 @@ def resumed_parity(tmp_path: Path, name: str, retry_session: str) -> Path:def test_a_resumed_arm_is_not_paired_with_its_partner_from_the_old_session(tmp_path):- # `--resume` re-runs the arm that died under a NEW session id in the SAME shard, so the- # completed partner is left under the old one. Keyed on the repetition alone the two pair, and- # every session-scoped volatile the normaliser missed reads as a difference between the arms.- # Two of them at one rung are enough for `derive_unstable` to call `settings` unstable AT THAT- # RUNG, and a real DOM regression at 100K then prints as expected variation and exits 0.+ # `--resume` re-runs the dead arm under a NEW session id, so session-scoped volatiles read as+ # arm differences; two at one rung make `derive_unstable` call `settings` unstable there and a+ # real 100K regression exits 0.null = resumed_parity(tmp_path, "null_resumed", "s2")unstable, _derived, _checks = U.unstable_set([null])assert ("r100K", "settings") not in unstable@@ -942,31 +918,27 @@ def test_a_resumed_arm_is_not_paired_with_its_partner_from_the_old_session(tmp_pdef test_a_cross_session_pair_carries_no_verdict_in_either_direction(tmp_path):- # And it is not silently dropped either. Both arms ran, so the reader is told the surface went- # unmeasured rather than being shown a pass -- which is the distinction the NOT COMPARABLE- # outcome exists to make.+ # And not silently dropped: both arms ran, so the reader is told the surface went unmeasured,+ # which is what NOT COMPARABLE exists to say.null = resumed_parity(tmp_path, "blind_resumed", "s2")verdicts = {r["verdict"] for _a, _s, _c, r in U.compare_all([null])[0]}assert verdicts == {U.P.NOT_COMPARABLE}- # NOT 1, because a cross-session pair is not a stable difference, and NOT 0 either: every pair- # here is NOT COMPARABLE, so the payload carries no verdict in either direction and `report`- # says so with 2 rather than letting CI go green on a run that compared nothing.+ # NOT 1 (a cross-session pair is not a stable difference) and NOT 0 either: every pair is NOT+ # COMPARABLE, so 2 rather than letting CI go green on a run that compared nothing.assert U.report([null], "t", U.UNSTABLE_ACTIONS) == 2def test_a_parity_arm_resumed_inside_the_same_session_still_pairs(tmp_path):- # The other direction, so the refusal above cannot pass by rejecting every resumed payload.- # Two attempts in ONE session are still one session, the pairs are real, and the instability- # they show is derived exactly as before.+ # The other direction: two attempts in ONE session are one session, the pairs are real, and+ # instability derives as before.null = resumed_parity(tmp_path, "same_session", "s1")unstable, _derived, _checks = U.unstable_set([null])assert ("r100K", "settings") in unstabledef test_a_parity_payload_with_no_session_ids_pairs_exactly_as_before(tmp_path):- # Payloads recorded before session ids existed carry none, so both arms resolve to "" and the- # new key term is inert. A refusal that also rejected these would blind the tool to every- # older run.+ # Pre-session-id payloads resolve both arms to '', so the term is inert; refusing them would+ # blind the tool to every older run.path = parity_run(tmp_path, "legacy_parity", [("r1K", "rep0", "A", "B")])assert U.report([path], "t", U.UNSTABLE_ACTIONS) == 1@@ -1019,10 +991,8 @@ def resumed_both_arms(tmp_path: Path, name: str) -> Path:def test_a_superseded_attempt_is_not_a_second_parity_observation(tmp_path):- # One repetition ran twice, so the payload holds two comparable pairs for it. Counted as two,- # the differing dead attempt and the matching retry are exactly `min_observations` and- # `derive_unstable` marks `settings` unstable at 100K on the strength of a reading the run- # itself threw away.+ # One repetition ran twice, so counting the dead attempt gives exactly `min_observations` and+ # marks `settings` unstable on a reading the run threw away.null = resumed_both_arms(tmp_path, "null_both_arms")assert len(U.collect([null])["pairs"]) == 1@@ -1034,18 +1004,15 @@ def test_a_superseded_attempt_is_not_a_second_parity_observation(tmp_path):def test_a_superseded_attempt_does_not_silence_a_real_parity_regression(tmp_path):- # The consequence, end to end. Scored against that unstable set, a genuine DOM difference on- # `settings` at 100K prints under "expected to vary" and the gate exits 0 on a regression.+ # The consequence end to end: a genuine 100K difference then prints under 'expected to vary' and the gate exits 0.unstable, _derived, _checks = U.unstable_set([resumed_both_arms(tmp_path, "null_silencer")])regressed = parity_run(tmp_path, "mine_both_arms", [("r100K", "rep0", "X", "REGRESSED")])assert U.report([regressed], "t", unstable) == 1def test_two_repetitions_of_one_pair_are_still_two_parity_observations(tmp_path):- # The control: superseding is keyed on the ATTEMPT, not on the cell id, so two repetitions of- # one rung inside one session remain two independent observations and still derive as- # unstable. A filter that kept only one reading per cell would pass the two tests above by- # deleting the evidence they are meant to preserve.+ # The control: superseding keys on the ATTEMPT, not the cell id, so two repetitions in one+ # session stay two observations.rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "session_id": "s1"}]for rep, treat in (("rep0", "Y"), ("rep1", "Z")):drained_arm(rows, "base", rep, "s1", "X", True)@@ -1059,9 +1026,7 @@ def test_two_repetitions_of_one_pair_are_still_two_parity_observations(tmp_path)def test_an_attempt_that_was_never_re_run_still_carries_its_parity_verdict(tmp_path):- # The other control: a cell that died and was never resumed is the LATEST attempt at itself,- # so its rows stay. Dropping every incomplete cell instead would blind the tool to the runs- # that have the most to say, and would silence the difference below rather than report it.+ # The other control: a cell that died and was never resumed is the latest attempt at itself, so its rows stay.rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "session_id": "s1"}]drained_arm(rows, "base", "rep0", "s1", "STABLE", True)drained_arm(rows, "treatment", "rep0", "s1", "DIFFERENT", False)@@ -1090,33 +1055,24 @@ def two_tier_parity(tmp_path: Path, name: str, fast: tuple[str, str], standard:def test_a_null_control_holding_two_tiers_is_refused_before_the_set_is_derived(tmp_path):- # THE POOLING THIS USED TO DEMONSTRATE IS GONE, and it was fixed underneath rather than here.- # The setup assertion used to be `("r100K", "settings") in unstable`: one differing 100K pair- # from the fast film and one matching 100K pair from the standard film pooled into exactly- # `min_observations`, marking the action unstable on the strength of two different films.- #- # `latest_attempt_rows` now keys an attempt on any attempt-stamped row rather than on the- # terminal cell row alone, and the two films write the SAME cell ids, so they are read as two- # attempts of one cell and the superseded one is dropped before pairing. Only the standard- # film's matching pair survives, so there is one observation and nothing is derived. Asserting- # the old pooling here would be asserting a bug that no longer exists.+ # THE POOLING THIS DEMONSTRATED IS GONE, fixed underneath rather than here.+ # `latest_attempt_rows` now keys on any attempt-stamped row, and the two films write the SAME+ # cell ids, so the superseded one is dropped before pairing and nothing is derived.null = two_tier_parity(tmp_path, "null_mixed", ("X", "Y"), ("Q", "Q"))unstable, _derived, _checks = U.unstable_set([null])assert ("r100K", "settings") not in unstable- # The refusal is kept regardless, as the layer that does not depend on the two films colliding- # on a cell id. Two tiers that walk different ladders need not collide at all, and a set- # derived across films is wrong whether or not de-duplication happened to absorb it.+ # The refusal is kept as the layer that does not depend on the two films colliding on a cell+ # id: a set derived across films is wrong either way.with pytest.raises(SystemExit) as exc:U.main([str(tmp_path / "mine_any"), "--null", str(null.parent)])assert "more than one tier" in str(exc.value)def test_two_mixed_tier_sets_do_not_pass_by_matching_each_other(tmp_path):- # The case the tier-mismatch WARNING cannot see: both sides were re-run at the other tier, so- # both sets are {fast, standard}, they compare EQUAL, and no warning fires. The pooled null- # control then silences `settings` at 100K and the real 100K regression in the payload prints- # as expected variation with the command exiting 0.+ # The case the tier-mismatch WARNING cannot see: both sides re-run at the other tier, so both+ # sets are {fast, standard}, compare EQUAL, and the pooled null control silences a real 100K+ # regression.null = two_tier_parity(tmp_path, "null_both", ("X", "Y"), ("Q", "Q"))mine = two_tier_parity(tmp_path, "mine_both", ("Q", "Q"), ("Q", "REGRESSED"))assert U.tier_of([null]) == U.tier_of([mine]) == {"fast", "standard"}@@ -1127,9 +1083,7 @@ def test_two_mixed_tier_sets_do_not_pass_by_matching_each_other(tmp_path):def test_a_payload_holding_two_tiers_is_refused_even_with_no_null_control(tmp_path):- # The declared unstable set is not derived from anything, so nothing is pooled -- but the- # payload's own 100K pairs still come from two films, and reporting them as repetitions of one- # measurement is the same misreading.+ # A declared set is derived from nothing, but the payload's own 100K pairs still come from two films.mine = two_tier_parity(tmp_path, "mine_alone", ("Q", "Q"), ("Q", "REGRESSED"))with pytest.raises(SystemExit) as exc:U.main([str(mine.parent)])@@ -1137,8 +1091,7 @@ def test_a_payload_holding_two_tiers_is_refused_even_with_no_null_control(tmp_padef test_one_tier_on_each_side_still_scores_in_both_directions(tmp_path):- # The control, so the refusal cannot pass by rejecting every run. A single-tier null control- # and a single-tier payload score exactly as before, and a real regression still exits 1.+ # The control: single-tier null control and payload score as before, and a real regression still exits 1.null = parity_run(tmp_path, "null_one", [("r100K", "rep0", "Q", "Q")])clean = parity_run(tmp_path, "mine_clean", [("r100K", "rep0", "Q", "Q")])regressed = parity_run(tmp_path, "mine_bad", [("r100K", "rep0", "Q", "REGRESSED")])@@ -1147,14 +1100,10 @@ def test_one_tier_on_each_side_still_scores_in_both_directions(tmp_path):def test_a_tier_mismatch_between_two_single_tier_sets_is_refused(tmp_path, capsys):- # This used to warn and then score anyway, which is the worst of the two options: the warning- # said the derived set does not transfer, and then the run was scored against it regardless.- # A set that does not transfer is not a weaker excuse than a real one, it is an arbitrary one.- #- # Exit 2, not 1. The payload below does carry a regression, so the old assertion of exit 1 was- # passing for a reason unrelated to the tier check. Refusing to answer and reporting a parity- # failure have to be distinguishable, or a refusal sends somebody hunting for a UI change that- # was never measured.+ # This used to warn and score anyway, the worst option: a set that does not transfer is+ # arbitrary, not merely weaker.+ # Exit 2, not 1: the payload does carry a regression, so the old exit-1 assertion passed for+ # an unrelated reason, and refusing to answer must be distinguishable from a parity failure.null = write(tmp_path,"null_fast",@@ -1170,10 +1119,8 @@ def test_a_tier_mismatch_between_two_single_tier_sets_is_refused(tmp_path, capsydef test_a_corpus_mismatch_between_two_valid_sides_is_refused(tmp_path, capsys):- # The likelier of the two in practice, and the one that had no check at all: each side is a- # perfectly valid single corpus, so `one_corpus` passes on both, and the null's set was still- # applied to a payload that rendered a different thread. Which actions race is a property of- # the thread the film drove, so that set describes something the payload never displayed.+ # The likelier case with no check at all: each side is a valid single corpus, so `one_corpus`+ # passes on both while the null's set describes a thread the payload never displayed.def at(name, corpus, cells):rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "corpus_hash": corpus}]for rung, rep, base_digest, treat_digest in cells:@@ -1191,8 +1138,7 @@ def test_a_corpus_mismatch_between_two_valid_sides_is_refused(tmp_path, capsys):def test_a_side_recorded_before_corpus_hashes_existed_is_still_scored(tmp_path, capsys):- # An absent hash is not a disagreement. Refusing it would reject every payload recorded before- # the field existed, which is the whole archive.+ # An absent hash is not a disagreement; refusing it would reject the whole archive.rows: list[dict] = [{"row_type": "run_meta", "tier": "standard", "corpus_hash": "v1"}]rows.append(parity_action("r100K.base.rep0", "settings", "Q"))rows.append(parity_action("r100K.treatment.rep0", "settings", "Q"))@@ -1203,8 +1149,8 @@ def test_a_side_recorded_before_corpus_hashes_existed_is_still_scored(tmp_path,def test_main_prints_a_mixed_unstable_set_without_dying(tmp_path, capsys):- # The set now holds bare action names and (rung, action) pairs together, and `sorted()` over- # the two raises TypeError. The one place that formats it is the run header.+ # The set holds bare action names and (rung, action) pairs together, so `sorted()` raises+ # TypeError; the run header is the one place that formats it.null = parity_run(tmp_path, "nullm", [("r1K", "rep0", "X", "Y"), ("r1K", "rep1", "X", "Z")])mine = parity_run(tmp_path, "minem", [("r1K", "rep0", "Q", "Q")])assert U.main([str(mine.parent), "--null", str(null.parent)]) == 0@@ -1274,8 +1220,7 @@ def test_a_probe_named_in_a_later_run_meta_is_still_caught(tmp_path):def test_a_failed_probe_free_gate_is_enough_on_its_own(tmp_path):- # Two independent records of one fact, so a payload from a version that emits only the gate- # is still refused.+ # Two independent records of one fact, so a payload emitting only the gate is still refused.path = probe_payload(tmp_path, "gated", None)with path.open("a", encoding = "utf-8") as fh:fh.write(@@ -1323,8 +1268,7 @@ def test_the_report_path_refuses_a_probed_payload_too(tmp_path):def test_a_null_probe_field_is_the_ordinary_scorable_case(tmp_path):- # Explicit null and absent must both score, or every payload written before the field- # existed becomes unreadable.+ # Explicit null and absent must both score, or every pre-field payload becomes unreadable.pooled, _ = F.load([probe_payload(tmp_path / "explicit", "clean", None)])assert pooled["message_menu.open_close_ms"]pooled, _ = F.load([payload(tmp_path / "absent", "clean", [(1000.0, 900.0)] * 4)])@@ -1376,7 +1320,6 @@ def test_the_composer_click_does_not_set_the_frame_floor(tmp_path):# ── the documented loop runs liveness before it reads a timing ───────-LOOP_DOC = Path(__file__).resolve().parents[2] / "CONTRIBUTING-perf.md"@@ -1459,11 +1402,10 @@ def test_the_documented_loop_asserts_liveness_before_it_reads_any_timing():def test_floor_table_prints_a_clean_verdict_from_a_run_that_liveness_voids(tmp_path, capsys):- # WHY THAT ORDER IS LOAD-BEARING, and the reason the line above is a gate rather than a note.- # The treatment build is 10% faster on the three repetitions it managed, and on the fourth it- # was so slow that `message_menu` never reached its slot. `paired` matches on the metrics BOTH- # arms recorded, so that repetition leaves the table with no trace except a smaller `n`, and- # the survivors print as a clean win over a tight floor.+ # WHY THAT ORDER IS LOAD-BEARING. The treatment is 10% faster on the three repetitions it+ # managed; on the fourth `message_menu` never reached its slot, and `paired` matches only+ # metrics BOTH arms recorded, so that repetition leaves no trace but a smaller `n` and the+ # survivors print as a clean win.mine = liveness_payload(tmp_path, "mine", [(1000.0, 900.0), (1000.0, 900.0), (1000.0, 900.0), (1000.0, None)])@@ -1481,9 +1423,8 @@ def test_floor_table_prints_a_clean_verdict_from_a_run_that_liveness_voids(tmp_pdef test_the_repetition_that_missed_its_slot_is_what_the_verdict_turns_on(tmp_path, capsys):- # The control for the test above. With the fourth repetition's timing present, the same run is- # VOID on the same floor: the clean win is an artefact of the reading that went missing, not a- # property of the change. Every other number in the payload is identical.+ # The control: with the fourth timing present the same run is VOID on the same floor, so the+ # clean win is an artefact of the missing reading. Every other number is identical.mine = liveness_payload(tmp_path, "mine", [(1000.0, 900.0), (1000.0, 900.0), (1000.0, 900.0), (1000.0, 2500.0)])@@ -1499,11 +1440,10 @@ def test_the_repetition_that_missed_its_slot_is_what_the_verdict_turns_on(tmp_paassert cli_main(["--assert-liveness", str(mine)]) == 0+# A null control is base against base, so its four paired ratios are this machine's noise. The+# fourth repetition hiccupped.# ── the null control needs the same liveness gate the treatment run gets ───────--# A null control is base against base, so its four paired ratios are the noise this machine puts-# between two identical builds. The fourth repetition is the one that hiccupped.NULL_WITH_A_NOISY_REPETITION = [(1000.0, 1000.0),(1010.0, 1012.0),@@ -1511,12 +1451,12 @@ NULL_WITH_A_NOISY_REPETITION = [(1000.0, 1300.0),]-# The same null control on a run where that fourth repetition was slow enough to miss the slot, so-# the reading that carried the noise is the reading that never happened.+# The same null control where that fourth repetition missed its slot, so the reading that+# carried the noise never happened.NULL_THAT_MISSED_THE_SLOT = NULL_WITH_A_NOISY_REPETITION[:3] + [(1000.0, None)]-# A treatment payload that is 10% faster, consistently, with a spread well inside its own effect:-# it clears gates 2 and 3 outright, so the only thing standing between it and `faster` is gate 1.+# A consistent 10% with a spread well inside its own effect: it clears gates 2 and 3, so only+# gate 1 stands between it and `faster`.MINE_TEN_PERCENT_FASTER = [(1000.0, 900.0),(1000.0, 905.0),@@ -1552,9 +1492,8 @@ def test_the_documented_loop_asserts_liveness_on_the_null_control_too():def test_the_doc_says_which_commands_in_the_loop_need_a_studio():- # The loop is now four offline commands and three that drive a browser, so "every command- # above except the last" bills `--assert-liveness`, `floor_table` and `ui_parity` for- # credentials none of them read. All three run against a payload file on disk.+ # Four offline commands and three that drive a browser, so 'every command above except the+ # last' bills three payload-file readers for credentials they never use.prose = credentials_prose()for offline in ("--assert-liveness", "floor_table", "ui_parity", "--report"):assert offline in prose, (@@ -1564,27 +1503,24 @@ def test_the_doc_says_which_commands_in_the_loop_need_a_studio():def test_a_missed_slot_in_the_null_control_prints_noise_as_a_result(tmp_path, capsys):- # WHY THE NULL CONTROL IS GATED TOO. The floor is `max(|null delta|, null spread)` over the- # repetitions that survived pairing, and `paired` keys on the metrics BOTH arms recorded, so a- # null repetition that missed its slot leaves no trace either. The repetition a run drops is- # the one that was slow enough to miss a budget, which is the noisiest one it had, so the loss- # is not symmetric: it can only tighten the floor.+ # WHY THE NULL CONTROL IS GATED TOO. The floor is max(|null delta|, null spread) over paired+ # repetitions, and a null repetition that missed its slot leaves no trace either: the dropped+ # one is the noisiest it had, so the loss can only tighten the floor.mine = liveness_payload(tmp_path, "mine", MINE_TEN_PERCENT_FASTER)null = liveness_payload(tmp_path, "null", NULL_THAT_MISSED_THE_SLOT)assert F.main(["--floor", str(null.parent), str(mine.parent)]) == 0table = capsys.readouterr().outassert "0.3 faster" in tableassert "1 metric(s) cleared all three gates." in table- # And the gate the loop used to run only on `outputs/mine` has nothing to say about it: the- # contributor's own payload is whole. The hole is in the run that set the bar.+ # And a gate run only on `outputs/mine` has nothing to say: the contributor's payload is+ # whole, the hole is in the run that set the bar.assert cli_main(["--assert-liveness", str(mine)]) == 0assert cli_main(["--assert-liveness", str(null)]) == 1def test_the_null_repetition_that_kept_its_reading_voids_the_same_result(tmp_path, capsys):- # The control for the test above, and the reading the missed slot removed. The treatment- # payload is byte-identical; only the null control differs, by the one repetition it managed to- # finish. Its floor is 30.1% rather than 0.3%, and the same 10% is noise.+ # The control: the treatment payload is byte-identical, only the null differs by its one+ # finished repetition, and its floor of 30.1% rather than 0.3% makes the same 10% noise.mine = liveness_payload(tmp_path, "mine", MINE_TEN_PERCENT_FASTER)null = liveness_payload(tmp_path, "null", NULL_WITH_A_NOISY_REPETITION)assert F.main(["--floor", str(null.parent), str(mine.parent)]) == 0@@ -1613,7 +1549,7 @@ def test_a_gate_from_an_attempt_that_never_closed_still_refuses_its_cell(tmp_pat# The first attempt completed cleanly and closed itself.rows.append(in_session(parity_action(cid, "settings", "STABLE"), "s1"))rows.append(parity_cell(cid, "treatment", "s1", "rep0", True))- # The resume re-ran the same cell, lost messages, and was killed before its `cell` row.+ # The resume re-ran the cell, lost messages, and died before its `cell` row.rows.append({"row_type": "run_meta", "tier": "standard", "session_id": "s2"})rows.append(in_session(parity_action(cid, "settings", "LOST"), "s2"))rows.append(@@ -1647,8 +1583,7 @@ def gated_then_resumed(tmp_path: Path, name: str) -> Path:{"row_type": "run_meta", "tier": "standard", "session_id": "s1"},{"row_type": "cell", "cell_id": "r100K.base.rep0", "session_id": "s1", "completed": True},timed_action("r100K.base.rep0", "s1", 100.0),- # The treatment arm lost its thread's middle. It renders fewer rows, so its timing is- # CHEAPER than a correct cell's, in the direction that flatters the arm.+ # The treatment arm lost its thread's middle, so it renders fewer rows and times CHEAPER, flattering the arm.{"row_type": "cell","cell_id": "r100K.treatment.rep0",@@ -1678,8 +1613,7 @@ def gated_then_resumed(tmp_path: Path, name: str) -> Path:"completed": False,},]- # The refusal has to hold on the payload BEFORE the resume, or the test below passes on a- # payload that was never quotable for some other reason.+ # The refusal must hold on the payload BEFORE the resume, or the test below passes for another reason.assert F.paired(F.read_rows(before), shard = "b") == {}return write(tmp_path, name, rows)diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.pyindex 212699602a0..5b4c0cecf08 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py@@ -381,8 +381,6 @@ def test_a_timed_out_rebuild_leaves_the_behavioural_run_with_no_verdict(tmp_pathassert "NOTHING WAS COMPARED" in out, outassert "ASSERTION FAILED ON ONE ARM" in out, outassert code == 1, out- # THE TWO MODES AGREE ON THIS PAYLOAD, checked rather than asserted in prose. The structural- # report is the one that already had this precedence rule written down, so it is the reference.assert U.report([shard], "structural on the same payload", frozenset()) == 1@@ -433,13 +431,12 @@ def test_every_named_first_to_break_action_has_an_invariant():assert action in B.INVARIANTS, f"{action} has no behavioural invariant declared"-# ── the four false greens the first review round found ──────────────-#-# Every one of these returned SUCCESS before the fix, which is the only reason they are grouped:-# they are four different routes to a UI verdict of "fine" over a comparison that either found-# nothing or declined to look.+# The four false greens the first review round found. Every one returned SUCCESS before the fix:+# four different routes to a UI verdict of 'fine' over a comparison that either found nothing or+# declined to look.+# ── the four false greens the first review round found ──────────────def test_two_full_mounts_of_different_lengths_is_a_difference_not_an_excuse():"""THE MOST SERIOUS ONE. Neither arm is windowing, so neither is holding anything back onpurpose, and the treatment renders fewer messages than the base -- a user-visible loss of@@ -1404,17 +1401,17 @@ def test_a_capture_that_saw_no_thread_at_all_falls_back_on_the_declaration(tmp_passert all(mode == U.WINDOWED for mode, _why in U.decide_modes([shard]).values())-# ── a cell that failed its own completeness gate carries no UI verdict ──-#-# `probe_thread_completeness` runs before the film and `record_completeness_gate` writes the-# verdict against the cell, so a windowed arm that kept its first page and its last one and lost-# everything between them says so in its own payload. `report/payload.py::excluded_from_rows`-# drops that cell from the PERFORMANCE score. `ui_parity.py` read no gate row except the windowed-# declaration, so the same cell's eighteen action rows were still scored for UI parity -- and the-# visible region is a window on the END of the thread, which such a store still fills, so the-# pairs matched and `--mode auto` exited 0 over a payload that had already recorded the loss.+# A cell that failed its own completeness gate carries no UI verdict. `probe_thread_completeness`+# runs before the film and `record_completeness_gate` writes the verdict against the cell, so a+# windowed arm that kept its first and last page and lost the middle says so in its own payload,+# and `report/payload.py::excluded_from_rows` drops it from the PERFORMANCE score. `ui_parity.py`+# read no gate row except the windowed declaration, so the same cell's eighteen action rows were+# still scored, and the visible region is a window on the END of the thread, which such a store+# still fills, so the pairs matched and `--mode auto` exited 0 over a payload that had already+# recorded the loss.+# ── a cell that failed its own completeness gate carries no UI verdict ──def _completeness_gate(cell_id,passed,@@ -2091,8 +2088,8 @@ def test_the_coverage_floor_sums_the_windowed_and_structural_halves(tmp_path, caassert "TOO LITTLE COMPARED" not in out, out# 1 is the digest regression the fixture carries at the 1K rung, so the floor did not mask it.assert code == 1, out- # And one more than the run can reach refuses it, which is what proves the count above is 2- # rather than the 3 a sum of the three reports would have produced.+ # And one more than the run can reach refuses it, which proves the count above is 2 rather than the+ # 3 a sum of the three reports would have produced.code = U.main([str(tmp_path / "sums"), "--min-compared", "3"])out = capsys.readouterr().outassert "TOO LITTLE COMPARED: 2 of 2" in out, outdiff --git a/tests/studio/studiobench/sweep/ui_parity.py b/tests/studio/studiobench/sweep/ui_parity.pyindex b5ddf493b66..590a0a6478b 100644--- a/tests/studio/studiobench/sweep/ui_parity.py+++ b/tests/studio/studiobench/sweep/ui_parity.py@@ -52,7 +52,6 @@ from pathlib import Pathfrom typing import Any, Optionalif __package__ in (None, ""): # pragma: no cover- # Running the file directly rather than as a module, which is the first thing anyone tries.sys.path.insert(0, str(Path(__file__).resolve().parents[4]))from tests.studio.studiobench.analysis import behaviour as B # noqa: E402@@ -64,57 +63,33 @@ from tests.studio.studiobench.scoring.from_payload import ( # noqa: E402)# The DECLARED unstable set, each entry carrying its mechanism. It lives in the studiobench-# package rather than here so that a test can require a mechanism for every entry, and so that-# this script and the null control cannot drift into two different opinions about which actions-# are trusted.-#-# Declared is not the same as true. `--null` replaces it with the set MEASURED from a base-vs-base-# run and reports every disagreement in both directions, which is the only way an entry here gets-# audited rather than inherited.+# package so a test can require a mechanism per entry and this script cannot drift from the null+# control. Declared is not true: `--null` replaces it with the MEASURED set.UNSTABLE_ACTIONS = frozenset(P.UNSTABLE_ACTIONS)-#: THE RUN'S OWN DECLARATION that an arm mounts a window, as `__main__.py` records it: the gate row-#: `windowed_readiness:{arm}` written when `--windowed-arm` names that side, and `readiness.mode` on-#: every cell row. Matched as literal strings rather than imported from `runtime/readiness.py`-#: because this script reads payloads written by other checkouts of the tool, where what has to be-#: matched is the string in the file and not the constant in this working tree.+#: THE RUN'S OWN DECLARATION that an arm mounts a window, as `__main__.py` records it. Matched as+#: literal strings rather than imported, because this script reads payloads from other checkouts.+# From `runtime/readiness.py`.WINDOWED_GATE = "windowed_readiness:"MODE_WINDOWED = "windowed"-#: THE CELL'S OWN VERDICT THAT IT LOST MESSAGES. `runtime/session.py::record_completeness_gate`-#: writes this row, against the cell, when the traversal to the top of the thread could not-#: establish that the arm still holds the whole conversation -- the head marker never mounted, or-#: the ordinals recorded on the way up have a hole in them. `report/payload.py::excluded_from_rows`-#: reads it as a failed gate and drops the cell from the PERFORMANCE score; this file read no gate-#: rows at all except the windowed declaration, so the same cell's action rows were still scored-#: for UI parity. An arm that keeps the first page and the last one and has lost the middle then-#: retains its first and last visible windows, and its bottom rows, its scroll extent and an-#: action-specific invariant such as `select_text` can all match: `--mode auto` printed a visible-#: pass and a behavioural pass and exited 0 over a payload that already recorded the loss.-#: Matched as a literal string for the same reason `WINDOWED_GATE` is.+#: THE CELL'S OWN VERDICT THAT IT LOST MESSAGES, written by `record_completeness_gate` when the+#: traversal could not establish the arm still holds the whole conversation. `excluded_from_rows`+#: drops such a cell from the PERFORMANCE score, but this file read no gate rows except the+#: windowed declaration, so an arm that lost the middle still printed a pass and exited 0.COMPLETENESS_GATE = "thread_complete"-#: THE CELL'S OWN VERDICT THAT IT STOPPED FOLLOWING THE REPLY IT WAS MEASURING. `runtime/session`-#: writes this row, against the cell, when the thread did not stay pinned for enough of the-#: streaming phase or too little of that phase had the stream attached at all. It belongs beside-#: the completeness gate rather than in a category of its own, because it invalidates the same-#: thing: a reply that scrolled out of the viewport and was unmounted by a windowed arm stops-#: costing anything to render, and the action rows measured around it are a reading of an arm that-#: was not showing the thing under test. `select_text` counts and bottom rows can still match-#: across such a pair, so `--mode auto` printed a behavioural pass and exited 0 over a payload-#: that had already recorded the arm losing the stream. Matched as a literal string for the same-#: reason the two gates above it are.+#: THE CELL'S OWN VERDICT THAT IT STOPPED FOLLOWING THE REPLY IT WAS MEASURING, written when the+#: thread did not stay pinned for enough of the streaming phase. It invalidates the same thing as+#: the completeness gate: a reply scrolled out and unmounted stops costing anything to render.FOLLOW_GATE = "follows_the_stream"#: The per-cell gates whose failure means this cell's action rows are not a reading of the build.-#: A gate that only qualifies one column -- `timer_clamp`, whose failure leaves every column but-#: `busy_pct` standing -- is deliberately NOT here; see `runtime/ab.py::INVALIDATING_CELL_GATES`,-#: which is the same list for the performance side.+#: A gate that only qualifies one column (`timer_clamp`) is deliberately NOT here.INVALIDATING_GATES: tuple[str, ...] = (COMPLETENESS_GATE, FOLLOW_GATE)-#: How each is named and what its failure means, for the refusal a reader actually sees. The gate-#: is described rather than spelled with its row name, because this string is read by somebody-#: deciding whether a run is usable, not by a parser.+#: How each is named and what its failure means, for the refusal a reader actually sees. Described+#: rather than spelled with its row name, because a person reads this deciding if a run is usable._GATE_LABEL: dict[str, str] = {COMPLETENESS_GATE: "completeness",FOLLOW_GATE: "stream-follow",@@ -124,17 +99,15 @@ _GATE_REASON: dict[str, str] = {FOLLOW_GATE: "the arm stopped following the streamed reply",}-#: How one action pair is scored. PER PAIR, never per payload: the readiness gate deliberately-#: permits a payload to hold fully mounted small rungs and windowed large ones, so a payload-wide-#: decision lets one windowed 100K capture suppress the structural digest on every fully mounted 1K-#: pair beside it, and an ordinary DOM regression at those rungs is then never looked for.+#: How one action pair is scored. PER PAIR, never per payload: the readiness gate permits fully+#: mounted small rungs beside windowed large ones, so a payload-wide decision lets one windowed+#: 100K capture suppress the structural digest on every 1K pair.WINDOWED = "windowed"STRUCTURAL = "structural"-#: The arms a pair is expected to have. Consulted BY NAME rather than by walking the rows that are-#: present, because the case that matters is the arm with NO row: an arm that died before it emitted-#: an action row leaves `sides` holding only the other one, and reading the declaration off the rows-#: present asks the surviving arm whether the missing one was windowed.+#: The arms a pair is expected to have, consulted BY NAME rather than by walking the rows present:+#: an arm that died before emitting an action row leaves `sides` holding only the other one, and+#: reading off the rows asks the surviving arm whether the missing one was windowed.ARMS = ("base", "treatment")@@ -312,19 +285,14 @@ def incomplete_cells(paths: list[Path]) -> dict[str, str]:if keep is not None and r.get("session_id") not in (None, keep):continuedetail = r.get("detail") if isinstance(r.get("detail"), dict) else {}- # NOT MEASURED IS NOT FAILED, and this job draws the line in exactly the same place- # the performance side does: `gate_detail_is_unmeasured` is THE definition, imported- # rather than restated. The predicate had been copied here, which is the drift- # `INVALIDATING_CELL_GATES` was centralised to prevent, arriving one level down.- #- # It matters most for the stream-coverage clause: that shortfall is a property of the- # scene schedule, identical on both arms, and this job compares DOM digests the- # shortfall does not touch at all. It refused 32 of 32 pairs here, the null control- # included.+ # NOT MEASURED IS NOT FAILED, drawn where the performance side draws it:+ # `gate_detail_is_unmeasured` is THE definition, imported rather than restated. It matters most+ # for the stream-coverage clause, identical on both arms and untouched by a DOM digest, which+ # refused 32 of 32 pairs here including the null control.if gate_detail_is_unmeasured(detail):continue- # FIRST FAILURE WINS, so a cell that failed both is not relabelled by whichever row- # happens to come second in the file.+ # FIRST FAILURE WINS, so a cell that failed both is not relabelled by whichever row comes second+ # in the file.if cid in out:continueout[cid] = (@@ -397,59 +365,35 @@ def collect("""out: dict[tuple, dict] = collections.defaultdict(dict)attempted = missing = 0- # TWO DIFFERENT "INCOMPLETE"S MEET HERE, and they are kept apart on purpose. `gate_failures`- # is per cell and comes from a gate the payload FAILED, and those rows are stamped and kept.- # `dropped_incomplete` counts action rows discarded because their cell never wrote a `cell`- # row at all, which only happens under `require_complete`. Sharing one name between them- # would make the first silently overwrite the second on the first shard scanned.+ # TWO DIFFERENT "INCOMPLETE"S MEET HERE: `gate_failures` is per cell from a gate the payload+ # FAILED and those rows are kept, while `dropped_incomplete` counts action rows discarded because+ # their cell never wrote a `cell` row. One shared name would let the first overwrite the second.+ # Action rows come from `latest_attempt_rows`.+ # Which only happens under `require_complete`.dropped_incomplete = 0no_cell_rows = 0for path in paths:shard = path.parent.namegate_failures = incomplete_cells([path])raw_rows = rows(path)- # A CELL THAT DID NOT FINISH IS NOT AN OBSERVATION -- ON THE NULL CONTROL ONLY, and the- # asymmetry is the point rather than an omission.- #- # Action rows are emitted as the film runs; the `cell` row is written when it ends. So an- # interrupted or in-flight cell leaves a complete-looking set of action rows that nothing- # owns, and they pair and compare exactly like real ones.- #- # ON THE NULL they must be dropped. The null's job is to say which actions are unstable,- # and an under-observed action reads as stable or undetermined instead -- which NARROWS- # the excuse set, so the result's ordinary noise then has nothing to hide behind. Scoring- # this sweep's own half-written film as a null control turned 0 false alarms into 5 of 30- # for exactly that reason. `floor_table.cell_metrics` refuses the same rows on the timing- # side.- #- # ON THE RESULT they must be kept, and `test_an_attempt_that_was_never_re_run_still_- # carries_its_parity_verdict` holds that: a cell that died is the latest attempt at- # itself, and a difference it observed is still a difference. Dropping it would silence a- # regression, which is the worse direction of the two. So the conservative choice has- # opposite signs on the two sides, because "conservative" means a different thing on each.- #- # READ THROUGH THE SAME FILTER THE ACTION ROWS ARE READ THROUGH. Taking `completed` from- # the raw stream while taking the action rows from `latest_attempt_rows` is the guard- # answering about a different attempt than the one it is guarding. `_resume_set` names the- # path that gets there without anybody doing anything unusual: an A/B pair is re-run WHOLE- # (`ab.skippable_cells`), so a resume re-runs an arm that had already succeeded, and if- # that retry is interrupted the payload holds a completed row and a LATER, unfinished row- # under the same deterministic `cell_id`. Read raw, the dead retry's action rows are- # admitted on the strength of the completion the run before them earned. On the null that- # is the worst direction: one valid repetition plus this one reaches `min_observations`,- # the action is called unstable, and that excuse hides a real result difference.- # Per file: the payload is append-only within one shard, and a cell id is reused across- # shards, so superseding has to be resolved inside the stream that appended it.+ # A CELL THAT DID NOT FINISH IS NOT AN OBSERVATION -- ON THE NULL CONTROL ONLY. Action rows are+ # emitted as the film runs and the `cell` row when it ends, so an interrupted cell leaves a+ # complete-looking set nothing owns. ON THE NULL they must be dropped: an under-observed action+ # reads as stable and NARROWS the excuse set (0 false alarms became 5 of 30). ON THE RESULT they+ # must be kept, because a difference a dead cell observed is still a difference. Read through the+ # SAME filter as the action rows, or the guard answers about a different attempt. Per file, since+ # a cell id is reused across shards, so superseding has to be resolved inside the stream that+ # appended it. The keep-on-the-result direction is held by+ # test_an_attempt_that_was_never_re_run_still_carries_its_parity_verdict.kept_rows = latest_attempt_rows(raw_rows)completed = {r.get("cell_id")for r in kept_rowsif r.get("row_type") == "cell" and r.get("completed")}- # A payload with no `cell` rows at all predates them, or is a fixture. Falling back is- # right; falling back SILENTLY is how a guard stops guarding, so it is counted and said.- # Asked of the raw stream on purpose: the question is whether this recorder ever wrote- # cell rows, not whether the surviving attempt happened to reach one.+ # A payload with no `cell` rows predates them, or is a fixture. Falling back is right; falling+ # back SILENTLY is how a guard stops guarding, so it is counted and said. Asked of the raw+ # stream: the question is whether this recorder ever wrote cell rows.has_cell_rows = any(r.get("row_type") == "cell" for r in raw_rows)if not has_cell_rows:no_cell_rows += 1@@ -463,8 +407,8 @@ def collect(# BEFORE the tally, so a selected report's counts are counts of the pairs it scored.if select is not None and key not in select:continue- # And before the tally for the same reason, but after the selection: a row this- # report never asked for is not an incomplete observation of it.+ # And before the tally for the same reason, but after the selection: a row this report never+ # asked for is not an incomplete observation of it.if require_complete and has_cell_rows and r.get("cell_id") not in completed:dropped_incomplete += 1continue@@ -473,9 +417,8 @@ def collect(attempted += 1else:missing += 1- # STAMPED, NOT DROPPED, for the same reason a failed capture is kept: the comparison- # layer has to be able to say that this pair carries no verdict, and a row deleted here- # would leave the pair looking like an action that simply never ran.+ # STAMPED, NOT DROPPED: the comparison layer has to be able to say this pair carries no verdict,+ # and a deleted row would leave the pair looking like an action that never ran.if cid in gate_failures:r = dict(r)r["_incomplete"] = gate_failures[cid]@@ -600,17 +543,11 @@ def decide_modes(paths: list[Path]) -> dict[tuple, tuple[str, str]]:if len(sides) == 2 and all(_mount_measured(r.get("parity")) for r in sides.values()):decided[key] = (STRUCTURAL, "")continue- # EITHER EXPECTED ARM, INCLUDING ONE THAT HAS NO ROW AT ALL.- #- # The declaration fallback used to be read off the rows this pair actually has, which- # covers a declared-windowed arm whose captures failed and misses the case one step worse:- # the arm failed before it emitted an action row, so `sides` holds only the other side, the- # loop never asks about the arm that is missing, and the pair is scored structurally. In a- # mixed-rung payload the fully mounted pairs then supply `matched > 0`, the missing windowed- # pair is filed as structurally NOT COMPARABLE, and the command exits 0 without ever running- # a windowed report for the rung that has no verdict at all.- #- # AND ONLY THIS PAIR'S OWN SHARD DECLARES IT. See `declared_windowed`.+ # EITHER EXPECTED ARM, INCLUDING ONE WITH NO ROW AT ALL. Reading the declaration off the rows a+ # pair has misses the case one step worse: the arm failed before emitting an action row, so the+ # loop never asks about it, the pair is scored structurally, and the command exits 0 with no+ # verdict for that rung. And only this pair's own shard declares it.+ # See `declared_windowed`.why = ""shard = key[0]for label in ARMS:@@ -763,10 +700,10 @@ def behaviour_report()continueout = B.compare_behaviour(sides["base"], sides["treatment"])- # THE SAME TWO QUESTIONS THE STRUCTURAL PATH ASKS, off the same functions; see- # `compare_all_with`. `compare_behaviour` cannot see either shape: an action not performed- # on one arm records no quantities to disagree about, and one that ran and failed its own- # assertion records perfectly ordinary ones.+ # THE SAME TWO QUESTIONS THE STRUCTURAL PATH ASKS, off the same functions: `compare_behaviour`+ # cannot see either shape, since an action not performed on one arm records no quantities to+ # disagree about and one that failed its own assertion records ordinary ones.+ # `compare_all_with` is the other half.idle_ = P.execution_verdict(sides["base"], sides["treatment"])out["one_sided"] = (idle_ or {}).get("one_sided") or ""out["idle_reason"] = (idle_ or {}).get("idle_reason") or ""@@ -779,14 +716,11 @@ def behaviour_report(print(f"\n{label} (BEHAVIOURAL MODE)")print(f" CLAIM: {P.CLAIM_BEHAVIOURAL}.")# Through the helper, not straight off the table: this mode's line names the coverage band it- # enforces, and the band lives in `behaviour`. Printing the raw template would emit the- # placeholders themselves and quietly stop reporting the numbers being applied.+ # enforces, and printing the raw template would emit the placeholders instead of the numbers.print(f" POLICY: {P.behaviour_policy(B.MIN_CLIPBOARD_COVERAGE, B.MAX_CLIPBOARD_COVERAGE)}.")- # WHICH REASON, and only if it is true. Forced behavioural mode on a payload that mounts its- # whole thread -- which is exactly how a NULL CONTROL is scored on the same scale as the- # windowed arm it is the control for -- used to print "one arm of this payload mounts a window- # of the thread" about a payload where neither arm does. A control that misdescribes itself in- # its own heading is not a small thing when the heading is what gets read.+ # WHICH REASON, and only if it is true: forced behavioural mode on a fully mounted payload --+ # which is how a NULL CONTROL is scored -- used to print "one arm mounts a window of the thread"+ # about a payload where neither does.if windowed:print(" One arm of this payload mounts a window of the thread, so the structural DOM digest"@@ -858,36 +792,23 @@ def behaviour_report(names = sorted({a for a, _s, _c, _v in idle})print(f"\n NOT EXERCISED: {', '.join(names)}")- # `not broken` IS THE PRECONDITION THE PARAGRAPH BELOW ARGUES FROM, and it used to be assumed- # rather than checked. An all-BROKEN payload reaches `matched == 0` too -- the two counters are- # independent -- so a run that had just listed its broken invariants went on to say that not- # one invariant was evaluated and that it carried no failure, immediately before returning 1.- # The artifact contradicted its own results and sent the reader off to find out why actions- # that had run did not run. `report` (`elif decided == 0`) and `visible_report` (`and not- # differing`) both already gate this banner on their own failure bucket; this is that gate.- #- # NOT GATED ON `build_failed`, which is the other half of the return condition and is NOT the- # same case: those two shapes are collected from pairs whose invariants were unreadable, so- # "not one behavioural invariant was evaluated" stays literally true when one fires, and the- # run is a failure for a directional reason rather than an invariant one. See- # `test_a_timed_out_rebuild_leaves_the_behavioural_run_with_no_verdict`, which asserts exactly- # that pairing -- NOTHING WAS COMPARED together with exit 1 -- on purpose.+ # `not broken` IS THE PRECONDITION THE PARAGRAPH BELOW ARGUES FROM: an all-BROKEN payload reaches+ # `matched == 0` too, so a run that had just listed its broken invariants went on to say none was+ # evaluated and it carried no failure, immediately before returning 1. NOT gated on+ # `build_failed`, which is collected from pairs whose invariants were unreadable.+ # test_a_timed_out_rebuild_leaves_the_behavioural_run_with_no_verdict asserts that pairing.if matched == 0 and not broken:- # NOTHING WAS VALIDATED, which is not the same as nothing being wrong. With every pair- # unchecked, not comparable or never exercised, `broken` is empty and the block above has- # just printed "Every declared behavioural invariant held on both arms" -- a sentence that- # is technically true of an empty set and reads as a pass. This is the same false green- # the digest path returns 2 for, and it is the more dangerous of the two here, because- # behavioural mode is what REPLACES the digest on a windowed arm: if it silently validates- # nothing then a windowed arm has no UI verdict at all while appearing to have passed one.+ # NOTHING WAS VALIDATED is not the same as nothing being wrong: with every pair unchecked,+ # `broken` is empty and the block above has just printed a sentence that is true of an empty set+ # and reads as a pass. More dangerous here, because behavioural mode REPLACES the digest.print("\n NOTHING WAS COMPARED. Not one behavioural invariant was evaluated on any pair, ""so this run carries no UI verdict -- neither a pass nor a failure. Treat it as an ""absent result and find out why the actions did not run.")- # AFTER the diagnosis prints and BEFORE the "could not tell" code, the ordering `report` states:- # a run can find a real regression while deciding nothing (the two shapes `build_differences`- # collects survive a pair whose invariants were unreadable), and "it failed" is more specific.+ # AFTER the diagnosis prints and BEFORE the "could not tell" code: a run can find a real+ # regression while deciding nothing, and "it failed" is more specific.+ # The two shapes are what `build_differences` returns.if broken or build_failed:return Outcome(1, frozenset(compared), _keys(results))if matched == 0:@@ -932,23 +853,17 @@ def visible_unstable_set(null_paths: list[Path] | None) -> frozenset[tuple[str,"""if not null_paths:return frozenset()- # AND ONLY FROM CELLS THAT FINISHED, which is the same admission rule `unstable_set` applies- # to the structural floor and `audit_null` to the audit. `collect` states the asymmetry: action- # rows are written as the film runs and the `cell` row when it ends, so a null-control cell- # that died mid-film leaves a complete-looking set of captures that nothing owns, and they pair- # and compare like real ones. Read raw here, one DIFFERING unfinished observation plus one- # matching completed one is exactly `min_observations`, `derive_unstable` calls that (rung,- # action) unstable, and `visible_report` then files a real visible difference at the same key- # under "differ against an identical build" and exits 0. The floor is the one place an- # under-observed action must read as undetermined rather than as an excuse.+ # AND ONLY FROM CELLS THAT FINISHED, the same admission rule `unstable_set` and `audit_null`+ # apply. A null-control cell that died mid-film leaves captures nothing owns, and read raw one+ # DIFFERING unfinished observation plus one matching completed one is exactly `min_observations`,+ # so a real visible difference is filed under "differ against an identical build" and exits 0.results, _got = compare_all_with(null_paths, P.compare_visible, "visible", require_complete = True)by_rung: dict[str, list[tuple[str, dict]]] = collections.defaultdict(list)for action, _shard, cell, r in results:- # A pair whose action never ran on both arms is not an observation of anything, in either- # direction. `derive_unstable` refuses to count a verdict it cannot read; this refuses to- # hand it one it should not read.+ # A pair whose action never ran on both arms is not an observation in either direction:+ # `derive_unstable` refuses to count a verdict it cannot read, and this refuses to hand it one.if r.get("_ran"):by_rung[rung_of_cell(cell)].append((action, r))out: set[tuple[str, str]] = set()@@ -994,12 +909,10 @@ def visible_report(found = build_differences(results, min_reps)compared: set[tuple[str, str, str]] = set()differing, unstable_bad, blind, idle, matched = [], [], [], [], 0- # THE RESIDUE, PRINTED. A message that was on screen during the action and had been unmounted- # again by the time the capture ran cannot be digested, and `compare_visible` refuses the pair- # for it. That refusal used to be invisible: the pair returned MATCH with the ordinals tucked- # into `not_digested` and nothing here read the key, so a rendering difference in the missing- # message left no trace anywhere in the output. It is collected across every verdict, because a- # DIFFER pair with a residue is also a pair whose report is incomplete.+ # THE RESIDUE, PRINTED. A message on screen during the action but unmounted by capture time+ # cannot be digested and `compare_visible` refuses the pair for it, which used to be invisible:+ # the pair returned MATCH with the ordinals in `not_digested` and nothing read the key. Collected+ # across every verdict, because a DIFFER pair with a residue is also incomplete.residue = []for action, shard, cell, r in results:if r.get("not_digested"):@@ -1009,13 +922,9 @@ def visible_report(elif r["verdict"] == P.NOT_COMPARABLE:blind.append((action, shard, cell, r))elif r["verdict"] == P.DIFFER:- # A SEVERE difference is never routed into the noise floor. See compare_visible: an- # action can be in the derived unstable set for an unrelated attribute and still be- # the action on which one arm lost the whole thread.- #- # AT THIS PAIR'S OWN RUNG. An action that differs against an identical build at 100K- # says nothing about the same action at 1K, where the thread is a fraction of the size- # and the film's slots land somewhere else entirely.+ # A SEVERE difference is never routed into the noise floor: an action can be in the derived+ # unstable set for an unrelated attribute and still be the action on which one arm lost the whole+ # thread. AT THIS PAIR'S OWN RUNG, since an action differing at 100K says nothing about 1K.noise = (rung_of_cell(cell), action) in unstable and not r.get("severe")(unstable_bad if noise else differing).append((action, shard, cell, r))compared.add((action, shard, cell))@@ -1079,8 +988,7 @@ def visible_report()else:# WHICH RUNG EACH FLOOR ENTRY WAS MEASURED AT, printed, because it is also the only rung it- # silences anything at. A floor that reads as a list of action names would look like it- # covers the whole payload.+ # silences anything at; a bare list of action names would look like it covers the payload.keys = sorted(unstable)shown = ", ".join(f"{rung or '?'} {action}" for rung, action in keys[:12])print(@@ -1092,13 +1000,13 @@ def visible_report(build_failed = print_build_differences(found, min_reps)if matched == 0 and not differing:- # The same false green every other mode here has already been fixed for: nothing compared- # is not the same as nothing wrong, and it must not exit 0.+ # The same false green every other mode here has been fixed for: nothing compared is not nothing+ # wrong, and it must not exit 0.print("\n NOTHING WAS COMPARED. Not one action pair yielded a visible-region verdict, so\n"" this run carries no UI verdict at all -- neither a pass nor a failure.")- # See the same three lines in `behaviour_report` and in `report`: the diagnosis prints, then a+ # See the same three lines in `behaviour_report` and `report`: the diagnosis prints, then a# failure outranks a refusal.if differing or build_failed:return Outcome(1, frozenset(compared), _keys(results))@@ -1167,8 +1075,8 @@ def compare_all_with(out = compare(sides["base"].get(key), sides["treatment"].get(key))out["_ran"] = ranidle = P.execution_verdict(sides["base"], sides["treatment"])- # WHICH ARM, not merely that one of them was idle. `execution_verdict` returns `one_sided`- # empty for a missed slot and for the symmetric case, and the arm that DID run otherwise.+ # WHICH ARM, not merely that one was idle: `execution_verdict` returns `one_sided` empty for a+ # missed slot and for the symmetric case, and the arm that DID run otherwise.out["one_sided"] = (idle or {}).get("one_sided") or ""out["idle_reason"] = (idle or {}).get("idle_reason") or ""out["idle_detail"] = (idle or {}).get("reason") or ""@@ -1195,12 +1103,9 @@ def compare_all(for (shard, rung, rep, sid, action), sides in sorted(got["pairs"].items()):cell = f"{rung} {rep}"if "base" not in sides or "treatment" not in sides:- # One arm never produced this row at all. Recorded rather than skipped: an action that- # ran on one arm and not the other is itself a difference between the arms.- #- # A resumed run lands here too, with the two arms under two session ids, and the- # session is named so the reader can tell that case from a surface one build never- # rendered. Both carry NOT_COMPARABLE, which is the honest verdict for either.+ # One arm never produced this row. Recorded rather than skipped: an action that ran on one arm+ # and not the other is itself a difference. A resumed run lands here too with two session ids, so+ # the session is named; both carry NOT_COMPARABLE, which is honest for either.results.append((action,@@ -1383,16 +1288,15 @@ def audit_null(by_rung[rung_of_cell(cell)].append((action, r))decided, undecided, differed, excused, out_of_scope = [], [], [], [], []- # Decided entries whose every observation came from a settled-match refusal. Kept apart from+ # Decided entries whose every observation came from a settled-match refusal, kept apart from# `decided` because the live tail was never read at all.on_settled = []for rung, pairs in sorted(by_rung.items()):for action, row in sorted(P.derive_unstable(pairs).items()):entry = (rung, action)- # OUT OF SCOPE IS NOT AN EXCUSE, and conflating the two is what made the empty scope- # read as a vacuous audit below. An excused action is one the fixture cannot perform- # and it is a HOLE in a question that was asked. An out-of-scope action is a question- # nobody asked, because no excuse could have changed the result's verdict for it.+ # OUT OF SCOPE IS NOT AN EXCUSE: an excused action is a HOLE in a question that was asked, while+ # an out-of-scope one is a question nobody asked, and conflating them made the empty scope read+ # as a vacuous audit.if scope is not None and entry not in scope:out_of_scope.append(entry)continue@@ -1405,25 +1309,13 @@ def audit_null(if row["unstable"]:differed.append(entry)- # AN ACTION THE NULL NEVER MEASURED AT ALL IS UNDECIDED, not absent. The loop above can only- # classify what `derive_unstable` produced, so a scoped (rung, action) with NO rows in the- # null payload fell into neither list and the audit never asked about it. One other scoped- # action being decided was then enough to return 0.- #- # That is the whole question this audit exists to put, answered by default. `unstable_set`- # unions the DECLARED names back in, so an action like `send_turn` that the null never- # observed is still excused by name, and a corroborated result difference on it passes.- # Reproduced end to end: scope {send_turn, settings}, null carrying no send_turn rows,- # audit 0, verdict 0, the regression printed under "expected to vary".- #- # Rows vanish entirely more easily than they look like they should: the null is collected with- # `require_complete = True`, so a cell that never finished takes every one of its action rows- # with it, and the action stops existing rather than becoming undetermined.- #- # Counted as MISSING as well as undecided, because the two are not the same reading. Measured- # and inconclusive means the null tried; never measured means it did not, and a reader chasing- # a failed audit needs to know which. `allow_undecided` still applies -- `image_upload` has no- # attachments button on this fixture and is waived by name whether it produced rows or not.+ # AN ACTION THE NULL NEVER MEASURED AT ALL IS UNDECIDED, not absent: the loop can only classify+ # what `derive_unstable` produced, so a scoped (rung, action) with no rows fell into neither list+ # and one other decided action was enough to return 0 -- with `unstable_set` unioning the DECLARED+ # names back in, a corroborated result difference on it then passed. Reproduced end to end: scope+ # {send_turn, settings}, no send_turn rows, audit 0, verdict 0, regression printed under+ # "expected to vary". Counted as MISSING as well as undecided, because measured-and-inconclusive+ # is a different reading from never measured.missing = []if scope is not None:seen = set(decided) | set(undecided) | set(excused)@@ -1440,16 +1332,14 @@ def audit_null("missing": missing,"decided_on_settled_thread": on_settled,}- # NOTHING TO DECIDE IS A PASS, and only when a scope said so. A result whose every action- # matched asks the null for no excuses at all, and there is no way to fail a question that- # was never put. This is not the vacuous case guarded below: the scope was computed from the- # result, so an empty one is a statement about the result rather than about the null.+ # NOTHING TO DECIDE IS A PASS, and only when a scope said so: a result whose every action matched+ # asks the null for no excuses, and there is no way to fail a question never put. The scope is+ # computed from the result, so an empty one is a statement about the result.if scope is not None and not scope:report_["reason"] = "the result needs no excuse from this null control"return 0, report_- # Everything excused is not a decided null control, it is a null control that measured- # nothing while naming a reason for each blank. Passing it would let the excuse list grow- # until the audit is vacuous.+ # Everything excused is not a decided null control but one that measured nothing while naming a+ # reason for each blank; passing it would let the excuse list grow until the audit is vacuous.if not decided:report_["reason"] = "no (rung, action) reached min_observations"return 1, report_@@ -1488,8 +1378,8 @@ def print_null_audit(rc: int, report_: dict, allow_undecided: frozenset) -> Nonef"({', '.join(sorted(allow_undecided))}) -- each one a hole")if report_.get("out_of_scope"):- # Counted apart from the excused, because it is not a hole: no excuse could have moved- # the result's verdict on these, so the null was never asked about them.+ # Counted apart from the excused, because it is not a hole: no excuse could have moved the+ # result's verdict on these.print(f" not audited, out of scope: {len(report_['out_of_scope'])} ""(the result's verdict for these does not turn on an excuse)"@@ -1514,9 +1404,8 @@ def print_null_audit(rc: int, report_: dict, allow_undecided: frozenset) -> Noneprint("\n These (rung, action) pairs never reached min_observations:")_missing = set(report_.get("missing") or ())for rung, action in report_["undecided"][:12]:- # Said apart from the rest: measured and inconclusive means the null tried and- # could not decide; NO ROWS means it never measured the action at all, which is a- # different thing to go and fix and used to be invisible here.+ # Said apart from the rest: measured and inconclusive means the null tried and could not decide,+ # while NO ROWS means it never measured the action at all.tail = " -- NO ROWS in the null at all" if (rung, action) in _missing else ""print(f" {action}@{rung}{tail}")print(@@ -1561,15 +1450,14 @@ def unstable_set(paths: list[Path] | None) -> tuple[frozenset, dict, dict]:derived[f"{action}@{rung}"] = rowif row["unstable"]:measured.add((rung, action))- # The cross-check stays pooled ON PURPOSE. It audits the DECLARED list, whose entries claim to- # hold at every rung, so the question it answers -- did this run ever see this action differ --- # is an action-level question. It is advisory output and carries no verdict.+ # The cross-check stays pooled ON PURPOSE: it audits the DECLARED list, whose entries claim to+ # hold at every rung, so the question is action-level. Advisory output, carrying no verdict.checks = P.cross_check(P.derive_unstable([(a, r) for a, _s, _c, r in results]), UNSTABLE_ACTIONS)- # UNION, not replacement. An action the null control could not reach -- `image_upload` has no- # visible attachments button on this fixture -- would otherwise silently move from "declared- # unstable" to "stable" on the strength of a measurement that never happened.+ # UNION, not replacement: an action the null could not reach (`image_upload` has no attachments+ # button on this fixture) would otherwise move from "declared unstable" to "stable" on a+ # measurement that never happened.return frozenset(measured) | frozenset(UNSTABLE_ACTIONS), derived, checks@@ -1599,7 +1487,6 @@ def in_arm_repeatability(paths: list[Path]) -> tuple[set, set]:not what the measurement is picking up."""got = collect(paths, require_complete = True)- # (shard, rung, session, action) -> {rep: side A row}side_a: dict[tuple, dict] = collections.defaultdict(dict)for (shard, rung, rep, sid, action), sides in got["pairs"].items():row = sides.get("base")@@ -1690,14 +1577,11 @@ def corroborated(entries: list[tuple], min_reps: int) -> tuple[list[tuple], listby_action[(entry[0], rung_of_cell(entry[2]), direction)].append(entry)firm, weak = [], []for group in by_action.values():- # DISTINCT repetitions, not rows: one repetition seen twice is one observation.- # DELIBERATELY NOT KEYED ON THE SHARD, and the shard is available in `e[1]`. Two shards- # carrying the same (rung, rep) cannot be told apart from ONE film recorded twice -- the- # duplicate-session case, where both copies are the same observation and counting them as- # two lets a single flake corroborate itself. Keying on the cell alone can only- # under-count, which costs a finding; keying on the shard can manufacture corroboration- # out of duplicated provenance, which is the failure this whole instrument is about.- # `test_one_repetition_seen_twice_is_not_two_observations` holds this direction.+ # DISTINCT repetitions, not rows: one repetition seen twice is one observation. DELIBERATELY NOT+ # KEYED ON THE SHARD -- two shards carrying the same (rung, rep) cannot be told from one film+ # recorded twice, where counting both lets a single flake corroborate itself. Keying on the cell+ # alone can only under-count; keying on the shard manufactures corroboration.+ # test_one_repetition_seen_twice_is_not_two_observations holds this direction.reps = {e[2] for e in group}(firm if len(reps) >= min_reps else weak).extend(group)return firm, weak@@ -1722,8 +1606,8 @@ def report("""results, got = compare_all(paths, select)if not results:- # An empty result is reported as an empty result. "No mismatches found" when nothing was- # ever compared is the exact shape of a check that silently does nothing.+ # An empty result is reported as an empty result: "No mismatches found" when nothing was compared+ # is the exact shape of a check that silently does nothing.print(f"\n{label}: NO PARITY DATA in {len(paths)} payload(s). "f"{got['missing']} action rows carried no digest. "@@ -1738,69 +1622,43 @@ def report(compared: set[tuple[str, str, str]] = set()matched = 0for action, shard, cell, r in results:- # COLLECTED BEFORE THE VERDICT BRANCHES AND OUTSIDE THE EXEMPTION, because it is not a- # digest. An action can run on both arms, produce two digests the instability exemption- # then excuses, and still have failed its own assertion on one arm only -- which is the- # two builds behaving differently and the one shape `ran` cannot see. `stop_generation`- # is the live example: `ran = True, expect_ok = stopped_ms is not None`, and it is on the- # declared unstable list, so a head that no longer stops generation was excused twice- # over. The exemption was measured for digest stability; applying it here would be- # applying it to a different quantity that happens to share an action name.- # The fifth element is the DIRECTION, and `corroborated` keys on it. Which arm's- # assertion failed is part of what is being claimed: a treatment failure in one- # repetition and a base failure in the next is a race that landed on either side, not- # one finding seen twice.+ # COLLECTED BEFORE THE VERDICT BRANCHES AND OUTSIDE THE EXEMPTION, because it is not a digest: an+ # action can produce two digests the instability exemption excuses and still have failed its own+ # assertion on one arm only, which is the one shape `ran` cannot see. `stop_generation` is the+ # live example: `ran = True, expect_ok = stopped_ms is not None`, and it is on the declared+ # unstable list, so a head that no longer stops generation was excused twice over.+ # The fifth element is the DIRECTION, and `corroborated` keys on it: a treatment+ # failure in one repetition and a base failure in the next is a race, not one finding twice.if r.get("expect_regressed"):expect_bad.append((action, shard, cell, [r.get("expect_reason", "")], r["expect_regressed"]))if r["verdict"] == P.NOT_APPLICABLE:- # NEITHER A PASS NOR A FAIL. The digest is the wrong question for this pair; see- # analysis/parity.applicability. It is bucketed separately so it cannot be summed into- # either column, and `--mode behaviour` is what answers it instead.- #- # BELOW the assertion collection above, and that ordering is the merge's one real- # decision. This branch `continue`s, so putting it first would drop a failed assertion- # whenever the pair also happened to be a windowed mount -- exactly what the comment- # above means by collecting outside the verdict branches. A windowed arm that stops- # stopping generation is the case that would have gone silent.+ # NEITHER A PASS NOR A FAIL: the digest is the wrong question for this pair, so it is bucketed+ # separately and `--mode behaviour` answers it instead. BELOW the assertion collection above,+ # and that ordering is the merge's one real decision: this branch `continue`s, so putting it+ # first would drop a failed assertion whenever the pair was also a windowed mount.inapplicable.append((action, shard, cell, [r.get("reason", "")]))continueif r["verdict"] == P.NOT_EXERCISED:# Same, for which arm stayed live: `one_sided` names the arm that DID run.entry = (action, shard, cell, [r.get("reason", "")], r.get("one_sided") or None)if r.get("one_sided"):- # ONE ARM RAN IT AND THE OTHER COULD NOT, which is not the missed-slot case.- # A missed slot is a runner losing a race and it costs coverage; an action that- # RUNS on one build and cannot be performed on the other is the two builds- # behaving differently, and it is the one regression shape that leaves no digest- # to differ. A control that stopped opening produces exactly this and nothing- # else, so filing it under coverage is how "the button no longer works" ships- # green. Held to the SAME corroboration bar as a differing digest below rather- # than failed on sight, because a contended runner can lose one arm's slot once.- #- # EXEMPTED BY `RACY_EXECUTION`, NOT BY THE DIGEST SET, and the distinction is the- # same one `expect_regressed` rests on. Every UNSTABLE_ACTIONS mechanism describes- # what makes the CAPTURE move; none of them says the action can fail to happen.- # Keyed on that list, nine of sixteen actions were permanently exempt from the one- # regression shape that leaves no digest to differ, so a broken composer taking- # `keystroke` down on the treatment arm in both repetitions exited 0. `scroll_after`- # was exempt without even having a `not_run` path to reach.- # ON THE RECORDED REASON, not the action name. Each of the three has non-racy- # not_run paths too -- send_turn's "no composer on the page", stop_generation's- # "the stop button is not present" -- and a treatment build that REMOVES either- # control records exactly the regression this exists to catch.+ # ONE ARM RAN IT AND THE OTHER COULD NOT, which is not the missed-slot case: a missed slot costs+ # coverage, while an action that runs on one build and cannot be performed on the other is the+ # one regression shape that leaves no digest to differ. Held to the SAME corroboration bar as a+ # differing digest. EXEMPTED BY `RACY_EXECUTION`, NOT BY THE DIGEST SET: every UNSTABLE_ACTIONS+ # mechanism describes what makes the CAPTURE move, so keyed on that list nine of sixteen actions+ # were permanently exempt. ON THE RECORDED REASON, not the action name._racy = P.racy_execution(action, r.get("idle_reason") or "")(one_sided_unstable if _racy else one_sided).append(entry)else:idle.append(entry)continue- # THE STYLE VERDICT IS COLLECTED BEFORE THE STRUCTURAL REFUSAL IS BUCKETED, because it is- # an INDEPENDENT reading and the refusal is not about it. The bounded computed-style probe- # is the only thing here that sees `display`, `visibility` or `pointer-events`, and a pair- # refused for landing at two points in one stream can still carry a real CSS regression on- # a settled surface. Sitting below the `continue`, that verdict was computed, attached to- # the row, and then dropped on the floor for exactly the pairs this PR creates most of.+ # THE STYLE VERDICT IS COLLECTED BEFORE THE STRUCTURAL REFUSAL IS BUCKETED, because it is an+ # INDEPENDENT reading: the bounded computed-style probe is the only thing here that sees+ # `display`, `visibility` or `pointer-events`, and a pair refused for landing at two points in+ # one stream can still carry a real CSS regression.if r.get("style_verdict") == P.DIFFER:style_bad.append((action, shard, cell, [r.get("style_reason", "")]))if r["verdict"] == P.NOT_COMPARABLE:@@ -1814,9 +1672,9 @@ def report(compared.add((action, shard, cell))(unstable_bad if is_unstable(unstable, action, cell) else stable_bad).append(entry)- # SPLIT BEFORE THE COUNTS ARE PRINTED. Printing "stable actions differing: 1" above a verdict- # of 0 is how a reader concludes the tool is lying to them; the headline number has to be the- # one the exit code is taken from.+ # SPLIT BEFORE THE COUNTS ARE PRINTED: "stable actions differing: 1" above a verdict of 0 is how+ # a reader concludes the tool is lying, so the headline number must be the one the exit code+ # comes from.stable_bad, uncorroborated = corroborated(stable_bad, min_reps)one_sided, one_sided_weak = corroborated(one_sided, min_reps)expect_bad, expect_weak = corroborated(expect_bad, min_reps)@@ -1873,10 +1731,8 @@ def report(f"pointer-events)")- # THE SAME SET, COUNTED TWICE, checked rather than assumed: `compared` is what the coverage- # floor is applied to, `scored` is the count this report has always printed, and `stable_bad`- # splits into itself plus `uncorroborated` above. The assert is here so a later edit to either- # branch cannot make the floor answer about a different set than the summary does.+ # THE SAME SET, COUNTED TWICE, checked rather than assumed, so a later edit to either branch+ # cannot make the coverage floor answer about a different set than the summary does.scored = matched + len(stable_bad) + len(uncorroborated) + len(unstable_bad)assert len(compared) == scored, (len(compared), scored)shortfall = coverage_shortfall(scored, len(results), min_compared)@@ -1925,28 +1781,18 @@ def report(for action, shard, cell, why, *_dir in one_sided_unstable[:8]:print(f" {action:<26} {shard} {cell}: {why[0]}")- # A DEMOTED DIFFERENCE IS STILL A COMPARISON. `corroborated` moves a difference seen in fewer- # than `min_reps` repetitions out of `stable_bad`, and that must not make the pair look like- # one that never yielded a verdict: the digest was taken on both arms, it differed, and the- # corroboration bar is the only reason it is not counted as a regression. Reading `matched`- # alone, a payload whose every difference was uncorroborated exited 2 "NOTHING WAS COMPARED"- # over a run that compared everything it had -- which is the opposite of the false green this- # guard exists to catch, and would make `--min-reps 2` fail runs it was meant to quieten.+ # A DEMOTED DIFFERENCE IS STILL A COMPARISON: `corroborated` moves a sub-`min_reps` difference out+ # of `stable_bad`, and reading `matched` alone a payload whose every difference was uncorroborated+ # exited 2 "NOTHING WAS COMPARED" over a run that compared everything it had.decided = matched + len(uncorroborated) + len(one_sided_weak)if stable_bad:print("\n UI PARITY DIFFERENCES ON STABLE ACTIONS -- these need explaining:")for action, shard, cell, moved in stable_bad:print(f" {action:<26} {shard} {cell}: {', '.join(moved[:4])}")elif decided == 0:- # NOT A PASS, and it must not print like one. Not one pair produced a digest verdict, so- # "no stable action rendered differently" would be true, reassuring and about nothing --- # the exact shape of a check that silently does nothing, which is what the NOT COMPARABLE- # bucket exists to prevent elsewhere in this file.- #- # THE GUARD USED TO REQUIRE `inapplicable`, so it only caught the windowed-mount route to- # an empty comparison. A run whose parity probes all failed, or whose slots were all- # missed, produced nothing but NOT COMPARABLE and NOT EXERCISED rows, no inapplicable ones,- # and exited 0 under a heading that reads as a clean structural pass.+ # NOT A PASS, and it must not print like one: with no pair producing a digest verdict, "no stable+ # action rendered differently" is true, reassuring and about nothing. The guard used to require+ # `inapplicable`, so a run whose parity probes all failed exited 0 under a clean-looking heading.print(f"\n NOTHING WAS COMPARED. Not one of the {len(results)} pair(s) here yielded a "f"structural verdict ({len(inapplicable)} windowed, {len(blind)} not comparable, "@@ -1966,9 +1812,8 @@ def report()if uncorroborated:- # Printed in full. A build renders the same way every time, so one repetition out of two- # is evidence of the run rather than of the build -- but it is still a reading, and a- # reader chasing an intermittent regression needs to see it rather than be told nothing.+ # Printed in full: one repetition out of two is evidence of the run rather than of the build, but+ # it is still a reading a reader chasing an intermittent regression needs to see.print(f"\n UNCORROBORATED -- differed in only one repetition, so not counted as a "f"regression:"@@ -1982,8 +1827,8 @@ def report(print(f" {action:<26} {shard} {cell}: {why[0]}")if idle:- # Named surfaces, deduplicated: what matters is WHICH actions this run never opened, not- # that it failed to open one of them sixteen separate times.+ # Named surfaces, deduplicated: what matters is WHICH actions this run never opened, not that it+ # failed to open one of them sixteen times.names = sorted({entry[0] for entry in idle})print(f"\n NOT EXERCISED -- {len(idle)} pair(s) over {len(names)} action(s) that did not "@@ -2003,19 +1848,13 @@ def report(for action, shard, cell, moved in unstable_bad[:8]:print(f" {action:<26} {shard} {cell}: {', '.join(moved[:3])}")# A one-sided action is a failure on the same footing as a differing digest: the two builds- # behaved differently and the difference happens to leave no digest to compare. `expect_bad`- # joins them for the reason given where it is collected: an assertion that failed on one arm- # only is the two builds behaving differently, and it is the one shape a digest cannot show.+ # behaved differently and the difference leaves no digest to compare.+ # `expect_bad` is collected ahead of the verdict branches.if stable_bad or one_sided or expect_bad:return Outcome(1, frozenset(compared), _keys(results))- # 2, the same code the empty-payload path uses, and for the same reason: the tool was asked a- # question it could not answer. Exiting 0 here would let CI go green on a run where the parity- # check was structurally incapable of firing -- whether because every pair was a windowed mount- # or because every capture failed. `decided`, not `matched`, for the reason given above it.- #- # AFTER the failure return, not before it: a run can find a real regression while deciding- # nothing -- `expect_bad` is collected ahead of the verdict branches and survives a pair that- # `continue`s -- and "it failed" is the more specific answer than "it could not tell".+ # 2, the same code the empty-payload path uses: the tool was asked a question it could not+ # answer, and exiting 0 would let CI go green on a run structurally incapable of firing.+ # `decided`, not `matched`. AFTER the failure return, since "it failed" is more specific.if decided == 0:return Outcome(2, frozenset(compared), _keys(results))return Outcome(0, frozenset(compared), _keys(results))@@ -2169,10 +2008,9 @@ def main(argv: list[str] | None = None) -> int:)ap.add_argument("--mode",- # `structural` and `behavior` are ALIASES, not extra modes. The report header prints- # "(STRUCTURAL MODE)" and the pull request template asks for "the structural digest", so- # a reader who follows either and types `--mode structural` should not be told it is not- # a choice. Same for the American spelling of behaviour.+ # `structural` and `behavior` are ALIASES, not extra modes: the report header prints+ # "(STRUCTURAL MODE)" and the PR template asks for "the structural digest", so a reader who types+ # either should not be told it is not a choice.choices = ("auto", "digest", "structural", "visible", "behaviour", "behavior"),default = "auto",help = (@@ -2233,28 +2071,16 @@ def main(argv: list[str] | None = None) -> int:args = ap.parse_args(argv)args.mode = {"structural": "digest", "behavior": "behaviour"}.get(args.mode, args.mode)- # THE MODE DECISION FIRST, before the unstable set is even derived. Deriving an unstable set- # from a null control and then not using it because the payload is windowed would print a page- # of scoring apparatus that has no bearing on the report underneath it.- #- # PER ACTION PAIR, and per payload and per invocation were both too coarse:- #- # per invocation `ui_parity normal_run windowed_run` scored the NORMAL run behaviourally- # too, so an ordinary DOM regression in a fully mounted arm went unreported- # because an unrelated payload on the same command line was windowed.- # per payload one payload holds several rungs, and the readiness gate deliberately- # permits an arm to mount everything at 1K and a window at 100K. One windowed- # large-rung capture then suppressed the structural digest on every fully- # mounted pair beside it, at the rungs where that digest is exactly the right- # question.- #- # `--mode visible|behaviour|digest` still forces every pair, because that is what forcing means.+ # THE MODE DECISION FIRST, before the unstable set is derived, or a windowed payload prints a+ # page of scoring apparatus with no bearing on the report. PER ACTION PAIR: per invocation scored+ # a normal run behaviourally because an unrelated payload on the same command line was windowed,+ # and per payload let one windowed 100K capture suppress the digest on every mounted pair.plan: list[dict] = []for pattern in args.payloads:paths = shards_of(pattern)if not paths:- # Kept with no pairs: the structural loop below is what reports a pattern that matched- # no payload, and it exits 2 for it.+ # Kept with no pairs: the structural loop below is what reports a pattern that matched no+ # payload, and it exits 2 for it.plan.append({"pattern": pattern, "paths": [], "windowed": set(), "structural": None})continueif args.mode == "digest":@@ -2279,8 +2105,8 @@ def main(argv: list[str] | None = None) -> int:"pattern": pattern,"paths": paths,"windowed": windowed,- # None, not an empty set, when the payload holds no action pairs at all: `report`- # is what says NO PARITY DATA and exits 2, and it has to be reached to say it.+ # None, not an empty set, when the payload holds no action pairs at all: `report` is what says+ # NO PARITY DATA and exits 2, and it has to be reached to say it."structural": (set(decided) - windowed) if decided else None,"why": next((w for _k, (m, w) in sorted(decided.items()) if m == WINDOWED), ""),}@@ -2288,10 +2114,9 @@ def main(argv: list[str] | None = None) -> int:worst = 0scored_windowed = 0- # THE COVERAGE FLOOR'S OWN BOOKKEEPING, kept per payload pattern and across every mode that- # pattern was scored in. A SET rather than a running total because `--mode auto` scores each- # windowed pair twice, once on the visible region and once on the invariants; see `Outcome`.- # Keyed by pattern because the floor is a question about one film; see `_floored`.+ # THE COVERAGE FLOOR'S OWN BOOKKEEPING, per payload pattern and across every mode. A SET rather+ # than a running total because `--mode auto` scores each windowed pair twice; keyed by pattern+ # because the floor is a question about one film.compared_pairs: dict[str, set[tuple]] = {}seen_pairs: dict[str, set[tuple]] = {}@@ -2320,48 +2145,35 @@ def main(argv: list[str] | None = None) -> int:f"{len(struct)} fully mounted pair(s) are scored structurally further down.")for key in sorted(win)[:8]:- # shard, rung, rep and the action. The session term of the key is left out of- # the line: it is part of a pair's identity, not part of naming it to a reader.+ # shard, rung, rep and the action. The session term is left out of the line: it is part of a+ # pair's identity, not part of naming it to a reader.print(f" windowed: {key[0]} {key[1]} {key[2]} {key[4]}")if len(win) > 8:print(f" ... and {len(win) - 8} more windowed pair(s)")scored_windowed += len(win)- # Per pattern, and reset here rather than outside the loop: two payload patterns are two- # films and must not pool their coverage, which is the rule `_floored` already states.+ # Per pattern, and reset here rather than outside the loop: two payload patterns are two films+ # and must not pool their coverage.windowed_compared: Optional[frozenset[tuple]] = Nonewindowed_seen: set[tuple] = set()- # BOTH, and in this order. Visible-region parity is the verdict the off-screen exemption- # asks for and it is the one that can FAIL a windowed arm for something a user would see.- # The behavioural invariants are the complement: they catch what a viewport comparison- # cannot, such as a clipboard that no longer carries the thread. Neither subsumes the other,- # so a windowed pair gets both and the run fails if either does.+ # BOTH, and in this order. Visible-region parity is the verdict the off-screen exemption asks for+ # and can FAIL a windowed arm for something a user would see; the behavioural invariants catch+ # what a viewport comparison cannot, such as a clipboard that no longer carries the thread.if args.mode in ("auto", "visible"):if vis_unstable is None:- # The floor, derived from the null control the caller passed. Without it an- # identical pair of builds outscores the arm under test. Derived once: it is a- # property of the null control, not of the payload being scored against it.+ # The floor, derived from the null control the caller passed; without it an identical pair of+ # builds outscores the arm under test. Derived once: it is a property of the null control.vis_null: list[Path] = []for pat in args.null:vis_null.extend(shards_of(pat))vis_unstable = visible_unstable_set(vis_null)vis_null_tiers = one_tier(vis_null, "null control")vis_null_corpora = one_corpus(vis_null, "null control")- # A FLOOR FROM ANOTHER FILM IS NOT THIS PAYLOAD'S FLOOR, ON EITHER AXIS. `tier_of`- # states the mechanism for the film: the slot spacing decides which actions land- # inside a live stream and so differ against themselves, and `--tier fast` then- # `--tier standard` is the documented way to work, which is exactly how a stale fast- # null control ends up on the command line of a standard run. `one_corpus` states the- # same mechanism one level down for the thread the film drove, and that axis is the- # quieter of the two: a corpus revision lands, the payload is re-recorded against it,- # and an older null control is still sitting in the directory the workflow globs.- # `one_tier` and `one_corpus` above refuse a null control that is internally mixed;- # `cross_side_mismatch` refuses one that is internally consistent but belongs to a- # different film or a different thread. The structural section further down says both- # out loud, and an ALL-WINDOWED payload -- which is every payload under- # `--mode visible` -- returns before it ever reaches that warning, so a floor measured- # on the wrong film or the wrong corpus silenced a real visible difference and the- # command exited 0 without a word about it. Refused rather than applied: an unfloored- # report says what it is missing, a wrongly floored one does not.+ # A FLOOR FROM ANOTHER FILM IS NOT THIS PAYLOAD'S FLOOR, ON EITHER AXIS. Slot spacing decides+ # which actions land inside a live stream, and `--tier fast` then `--tier standard` is the+ # documented way to work, which is how a stale fast null control reaches a standard run; the+ # corpus axis is quieter, since a re-recorded payload leaves an older null in the globbed+ # directory. `one_tier`/`one_corpus` refuse an internally mixed null, `cross_side_mismatch`+ # refuses a consistent one belonging to a different film. Refused rather than applied.floor = vis_unstablemismatch = cross_side_mismatch(one_tier(paths, "payload"),@@ -2391,54 +2203,25 @@ def main(argv: list[str] | None = None) -> int:min_reps = args.min_reps,)worst = max(worst, int(beh))- # Only where behavioural mode is the ONLY mode, i.e. `--mode behaviour`. Under- # `--mode auto` it adds nothing to the numerator; see below.+ # Only where behavioural mode is the ONLY mode, i.e. `--mode behaviour`. Under `--mode auto` it+ # adds nothing to the numerator; see below.if windowed_compared is None:windowed_compared = frozenset(beh.compared)windowed_seen |= set(beh.seen)- # NOT UNIONED ACROSS THE TWO WINDOWED MODES. The union is right for the structural set- # below, which is a DISJOINT set of pairs -- coverage there really does add up. It is wrong- # here: under `--mode auto` these two are not two slices of the film, they are two verdicts- # on the SAME pairs, which is what the comment above says when it says neither subsumes the- # other. Unioned, one stood in for the other: behavioural mode reaching all 16 pairs while- # visible capture succeeded on 1 and returned NOT COMPARABLE for the other 15 still put 16- # keys in the set, cleared `--min-compared 16`, and exited 0 with essentially the whole- # appearance surface unmeasured.- #- # AND THE TWO ARE NOT INTERCHANGEABLE, so this is not an intersection either. The modes are- # asymmetric in what they even claim to reach: visible-region parity applies to EVERY- # windowed pair, while a behavioural invariant only exists for the handful of actions that- # declare one -- every other pair is reported UNCHECKED, which is explicitly not a pass and- # equally not a shortfall. Intersecting would therefore fail a run for pairs behavioural- # mode never had an opinion about, which is a false red measured at 1 of 4 on the coverage- # fixture here.- #- # So in `auto` the numerator is the visible verdict, the one that is owed on every pair.- # Behavioural coverage cannot vouch for the appearance of a pair the viewport comparison- # could not read, which is exactly the substitution above.- #- # `seen` stays a union: it is the denominator, the pairs the reports were OFFERED, and a- # pair offered to either was offered. Shrinking it alongside the numerator would hide the- # shortfall it exists to express.+ # NOT UNIONED ACROSS THE TWO WINDOWED MODES. The union is right for the structural set below,+ # which is DISJOINT; here the two are two verdicts on the SAME pairs, and unioned one stood in+ # for the other -- behavioural mode reaching all 16 pairs while visible capture succeeded on 1+ # still cleared `--min-compared 16`. Not an intersection either: a behavioural invariant only+ # exists for the few actions that declare one, so intersecting fails a run for pairs it had no+ # opinion about. So in `auto` the numerator is the visible verdict; `seen` stays a union.if windowed_compared is not None:compared_pairs.setdefault(pattern, set()).update(windowed_compared)seen_pairs.setdefault(pattern, set()).update(windowed_seen)- # AHEAD OF THE STRUCTURAL EARLY RETURN, because this mode does not read the plan at all.- #- # It takes its shards from `args.payloads` directly, audits each one AS a null control, resets- # `worst` and returns unconditionally -- so it shares nothing with the windowed/structural- # split it used to sit under. Below the return it was simply unreachable whenever `remaining`- # came out empty, and that is not an exotic payload: `--mode visible` and `--mode behaviour`- # both hard-set `structural` to an empty set for EVERY pattern, so `--audit-null` with either- # of them could never run the audit at all, and auto mode joined them the moment every pair- # classified as windowed. The command then exited 0 out of the ordinary visible report, having- # silently skipped the audit the caller explicitly asked for and the `--compared-in`- # enforcement that goes with it -- an option whose whole promise is to FAIL unless the null- # decided the actions the result needs an excuse for.- #- # Moved here rather than duplicated above the return: one call site, and the auto-mode path- # that already worked reaches it after exactly the same windowed reports as before.+ # AHEAD OF THE STRUCTURAL EARLY RETURN, because this mode does not read the plan at all: it takes+ # shards from `args.payloads`, audits each AS a null control and returns unconditionally. Below+ # the return it was unreachable whenever `remaining` came out empty, which `--mode visible` and+ # `--mode behaviour` guarantee, so `--audit-null` with either silently skipped the audit.if args.audit_null:allow = frozenset(a.strip() for a in args.allow_undecided.split(",") if a.strip())scope = None@@ -2471,15 +2254,11 @@ def main(argv: list[str] | None = None) -> int:worst = max(worst, rc)return worst- # THE STRUCTURAL EARLY RETURN, and the reason there is now a floor check on both sides of it.- # Everything below is the structural half, and a payload with no fully mounted pair has none, so- # returning here is right for all of it. `--min-compared` was not one of those things: it is a- # COVERAGE floor asking whether the film ran, and a windowed run's film runs exactly as much.- #- # This return is the FOURTH cross-cutting enforcement found sitting below it -- the null- # control's tier axis, then its corpus axis, then `--audit-null`, then this. The first three- # were hoisted one at a time; so the floor goes through `coverage_shortfall` on BOTH exits, over- # the pairs every mode compared, with the reports handing those pairs back.+ # THE STRUCTURAL EARLY RETURN, and why there is now a floor check on both sides of it. Everything+ # below is the structural half, which a payload with no fully mounted pair has none of, but+ # `--min-compared` is a COVERAGE floor asking whether the film ran, and a windowed run's film runs+ # exactly as much. The FOURTH cross-cutting enforcement found sitting below it, so the floor goes+ # through `coverage_shortfall` on BOTH exits.remaining = [e for e in plan if e["structural"] is None or e["structural"]]if not remaining:return _floored(worst, compared_pairs, seen_pairs, args.min_compared)@@ -2512,10 +2291,9 @@ def main(argv: list[str] | None = None) -> int:)print(" pass --null OUTDIR of a base-vs-base run to derive it instead.")- # `null_tiers` is the set `one_tier` already refused a mixed null control for, above. NOT- # reassigned from `tier_of` here: that would be the same set without the refusal in front of it,- # and `worst` is NOT reset either -- the windowed reports above have already run and a- # behavioural or visible failure there may not be dropped by the structural pass below.+ # `null_tiers` is the set `one_tier` already refused a mixed null control for. NOT reassigned+ # from `tier_of` here, which would be the same set without the refusal in front of it, and `worst`+ # is NOT reset either: a windowed failure above may not be dropped by the structural pass.scored_structural = 0for entry in remaining:pattern, paths, select = entry["pattern"], entry["paths"], entry["structural"]@@ -2524,21 +2302,18 @@ def main(argv: list[str] | None = None) -> int:worst = max(worst, 2)continuetiers = one_tier(paths, "payload")- # REFUSED, NOT WARNED. A tier or corpus the null control was not recorded against makes- # its unstable set inapplicable to this payload, and a warning printed above a verdict is- # read as a verdict. `cross_side_mismatch` covers the tier this used to warn about and the- # corpus it did not check at all.+ # REFUSED, NOT WARNED: a tier or corpus the null was not recorded against makes its unstable set+ # inapplicable, and a warning printed above a verdict is read as a verdict. `cross_side_mismatch`+ # covers the tier this used to warn about and the corpus it did not check at all.corpora = one_corpus(paths, "payload")mismatch = cross_side_mismatch(tiers, null_tiers, corpora, null_corpora)if mismatch:print(f"\n REFUSING to score {pattern}: {mismatch}")worst = max(worst, 2)continue- # CONFINED TO THE RUNNER BEING SCORED, and only ever downward. The exemptions above were- # measured in the other matrix job, on another machine, minutes away; side A of THIS- # payload is the same build in every repetition, so it can say whether this machine- # reproduces them. One that it positively contradicts is dropped, one it could not decide- # is kept.+ # CONFINED TO THE RUNNER BEING SCORED, and only ever downward: the exemptions above were measured+ # in another matrix job on another machine, while side A of THIS payload is the same build in+ # every repetition. One it positively contradicts is dropped; one it could not decide is kept.effective = unstableif derived:local_unstable, local_stable = in_arm_repeatability(paths)@@ -2552,30 +2327,25 @@ def main(argv: list[str] | None = None) -> int:)label = f"UI PARITY: {pattern}"if select is not None and entry["windowed"]:- # WHICH PAIRS THESE ARE, in the heading. A structural section that silently covered- # part of a payload would read as a verdict on all of it.+ # WHICH PAIRS THESE ARE, in the heading: a structural section that silently covered part of a+ # payload would read as a verdict on all of it.label += (f" ({len(select)} fully mounted pair(s) of "f"{len(select) + len(entry['windowed'])})")scored_structural += len(select) if select is not None else 0- # BOTH SIDES OF THE MERGE, and the argument list is why this one had to be read rather- # than chosen. `select` is the FOURTH positional here; the incoming call omitted it,- # because on that branch this function had no windowed split to select from. Taken as- # written it would have bound `args.min_reps` to `select` and `args.min_compared` to- # `min_reps`, leaving the coverage floor at its default of 0 -- a silently disabled guard- # and a structural pass scoring against an integer, with nothing failing to say so.- # `min_compared` IS NOT PASSED DOWN ANY MORE: that is the change, not an omission. A floor- # applied inside each report is checked separately per report, so a run that compared nine- # windowed pairs and nine structural ones would fail a floor of sixteen twice over having- # compared eighteen. `report` keeps the parameter (the selftests call it directly and it is- # a complete report on its own); `main` collects the pairs and applies the floor below.+ # BOTH SIDES OF THE MERGE, and the argument list is why this had to be read rather than chosen:+ # `select` is the FOURTH positional and the incoming call omitted it, which would have bound+ # `args.min_reps` to `select` and left the coverage floor at 0. `min_compared` is deliberately NOT+ # passed down any more: a floor applied inside each report is checked per report, so a run+ # comparing nine windowed and nine structural pairs would fail a floor of sixteen twice over.+ # `args.min_compared` was the argument that went missing.+ # `report` (`elif decided == 0`) and `visible_report` (`and not not_run`) each check their own.struct = report(paths, label, effective, select, args.min_reps)worst = max(worst, int(struct))note(pattern, struct)- # COMBINED, and the worst outcome wins. A behavioural failure on one payload is not cancelled- # by a structural pass on another, and a structural failure at the fully mounted rungs is not- # cancelled by the windowed rungs passing their own two modes.+ # COMBINED, and the worst outcome wins: a behavioural failure on one payload is not cancelled by+ # a structural pass on another.if scored_windowed and scored_structural:print(f"\nCOMBINED EXIT STATUS {worst}: {scored_windowed} pair(s) scored on the visible "diff --git a/unsloth-cli.py b/unsloth-cli.pyindex 6ed73d10447..0e1db20a287 100644--- a/unsloth-cli.py+++ b/unsloth-cli.py@@ -109,8 +109,7 @@ def _save_or_push_model(model, tokenizer, args, is_mlx):print("Warning: The model is not saved!")return- # Enter the GGUF branch when saving or pushing GGUF, so --push_gguf works- # without --save_gguf (the local save is guarded separately below).+ # Enter the GGUF branch when saving or pushing GGUF, so --push_gguf works without --save_gguf.if args.save_gguf or args.push_gguf:if not args.save_gguf:print("Warning: --save_gguf not set, pushing GGUF to hub without saving locally.")@@ -193,7 +192,6 @@ def run(args):if args.use_dora and is_mlx:raise NotImplementedError("DoRA is not supported for MLX training yet.")- # Load model and tokenizerdevice_map, distributed = _prepare_device_map(is_mlx)model, tokenizer = FastLanguageModel.from_pretrained(model_name = args.model_name,@@ -203,7 +201,6 @@ def run(args):device_map = device_map,)- # Configure PEFT modelmodel = FastLanguageModel.get_peft_model(model,r = args.r,@@ -237,7 +234,7 @@ def run(args):### Response:{}"""- EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN+ EOS_TOKEN = tokenizer.eos_tokendef formatting_prompts_func(examples):instructions = examples["instruction"]@@ -257,7 +254,6 @@ def run(args):)dataset = loader.load_from_file(args.raw_text_file)elif args.dataset.endswith((".txt", ".md", ".json", ".jsonl")):- # Auto-detect local raw text filesloader = _raw_text_loader_for_backend(RawTextDataLoader, tokenizer, is_mlx)dataset = loader.load_from_file(args.dataset)else:@@ -268,20 +264,16 @@ def run(args):else:dataset = load_dataset(args.dataset, split = "train")- # Format structured datasetsdataset = dataset.map(formatting_prompts_func, batched = True)return dataset- # Load dataset using smart loaderdataset = load_dataset_smart(args)print("Data is formatted and ready!")- # Configure training argumentstraining_args = _build_sft_config(SFTConfig, args, is_mlx, is_bfloat16_supported())if distributed:training_args.ddp_find_unused_parameters = False- # Initialize trainertrainer = SFTTrainer(model = model,processing_class = tokenizer,diff --git a/unsloth_cli/__init__.py b/unsloth_cli/__init__.pyindex 1029a100e19..fc601c4eeb2 100644--- a/unsloth_cli/__init__.py+++ b/unsloth_cli/__init__.py@@ -4,9 +4,7 @@import os as _osimport sys as _sys-# Are we the `unsloth` console script, rather than a library import? Both the-# stream guard below and the `-np<N>` rewrite further down are entry-point-# behaviour and must not reach into a host application that imports us.+# Entry-point-only behaviour (stream guard, -np<N> rewrite): must not reach a host that imports us._entry_base = _os.path.basename(_sys.argv[0]).lower() if _sys.argv else ""_is_entry_point = _entry_base in {"unsloth", "unsloth.exe"}_windows_studio_mutation_entry = (@@ -57,14 +55,9 @@ if _is_entry_point:from unsloth_cli._system_dir_guard import check_working_directory as _check_working_directory-# Running from System32 or any subdir WILL cause errors if not prevented. A-# command the folder cannot affect (the ones Unsloth Desktop spawns, issue #8510)-# moves out of it; everything else stops in the callback below.-#-# Before the command imports, since unsloth_cli.commands.studio resolves-# STUDIO_HOME at import time and a relative UNSLOTH_STUDIO_HOME would otherwise-# be pinned to the folder we are leaving. The message waits for typer to render-# it. A library import reaches the same check from the callback instead.+# Running from System32 or a subdir breaks commands; move out before the command imports, since+# commands.studio resolves STUDIO_HOME at import time (issue #8510).+# A relative UNSLOTH_STUDIO_HOME would otherwise resolve against System32._startup_guard = (_check_working_directory(_sys.argv[1:], _os.environ, _sys.platform) if _is_entry_point else None)@@ -107,14 +100,11 @@ def _prepare_entry_point():return_reconfigure_entry_point_streams()_expand_attached_np_short()- # Set last, so a raise leaves the work retryable rather than silently- # skipped. Neither call can currently raise -- the first swallows everything- # and the second is pure argv manipulation -- but the ordering costs nothing.+ # Set last, so a raise leaves the work retryable rather than silently skipped._entry_point_prepared = True-# Canonicalise `-np<N>` only under the `unsloth` console-script;-# third-party scripts that import unsloth_cli keep their argv intact.+# Canonicalise `-np<N>` only under the console-script; imports keep their argv intact.if _is_entry_point:_prepare_entry_point()del _entry_base, _is_entry_point@@ -176,8 +166,7 @@ def _invocation_args(ctx):return list(captured)if not ctx.invoked_subcommand:return _sys.argv[1:]- # No capture and no tail to read: assume it holds a path, so an invocation- # that cannot be read in full is refused rather than relocated.+ # No capture and no tail: assume it holds a path, so refuse rather than relocate.return [ctx.invoked_subcommand, *(list(getattr(ctx, "args", None) or []) or ["..."])]@@ -193,13 +182,11 @@ def main(help = "Show version and exit.",),):- # Consume the import-time result once: a host calling the app repeatedly can- # chdir between calls, so each later call is checked afresh.+ # Consume the import-time result once: a host can chdir between repeated app() calls.global _startup_guard_guard, _startup_guard = _startup_guard, Noneif _guard is None:- # A host reaches this after commands.studio has resolved STUDIO_HOME at- # import time, so moving now would leave that cached root behind.+ # A host reaches this after commands.studio cached STUDIO_HOME, so moving now strands that root._guard = _check_working_directory(_invocation_args(ctx),_os.environ,@@ -233,8 +220,7 @@ if not _windows_studio_mutation_entry:help = "Deprecated alias for `unsloth start`.",)- # top-level `unsloth run` aliases `unsloth studio run`; same context- # so unknown flags still pass through to llama-server.+ # top-level `unsloth run` aliases `unsloth studio run`; same context so unknown flags pass through to llama-server.app.command("run",context_settings = {diff --git a/unsloth_cli/__main__.py b/unsloth_cli/__main__.pyindex 56b9b49ded6..c89bd6c6dc9 100644--- a/unsloth_cli/__main__.py+++ b/unsloth_cli/__main__.py@@ -41,15 +41,11 @@ Output is identical to the console script, which takes three things:import sys-# Before the import, so a direct `python path/to/unsloth_cli/__main__.py` run-# takes the console-script gate in __init__ rather than needing the call below.+# Before the import, so a direct `python .../__main__.py` run takes the console-script gate in __init__.sys.argv[0] = "unsloth"import unsloth_cli # noqa: E402unsloth_cli._prepare_entry_point()-# sys.exit, like the generated console script's `sys.exit(app())`. Typer raises-# SystemExit itself in standalone mode, so today both spellings exit the same way,-# but a returned value has to become the exit status here too or the two routes-# stop agreeing the moment one exists.+# A returned value must become the exit status, like the console script's `sys.exit(app())`.sys.exit(unsloth_cli.app(prog_name = "unsloth"))diff --git a/unsloth_cli/_inference.py b/unsloth_cli/_inference.pyindex 8a2a80d3ea6..ae04ccd77f9 100644--- a/unsloth_cli/_inference.py+++ b/unsloth_cli/_inference.py@@ -14,10 +14,9 @@ from typing import List, Literal, Optionalimport typer-# Canonical speculative-decoding modes, mirroring the backend's-# _CANONICAL_SPEC_MODES. Named once so the CLI's option annotations, the HTTP-# payload builders and the in-process loader cannot drift apart when a mode is-# added; typer reads it at runtime to validate --speculative-type.+# Canonical speculative-decoding modes, mirroring the backend's _CANONICAL_SPEC_MODES. Named once+# so the CLI's option annotations, the HTTP payload builders and the in-process loader cannot+# drift apart; typer reads it at runtime to validate --speculative-type.SpeculativeType = Literal["auto", "mtp", "dspark", "dflash", "ngram", "mtp+ngram", "off", "ngram-simple"]@@ -60,10 +59,10 @@ def urlopen_no_redirect(request, timeout):return _no_redirect_opener.open(request, timeout = timeout)-# /api/inference/load and /unload pad their body so a proxy cannot time a slow load-# out, committing the 200 before the work finishes. A failure found after that travels-# only in-band under this key (studio/backend/routes/inference.py), so a client that-# treats any 200 as success reports a failed load as a successful one.+# /api/inference/load and /unload pad their body so a proxy cannot time a slow load out,+# committing the 200 before the work finishes. A failure found after that travels only in-band+# under this key, so a client that treats any 200 as success reports a failed load as a+# successful one._DEFERRED_ERROR_KEY = "_deferred_error"@@ -141,14 +140,13 @@ def ensure_studio_backend_path() -> None:def configure_quiet_logging() -> None:import logging- # The CLI never configures structlog, so without this every backend INFO- # line prints. LOG_LEVEL is exported so the worker subprocess inherits it.+ # The CLI never configures structlog, so without this every backend INFO line prints. LOG_LEVEL+ # is exported so the worker subprocess inherits it.level_name = os.environ.setdefault("LOG_LEVEL", "WARNING").upper()level = getattr(logging, level_name, logging.WARNING)os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")- # Quieting logs must not fail a command before the import that really needs- # structlog gets to report itself.+ # Quieting logs must not fail a command before the import that really needs structlog gets to report itself.try:import structlogexcept ModuleNotFoundError:@@ -274,8 +272,8 @@ def visible_text(text: str, show_thinking: bool) -> str:def stream_to_stdout(stream, show_thinking: bool) -> str:- # Backends yield the full text-so-far on each step (llama.cpp ends with a- # metadata dict, skipped); print the growing tail, return the raw text.+ # Backends yield the full text-so-far on each step (llama.cpp ends with a metadata dict,+ # skipped); print the growing tail, return the raw text.raw = ""shown = ""for chunk in stream:@@ -318,9 +316,8 @@ def collect_stream(stream, show_thinking: bool) -> str:def raise_on_streamed_error(stream):- # Match real backend errors by type (GenStreamError), not the "Error:" text- # prefix, so a completion whose text opens with "Error:" is not misread as a- # backend failure.+ # Match real backend errors by type (GenStreamError), not the "Error:" text prefix, so a+ # completion whose text opens with "Error:" is not misread as a backend failure.try:ensure_studio_backend_path()from core.inference.orchestrator import GenStreamError@@ -402,8 +399,8 @@ class ChatBackend:return self._backend.generate_chat_response(**gen_kwargs)def close(self) -> None:- # Shut the worker down directly: the graceful unload_model waits for- # an ack that compare mode can swallow, hanging exit for minutes.+ # Shut the worker down directly: the graceful unload_model waits for an ack that compare mode can+ # swallow, hanging exit for minutes.try:if self._kind == "gguf":self._backend.unload_model()@@ -620,8 +617,8 @@ def find_studio_server(timeout: float = 3.0) -> Optional[str]:import urllib.requestbase = os.environ.get("UNSLOTH_STUDIO_URL", "http://127.0.0.1:8888").rstrip("/")- # Try the concrete loopback addresses in order and return the first that- # answers, so the rest of the flow talks to that exact address.+ # Try the concrete loopback addresses in order and return the first that answers, so the rest of+ # the flow talks to that exact address.for candidate in _loopback_candidate_bases(base):request = urllib.request.Request(f"{candidate}/api/health", headers = {"User-Agent": _USER_AGENT}@@ -673,10 +670,9 @@ def verify_studio_identity(base: str, timeout: float = 3.0) -> bool:parsed = urlparse(base)host = parsed.hostname or ""port = parsed.port or (443 if parsed.scheme == "https" else 80)- # Resolve to one concrete address and talk to *that* address, then bind the- # proof to (address, port). A name like localhost can resolve to a squatter on- # ::1 while the real Unsloth is on 127.0.0.1; connecting to the resolved IP and- # binding to it means a proof relayed from a different address/port won't match.+ # Resolve to one concrete address and talk to *that* address, then bind the proof to (address,+ # port). A name like localhost can resolve to a squatter on ::1 while the real Unsloth is on+ # 127.0.0.1.try:ip = socket.getaddrinfo(host, port, type = socket.SOCK_STREAM)[0][4][0]except Exception:@@ -689,8 +685,8 @@ def verify_studio_identity(base: str, timeout: float = 3.0) -> bool:headers = {"User-Agent": _USER_AGENT, "Host": parsed.netloc},)try:- # No redirects: a 302 could relay a real Unsloth's proof (see urlopen_no_redirect).- # Cap the read: the server is still unverified, so don't trust its length.+ # No redirects: a 302 could relay a real Unsloth's proof (see urlopen_no_redirect). Cap the read:+ # the server is still unverified.with urlopen_no_redirect(request, timeout = timeout) as response:proof = json.loads(response.read(65536).decode() or "{}").get("proof")except Exception:@@ -780,9 +776,8 @@ class HttpChatBackend:if spec_draft_n_max is not None:payload["spec_draft_n_max"] = spec_draft_n_maxtry:- # Read the body, don't close at the headers: a slow load commits its 200- # early and pads until done, so closing here would generate mid-load and- # discard the only report of a late failure.+ # Read the body, don't close at the headers: a slow load commits its 200 early and pads until+ # done, so closing here would generate mid-load and discard the only report of a late failure.read_json_checking_deferred_error(self._base + "/api/inference/load",self._request("POST", "/api/inference/load", payload),@@ -826,8 +821,7 @@ class HttpChatBackend:)def cumulative():- # Accumulate SSE deltas into the full-text-so-far convention the- # stream helpers expect.+ # Accumulate SSE deltas into the full-text-so-far convention the stream helpers expect.text = ""with resp:for raw_line in resp:@@ -852,8 +846,8 @@ class HttpChatBackend:if not delta:continuetext += delta- # An emoji can arrive split across two deltas as lone- # surrogate halves: hold back a trailing half, merge pairs.+ # An emoji can arrive split across two deltas as lone surrogate halves: hold back a trailing+ # half, merge pairs.visible = textif "\ud800" <= visible[-1] <= "\udbff":visible = visible[:-1]@@ -881,8 +875,8 @@ def connect_studio_server(if not base_url:return None- # Explicit server (UNSLOTH_STUDIO_URL) we can't safely attach to -> fail loudly;- # opportunistic local discovery just falls back to a local load.+ # Explicit server (UNSLOTH_STUDIO_URL) we can't safely attach to means fail loudly; opportunistic+ # local discovery just falls back to a local load.explicit = bool(os.environ.get("UNSLOTH_STUDIO_URL"))def _refuse(reason: str):@@ -895,8 +889,8 @@ def connect_studio_server()raise typer.Exit(code = 1)- # Only hand the self-issued JWT (signed with the local secret) to loopback: a- # remote URL is unverified and a real remote Unsloth would reject it anyway.+ # Only hand the self-issued JWT (signed with the local secret) to loopback: a remote URL is+ # unverified and a real remote Unsloth would reject it anyway.if not is_loopback_url(base_url):return _refuse("it isn't a local Unsloth, so a self-issued token can't "diff --git a/unsloth_cli/_model_catalog.py b/unsloth_cli/_model_catalog.pyindex 29ca049af71..34555d3f2a9 100644--- a/unsloth_cli/_model_catalog.py+++ b/unsloth_cli/_model_catalog.py@@ -137,19 +137,17 @@ def exported_entries() -> List[ModelEntry]:if export_type == "gguf" and _gguf_export_task(path, name, base) in NON_CHAT_TASKS:continueif export_type == "gguf":- # No complete quant means every candidate is zero-byte or short a shard, which- # survives resolve_model_config() and fails only at load time. Drop it only when- # positively unloadable: a falsy result also means "could not tell", and hiding a- # real export on a transient read is worse than listing it.+ # No complete quant means every candidate is zero-byte or short a shard, which survives+ # resolve_model_config() and fails only at load time. Drop it only when positively unloadable: a+ # falsy result also means "could not tell".load_id = _preferred_complete_gguf(path)if not load_id and not _local_dir_holds_a_payload(Path(path)):continueelse:load_id = None- # Positively loadable, not merely "not torn". scan_exported_models types a- # checkpoint as "lora" on adapter_config.json alone, so an interrupted export- # leaves a config and no weights: nothing torn because there is no payload, and- # the picker offered a directory load_peft_weights raises ValueError on.+ # Positively loadable, not merely "not torn". scan_exported_models types a checkpoint as "lora" on+ # adapter_config.json alone, so an interrupted export leaves a config and no weights and the+ # picker offered a directory load_peft_weights raises ValueError on.if not _local_dir_holds_a_payload(Path(path)):continueentries.append(ModelEntry("Fine-tunes", name, export_type, load_id or path))@@ -190,14 +188,13 @@ def _reachable_snapshots(repo_path: Path, load_id: Optional[str] = None) -> Listreturn []pinned = _pinned_snapshot(repo_path, load_id)if pinned is not None:- # Returned as validated, not re-checked against `available`. inventory_scan resolves- # the snapshot it pins while cache_path keeps the configured spelling, so under a- # symlinked cache root the two name one directory and fail lexical equality, and the- # membership test would drop a good pin back onto the ref it was pinned away from.+ # Returned as validated, not re-checked against `available`: inventory_scan resolves the snapshot+ # it pins while cache_path keeps the configured spelling, so under a symlinked cache root the two+ # name one directory and fail lexical equality, and the membership test would drop a good pin.return [pinned]try:- # ValueError too: an undecodable ref raises UnicodeDecodeError, which is not an- # OSError, and uncaught it leaves _safe hiding every Downloaded row over one repo.+ # ValueError too: an undecodable ref raises UnicodeDecodeError, which is not an OSError, and+ # uncaught it leaves _safe hiding every Downloaded row over one repo.ref = (repo_path / "refs" / "main").read_text(encoding = "utf-8").strip()except (OSError, ValueError):ref = ""@@ -219,8 +216,8 @@ def _complete_quants(snapshot: Path) -> Optional[set]:complete = complete_snapshot_variants(str(snapshot))except Exception:return None- # Empty here means the check could not tell, not that nothing loads: a genuinely- # incomplete repo arrives partial and never reaches the picker. Describe what is on disk.+ # Empty here means the check could not tell, not that nothing loads: a genuinely incomplete repo+ # arrives partial and never reaches the picker.return set(complete) or None@@ -425,8 +422,8 @@ def cached_entries() -> List[ModelEntry]:# A cached embedding/CLIP repo has task None like any chat repo; can_chat is the gate.if row.get("capabilities", {}).get("can_chat") is False:continue- # A diffusion repo also carries no task, and its pipeline root has no config for- # can_chat to read, so neither gate above catches it.+ # A diffusion repo also carries no task, and its pipeline root has no config for can_chat to read,+ # so neither gate above catches it.if row.get("diffusers"):continueentries.append(ModelEntry("Downloaded", row["repo_id"], "", _cached_model_load_id(row)))@@ -484,15 +481,15 @@ def _local_dir_holds_a_payload(path: Path) -> bool:)from utils.paths.path_utils import is_appledouble_metadata- # A pipeline keeps its weights in component subdirs, so the torn test below reads an- # empty root and would call every pipeline unserviceable.+ # A pipeline keeps its weights in component subdirs, so the torn test below reads an empty root+ # and would call every pipeline unserviceable.if _is_diffusers_pipeline_dir(path):return Trueif _local_payload_is_torn(path):return False# iterdir, not glob("*.gguf"): the glob is case-sensitive on Linux and macOS while- # _is_main_gguf_filename lowercases first, so a folder holding Model.GGUF was classified- # as a GGUF model and then dropped by this gate, which decides whether it is listed.+ # _is_main_gguf_filename lowercases first, so a folder holding Model.GGUF was classified as a+ # GGUF model and then dropped by this gate.try:children = list(path.iterdir())except OSError:@@ -572,8 +569,8 @@ def local_folder_entries() -> List[ModelEntry]:) or model.partial:continueis_gguf = model.model_format == "gguf" or model.path.lower().endswith(".gguf")- # No format gate: _dir_model_format reports only "gguf" or None, so a safetensors- # checkpoint arrives as None and a "safetensors" literal dropped every non-GGUF model.+ # No format gate: _dir_model_format reports only "gguf" or None, so a safetensors checkpoint+ # arrives as None and a "safetensors" literal dropped every non-GGUF model.if _local_model_task(model) in NON_CHAT_TASKS:continue# No format gate, so embedding and CLIP exports get through; only this stops them.@@ -584,10 +581,9 @@ def local_folder_entries() -> List[ModelEntry]:if _local_is_a_diffusers_pipeline(model):continuetarget = model.load_id or model.id- # A GGUF DIRECTORY goes through detect_gguf_model, which sorts by size and takes the- # largest complete file, commonly the F16, while cached and exported rows resolve a- # Q4-class quant. Same folder, dramatically bigger load by source alone, so an OOM- # rather than a preference. Resolve it the same way here.+ # A GGUF DIRECTORY goes through detect_gguf_model, which sorts by size and takes the largest+ # complete file, commonly the F16, while cached and exported rows resolve a Q4-class quant. Same+ # folder, dramatically bigger load by source alone, so an OOM rather than a preference.if is_gguf:target = _preferred_complete_gguf(target) or targetentries.append(diff --git a/unsloth_cli/_studio_deps.py b/unsloth_cli/_studio_deps.pyindex 4438f77fb49..b236bf69c0a 100644--- a/unsloth_cli/_studio_deps.py+++ b/unsloth_cli/_studio_deps.py@@ -192,8 +192,8 @@ def _venv_site_packages(root: Path) -> List[Path]:if versioned.is_dir():return [versioned]- # Test fixtures and incomplete venvs may not have a runnable interpreter or- # version entry yet. One directory is unambiguous; several are not.+ # Test fixtures and incomplete venvs may not have a runnable interpreter or version entry yet.+ # One directory is unambiguous; several are not.candidates = sorted(root.glob("lib/python*/site-packages"))return candidates if len(candidates) == 1 else []@@ -235,9 +235,8 @@ def _distributions_in(root: Path) -> Optional[tuple[Dict[str, str], set[str]]]:if name:canonical = _canonical(name)version = dist.version or ""- # pip renames the outgoing record to a `~` sibling while it lays- # the replacement down, so a kill there can leave the backup as- # the only record. It reads as one healthy version, but pip calls+ # pip renames the outgoing record to a `~` sibling while laying the replacement down, so a kill+ # there can leave the backup as the only record: it reads as one healthy version, but pip calls# it an invalid distribution and the payload is renamed with it.if not version or canonical in found or stem.startswith("~"):conflicts.add(canonical)@@ -258,9 +257,9 @@ def _requirements_root_in(root: Path) -> Optional[Path]:reqs = path / "studio" / "backend" / "requirements"if reqs.is_dir():return reqs- # An editable install can keep studio/ only in its source checkout. Ask the- # foreign interpreter to follow its .pth/finder instead of treating the- # absent site-packages copy as an incomplete environment.+ # An editable install can keep studio/ only in its source checkout. Ask the foreign interpreter+ # to follow its .pth/finder instead of treating the absent site-packages copy as an incomplete+ # environment.probe = ("import json, pathlib, studio; ""print(json.dumps(str(pathlib.Path(studio.__file__).resolve().parent / "@@ -312,8 +311,7 @@ def install_state(extra_roots: Sequence[Path] = (), deep: bool = False) -> dict:"missing": [],"reason": "studio_install_manifest_missing",}- # The requested managed venv is the subject, even though the helper above- # came from this CLI's own tree.+ # The requested managed venv is the subject, even though the helper above came from this CLI's own tree.root = _managed_root(extra_roots) or _venv_root_for_module(module)foreign = root is not None and _resolved(root) != _resolved(Path(sys.prefix))foreign_distributions = _distributions_in(root) if foreign else None@@ -321,9 +319,9 @@ def install_state(extra_roots: Sequence[Path] = (), deep: bool = False) -> dict:installed_conflicts = foreign_distributions[1] if foreign_distributions is not None else set()req_root = _requirements_root_in(root) if foreign else Noneif foreign and (foreign_distributions is None or req_root is None):- # Never answer for the caller when the requested environment cannot be- # inspected. That can turn a torn or ambiguous managed venv into a- # healthy result merely because the external CLI has matching packages.+ # Never answer for the caller when the requested environment cannot be inspected: that can turn+ # a torn or ambiguous managed venv into a healthy result merely because the external CLI has+ # matching packages.return {"ok": False,"manifest_ok": False,@@ -343,22 +341,19 @@ def install_state(extra_roots: Sequence[Path] = (), deep: bool = False) -> dict:kwargs["installed_conflicts"] = installed_conflictsif deep and _verify_install_supports(module, "deep"):kwargs["deep"] = True- # Without that venv's site-packages the scan answers for this- # interpreter's tree. Guarded separately: a module new enough- # for `deep` may predate `scan_paths`.+ # Without that venv's site-packages the scan answers for this interpreter's tree. Guarded+ # separately: a module new enough for `deep` may predate `scan_paths`.if _verify_install_supports(module, "scan_paths"):paths = [str(path) for path in _venv_site_packages(root)]if paths:kwargs["scan_paths"] = pathsreturn module.verify_install(**kwargs)- # Off for `desktop-capabilities`: the Tauri preflight times it out at- # 10s, and a probe that overruns repairs a healthy venv. `verify-install`- # is untimed, so it opts in.+ # Off for `desktop-capabilities`: the Tauri preflight times it out at 10s and a probe that+ # overruns repairs a healthy venv. `verify-install` is untimed, so it opts in.deep_kwargs = {"deep": True} if (deep and _verify_install_supports(module, "deep")) else {}state = module.verify_install(root = root, **deep_kwargs)if foreign and not state["deps_ok"]:- # The manifest came from another venv but the dependency walk ran- # here, so it says nothing about that venv.+ # The manifest came from another venv but the dependency walk ran here, so it says nothing about that venv.state = dict(state, deps_ok = True, missing = [])state["ok"] = state["manifest_ok"]state["reason"] = None if state["ok"] else state["reason"]@@ -405,11 +400,10 @@ def _scan_paths() -> Dict[str, list]:return {"path": paths} if paths else {}-# Top-level dirs several wheels write into, so one uninstall deletes another's-# files and the survivor's RECORD describes a file nothing recreates. einx and-# torchao both ship test/conftest.py, and install_python_stack.py-# force-reinstalls torchao every update; unsloth_zoo <= 2026.8.5 shipped-# tests/ and scripts/ into the same squatted namespace.+# Top-level dirs several wheels write into, so one uninstall deletes another's files and the+# survivor's RECORD describes a file nothing recreates. einx and torchao both ship+# test/conftest.py and install_python_stack.py force-reinstalls torchao every update;+# unsloth_zoo <= 2026.8.5 shipped tests/ and scripts/ into the same squatted namespace._SHARED_NON_RUNTIME_ROOTS = frozenset(("test",@@ -426,9 +420,9 @@ _SHARED_NON_RUNTIME_ROOTS = frozenset())-# Rewritten in place by our own setup: setup.ps1/setup.sh run `npm install`-# inside the installed tree, and npm dedupes hoisted entries under-# legacy-peer-deps, shrinking the lockfile below its recorded size.+# Rewritten in place by our own setup: setup.ps1/setup.sh run `npm install` inside the installed+# tree, and npm dedupes hoisted entries under legacy-peer-deps, shrinking the lockfile below its+# recorded size._INSTALLER_REWRITTEN_NAMES = frozenset(("package-lock.json",))@@ -486,8 +480,8 @@ def _installed_distribution_groups():version = Nonegroups.setdefault(_canonical(name), []).append((dist, name, bool(version)))continue- # Wheel metadata directory names escape name separators as underscores,- # so splitting off the final version is unambiguous.+ # Wheel metadata directory names escape name separators as underscores, so splitting off the+ # final version is unambiguous.path = getattr(dist, "_path", None)stem = os.path.basename(os.fspath(path)) if path is not None else ""path_name, separator, _version = stem.removesuffix(".dist-info").rpartition("-")@@ -582,15 +576,14 @@ def damaged_installed_files(limit: int = 8) -> List[str]:entries: List[tuple] = []owners: Dict[str, int] = {}for records in _installed_distribution_groups().values():- # An unreadable record cannot be trusted to describe the package tree- # any more than a duplicated one can.+ # An unreadable record cannot be trusted to describe the package tree any more than a duplicated one can.ambiguous = len(records) > 1 or not all(readable for _dist, _name, readable in records)for dist, name, _readable in records:try:record = dist.read_text("RECORD")except Exception:- # An unreadable or absent RECORD says nothing about damage: editable- # installs and system packages legitimately have none.+ # An unreadable or absent RECORD says nothing about damage: editable installs and system+ # packages legitimately have none.continueif not record:continue@@ -599,8 +592,8 @@ def damaged_installed_files(limit: int = 8) -> List[str]:# A trailing slash is a directory entry, which has nothing to check.if not rel or rel.endswith("/"):continue- # Installer-owned metadata is rewritten in place and drifts from the- # size recorded inside itself; .pyc is regenerated from source.+ # Installer-owned metadata is rewritten in place and drifts from the size recorded inside+ # itself; .pyc is regenerated from source.if ".dist-info/" in rel or ".egg-info/" in rel or rel.endswith(".pyc"):continuetry:@@ -608,14 +601,13 @@ def damaged_installed_files(limit: int = 8) -> List[str]:except Exception:continuekey = os.path.normcase(str(target))- # Before either filter: a row we cannot verify still owns the path- # it claims, so another distribution's size is ambiguous too.+ # Before either filter: a row we cannot verify still owns the path it claims, so another+ # distribution's size is ambiguous too.owners[key] = owners.get(key, 0) + 1if ambiguous or _shared_non_runtime(rel):continue- # The size field is optional and real wheels do leave it blank. Keep- # the row anyway with an unknown size: existence is still checkable,- # and dropping the row meant a deletion went unreported.+ # The size field is optional and real wheels do leave it blank. Keep the row with an unknown+ # size: existence is still checkable, and dropping it meant a deletion went unreported.recorded: Optional[int] = Noneif len(row) >= 3 and row[2] and not _installer_rewritten(rel):try:@@ -629,15 +621,13 @@ def damaged_installed_files(limit: int = 8) -> List[str]:try:info = target.stat()except OSError:- # Multiple ownership makes the recorded SIZES ambiguous; it cannot- # explain the file being gone, so this branch runs for shared paths- # too.+ # Multiple ownership makes the recorded SIZES ambiguous; it cannot explain the file being gone,+ # so this branch runs for shared paths too.found.append(f"{name}: {rel} is missing")else:if not stat.S_ISREG(info.st_mode):- # A directory standing in for a recorded module still imports as- # something else, and on POSIX its st_size (commonly 4096) can- # sail past the shrinkage test.+ # A directory standing in for a recorded module still imports as something else, and on POSIX+ # its st_size (commonly 4096) can sail past the shrinkage test.found.append(f"{name}: {rel} is not a regular file")elif owners[key] == 1 and recorded is not None and info.st_size < recorded:found.append(f"{name}: {rel} is {info.st_size} bytes, expected {recorded}")@@ -653,10 +643,9 @@ def running_outside_managed_venv(extra_roots: Sequence[Path] = ()) -> bool:anything answered from this interpreter would then describe the wrong tree."""if not (Path(sys.prefix) / "pyvenv.cfg").is_file():- # Not a venv at all. On Colab setup.sh deliberately installs the backend- # into the system Python, whose distro-packaged RECORDs legitimately list- # files the distro never installed (PEP 627), so a file check here- # describes the distro rather than Unsloth.+ # Not a venv at all. On Colab setup.sh deliberately installs the backend into the system Python,+ # whose distro-packaged RECORDs legitimately list files the distro never installed (PEP 627), so+ # a file check here describes the distro rather than Unsloth.return Truereturn _managed_root(extra_roots) is not None@@ -672,9 +661,9 @@ def _missing_studio_packages() -> List[str]:return []-# studio.txt names distributions, ModuleNotFoundError names the import. Only-# pairs differing by more than PEP 503 normalisation need an entry, and each-# import name below is itself a real but unrelated PyPI project.+# studio.txt names distributions, ModuleNotFoundError names the import. Only pairs differing by+# more than PEP 503 normalisation need an entry, and each import name below is itself a real but+# unrelated PyPI project._IMPORT_TO_DISTRIBUTION = {"jwt": "pyjwt","docx": "python-docx",@@ -693,12 +682,11 @@ def studio_backend_imports(feature: str = "This command", *, studio_only: bool =yieldexcept ModuleNotFoundError as exc:studio_missing = _missing_studio_packages()- # The failed import may not be a studio dependency at all: `train`- # reaches torch through the same wrapper, so only offer the extra when- # it helps.+ # The failed import may not be a studio dependency at all: `train` reaches torch through the same+ # wrapper, so only offer the extra when it helps.trigger = exc.name or ""- # Match on the owning distribution, never the import: `pip install jwt`- # (or fastmcp.server) installs the wrong thing or nothing at all.+ # Match on the owning distribution, never the import: `pip install jwt` (or fastmcp.server)+ # installs the wrong thing or nothing at all.top = trigger.split(".", 1)[0]needed = _IMPORT_TO_DISTRIBUTION.get(top, top)wanted = _canonical(needed)diff --git a/unsloth_cli/_studio_runtime_gate.py b/unsloth_cli/_studio_runtime_gate.pyindex 9e9ce3eba34..b0cd6ead44b 100644--- a/unsloth_cli/_studio_runtime_gate.py+++ b/unsloth_cli/_studio_runtime_gate.py@@ -330,15 +330,10 @@ def ensure_managed_environment_is_idle(studio_home: Path) -> None:if int(process.get("ProcessId") or -1) > 0}- # The updater may itself have been entered through the managed console shim,- # so exempt verified launcher ancestors only: a managed backend that starts- # an update as its child must still block replacement.- #- # venv\Scripts\python.exe is a redirector (bpo-34977): it starts base Python- # as a child and waits, so a venv launch arrives as python.exe -> us. Our- # image is then the base interpreter while sys.executable still names the- # redirector, which identifies our direct parent as that launcher rather- # than a live consumer. Exempt that one hop only; above it must be a shim.+ # Exempt verified launcher ancestors only: a managed backend starting an update as its child must+ # still block replacement.+ # venv Scripts\python.exe is a redirector (bpo-34977), so exempt that one hop: our image is base+ # Python while sys.executable names the shim.excluded_pids = {os.getpid()}descendant_pid = os.getpid()self_executable = (process_by_pid.get(os.getpid()) or {}).get("ExecutablePath")diff --git a/unsloth_cli/_studio_stage.py b/unsloth_cli/_studio_stage.pyindex abd3cc4338c..dae50bde94d 100644--- a/unsloth_cli/_studio_stage.py+++ b/unsloth_cli/_studio_stage.py@@ -188,8 +188,7 @@ def probe_console_script(venv: Path, env: dict[str, str]) -> None:try:result = _run([str(script), "-h"], cwd = venv.parent, env = env)except OSError as exc:- # A shebang naming an interpreter that is not there fails here, not with a- # return code: the kernel refuses the exec.+ # A shebang naming a missing interpreter fails here, not with a return code: the kernel refuses the exec.raise StageError(f"staged launcher is not executable: {exc}") from excif result.returncode != 0:raise StageError(f"staged launcher failed to start: {result.stderr.strip()[-2000:]}")diff --git a/unsloth_cli/_system_dir_guard.py b/unsloth_cli/_system_dir_guard.pyindex d7067547e62..39afbddd850 100644--- a/unsloth_cli/_system_dir_guard.py+++ b/unsloth_cli/_system_dir_guard.py@@ -19,9 +19,8 @@ STUDIO_HOME against the working directory.import os as _os-# Set by Unsloth Desktop on every CLI child it owns (process.rs). Forging it-# grants nothing: anyone who can set a child's environment can set its working-# directory, and the move lands inside the caller's own account.+# Set by Unsloth Desktop on every CLI child it owns (process.rs); forging it grants nothing,+# the move lands in the caller's own account.DESKTOP_MANAGED_ENV = "UNSLOTH_DESKTOP_MANAGED"# The directory process.rs pins, so an older desktop lands in the same place.@@ -59,8 +58,7 @@ def windows_roots(roots.append(value)if roots:return roots- # No Windows installation found: keep the guard alive on SystemRoot or the- # default, never on a user-settable value.+ # No Windows installation found: keep the guard on SystemRoot or the default, never on a user-settable value.return [system_root or r"C:\Windows"]@@ -144,9 +142,8 @@ def _outside_windows(candidate, windirs, pathmod, sep):norm = _normalize(candidate, pathmod)for windir in windirs:windir_norm = _normalize(windir, pathmod)- # A root-relative candidate carries no drive, so it equals no- # drive-qualified root: "\Windows\System32\config\systemprofile" is- # SYSTEM's profile on whichever drive, so compare that spelling too.+ # A root-relative candidate carries no drive, so compare that spelling too:+ # "\Windows\System32\config\systemprofile" is SYSTEM's profile on whichever drive.for form in (windir_norm, pathmod.splitdrive(windir_norm)[1]):if not form:continue@@ -181,8 +178,7 @@ def safe_user_dir(for candidate in candidates:if not _outside_windows(candidate, windirs, pathmod, sep):continue- # USERPROFILE and ~ can name the public profile themselves, so the check- # is on the folder, not on which variable it came from.+ # USERPROFILE and ~ can name the public profile, so check the folder, not which variable it came from.if (not allow_publicand public@@ -193,11 +189,9 @@ def safe_user_dir(return None-# Commands the desktop runs that take no path from a user: Unsloth resolves its-# venv, llama.cpp, auth, pid files and logs from the Unsloth home. `update` is here-# so an older desktop can still upgrade from the tray. Everything else keeps the-# hard error. Matched whole, not on the first word, since `studio update --local-# <path>` resolves that path against the working directory.+# Commands the desktop runs that take no path from a user; `update` is here so an older desktop+# can still upgrade from the tray. Matched whole, since `studio update --local <path>`+# resolves against the working directory._STUDIO_COMMANDS = (("provision-desktop-auth",),("desktop-capabilities",),@@ -226,9 +220,8 @@ def _is_desktop_backend_launch(rest):return True-# Subcommands that take a path from the caller, by argument or by environment.-# `run` takes --model and a raw llama-server tail; `update --local` installs-# from a checkout the caller names.+# Subcommands that take a path from the caller: `run` takes --model and a raw llama-server+# tail, `update --local` a checkout._PATH_TAKING_STUDIO_COMMANDS = ("run", "update")@@ -238,8 +231,7 @@ def _carries_a_value(arg):return Trueif arg.startswith("--"):return "=" in arg- # A short option carries its value in the same token from the second- # character on: `-f.\dist` is Click's spelling of `--frontend .\dist`.+ # A short option carries its value in the same token: `-f.\dist` is Click's spelling of `--frontend .\dist`.return len(arg) > 2@@ -251,10 +243,8 @@ def _takes_a_path(rest):return Falseif rest[0] in _PATH_TAKING_STUDIO_COMMANDS:return tuple(rest) not in _STUDIO_COMMANDS- # rest[0] is the subcommand name, skipped unless it is a flag (then there is- # no subcommand). An attached value hides inside its own token, so a leading- # dash does not clear it: Click reads both `--frontend=.\dist` and the short- # `-f.\dist` as a value, and either carries a path a relocation would rebase.+ # rest[0] is the subcommand name, skipped unless it is a flag. A leading dash does not clear+ # an attached value: Click reads `--frontend=.\dist` and `-f.\dist` as values with a path.tail = rest if rest[0].startswith("-") else rest[1:]return any(_carries_a_value(arg) for arg in tail)@@ -268,21 +258,18 @@ def is_relocatable_invocation(argv, environ):args = [arg for arg in argv if arg]if not args:return False- # Click handles top-level -h/--help/--version eagerly, before the callback- # this runs from, so only `studio --help` actually arrives here.+ # Click handles top-level -h/--help/--version eagerly, so only `studio --help` arrives here.if all(arg in _HELP_FLAGS for arg in args):return Trueif args[0] != "studio":- # The marker is inherited by everything the backend spawns, so it- # authorises the desktop's studio commands and nothing else: rebasing- # `train --dataset .\data.json` under a stray one would be worse than the- # refusal it replaced.+ # The marker is inherited by everything the backend spawns, so it authorises the desktop's+ # studio commands only: rebasing `train --dataset .\data.json` under a stray one is worse+ # than refusing.return Falserest = args[1:]if environ.get(DESKTOP_MANAGED_ENV) == "1" and not _takes_a_path(rest):- # The marker covers a desktop build whose command shape this CLI does not- # know yet, but must not widen the set to path-carrying commands: `studio- # run --model .\local.gguf` from a marked shell would be rebased.+ # The marker covers desktop builds whose command shape this CLI does not know yet, but must+ # not widen to path-carrying commands like `studio run --model .\local.gguf`.return Trueif rest and all(arg in _HELP_FLAGS for arg in rest):return True@@ -291,11 +278,9 @@ def is_relocatable_invocation(argv, environ):return tuple(rest) in _STUDIO_COMMANDS-# Path overrides the caller may have written relative to the folder being left.-# Unsloth resolves them with Path.resolve(), which anchors a relative value to the-# working directory, so moving first would silently retarget them.+# Path overrides written relative to the folder being left: Unsloth resolves them with+# Path.resolve(), so moving first would silently retarget them._RELATIVE_PATH_ENV = (- # Unsloth roots: storage_roots.py."UNSLOTH_STUDIO_HOME","STUDIO_HOME","UNSLOTH_STUDIO_DOCUMENTS_HOME",@@ -303,7 +288,6 @@ _RELATIVE_PATH_ENV = ("UNSLOTH_STUDIO_SANDBOX_HOME",# `studio update` reads it, and that command relocates."STUDIO_LOCAL_REPO",- # Engine and tool locations the user may point somewhere of their own."UNSLOTH_LLAMA_CPP_PATH","UNSLOTH_LLAMA_CPP_SCRIPTS_DIR","UNSLOTH_SD_CPP_PATH",@@ -312,16 +296,14 @@ _RELATIVE_PATH_ENV = ("WHISPER_SERVER_PATH","SD_CLI_PATH","SD_SERVER_PATH",- # Model files llama-server reads straight from the environment, and Unsloth- # reads back when it sizes a launch (llama_cpp.py). The URL and HF-repo- # spellings are deliberately absent: they name no local file.+ # Model files llama-server reads from the environment and Unsloth reads back when sizing a+ # launch (llama_cpp.py). URL and HF-repo spellings are absent: they name no local file."LLAMA_ARG_MODEL","LLAMA_ARG_MMPROJ","LLAMA_ARG_MODEL_DRAFT","LLAMA_ARG_SPEC_DRAFT_MODEL",- # Read straight from the environment as a file path: the ASIC table by- # unsloth/import_fixes.py, the vLLM cache root by Unsloth when the caller set- # it themselves (storage_roots.py only fills a blank one).+ # Read straight from the environment as a file path: the ASIC table (import_fixes.py) and the+ # vLLM cache root when the caller set it (storage_roots.py fills only a blank one)."AMDGPU_ASIC_ID_TABLE_PATH","VLLM_CACHE_ROOT",# A custom ggml backend, preserved into the llama.cpp child (llama_cpp.py).@@ -337,7 +319,6 @@ _RELATIVE_PATH_ENV = ("OLLAMA_MODELS","DG_VISUAL_BIN","UNSLOTH_DG_SHIM",- # Caches."UNSLOTH_COMPILE_LOCATION","TORCHINDUCTOR_CACHE_DIR","UNSLOTH_DIFFUSION_COMPILE_CACHE_DIR",@@ -348,11 +329,10 @@ _RELATIVE_PATH_ENV = ("HF_XET_CACHE","HF_DATASETS_CACHE","HF_ASSETS_CACHE",- # The credential file: a relative value would follow the child and lose- # access to gated repos.+ # The credential file: a relative value would follow the child and lose access to gated repos."HF_TOKEN_PATH",- # Read as written, and authoritative when non-blank (storage_roots.py), so- # `unsloth studio update` would install from a different cache after a move.+ # Authoritative when non-blank (storage_roots.py), so `unsloth studio update` would install+ # from a different cache after a move."UV_CACHE_DIR","TRANSFORMERS_CACHE","SENTENCE_TRANSFORMERS_HOME",@@ -365,26 +345,20 @@ _RELATIVE_PATH_ENV = ("CUDA_ROOT",)-# Pinned when it can be, but never at the cost of the move. `studio update-# --local` is the only command that reads STUDIO_LOCAL_REPO and it takes a path,-# so it keeps the hard error; the bare update that does relocate drops the value-# before anything reads it (commands/studio.py). Refusing a System32 update over-# a stale setting nothing was going to look at would defeat the fallback this-# guard exists for, so an unresolvable one is left behind instead.+# Pinned when possible, never at the cost of the move: only `studio update --local` reads+# STUDIO_LOCAL_REPO and it keeps the hard error, so an unresolvable one is left behind rather+# than defeating the fallback._BEST_EFFORT_ENV = frozenset(("STUDIO_LOCAL_REPO",))# The most a Windows environment variable holds, terminator included._WINDOWS_ENV_VALUE_LIMIT = 32767-# The separator is Windows', not the host's: this guard only ever runs on-# Windows, and os.pathsep would split "D:\\shared" apart anywhere else.+# The separator is Windows', not the host's: os.pathsep would split "D:\shared" apart anywhere else._PATH_LIST_SEPARATOR = ";"-# Values holding several separated directories, anchored entry by entry. A-# relative PYTHONPATH entry is resolved at import time, so a move would let-# whatever sits in the new directory shadow a managed import. PATH is left out on-# purpose: refusing the whole move over one unresolvable entry in a list that long-# would cost more than it protects.+# Multi-directory values, anchored entry by entry: a relative PYTHONPATH entry is resolved at+# import time and would let the new directory shadow a managed import. PATH is left out:+# refusing the whole move over one unresolvable entry costs more than it protects._PATH_LIST_ENV = ("UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH","CUDA_RUNTIME_DLL_DIR",@@ -416,9 +390,8 @@ def pin_relative_overrides(continueif anchored is not None:if len(anchored) >= _WINDOWS_ENV_VALUE_LIMIT:- # Anchoring a value that was already near the limit can cross it,- # and a variable Windows will not accept is a failure to report- # here rather than one to discover in the next process.+ # Anchoring a value already near the limit can cross it, and a variable Windows will not+ # accept is a failure to report here.raise ValueError(f"{name} does not fit in an environment variable once it ""names its folder in full"@@ -429,8 +402,7 @@ def pin_relative_overrides(raw = environ.get(name) or ""if not raw.strip():continue- # Each entry is anchored on its own: one relative entry changes what the- # whole list means.+ # Each entry is anchored on its own: one relative entry changes what the whole list means.entries = raw.split(_PATH_LIST_SEPARATOR)anchored_entries = [_anchor_list_entry(name, e, cwd, pathmod, abspath, expandvars, expanduser)@@ -439,8 +411,6 @@ def pin_relative_overrides(if anchored_entries != entries:joined = _PATH_LIST_SEPARATOR.join(anchored_entries)if len(joined) >= _WINDOWS_ENV_VALUE_LIMIT:- # A value Windows will not accept is a failure to report here,- # not one to discover when the next process is started.raise ValueError(f"{name} does not fit in an environment variable once each ""entry names its folder in full"@@ -450,22 +420,17 @@ def pin_relative_overrides(return pinned-# Values a consumer deliberately does not read as a plain path: anchoring one-# changes what it means rather than moving a folder. Each exemption names the-# variables whose reader proves it, since the syntax is only special there and a-# directory really called "[llama]" or "%data%" is legal on Windows.+# Values a consumer does not read as a plain path: anchoring one changes its meaning. Each+# exemption names the reader that proves it, since a directory really called "[llama]" is+# legal on Windows.-# MLX_HOSTFILE holds either a filename or the host list itself, as JSON-# (`unsloth_cli/_inference.py`, `_json_rank_count_from_env`).+# MLX_HOSTFILE holds either a filename or the host list itself as JSON (_inference.py, _json_rank_count_from_env)._INLINE_JSON_ENV = frozenset(("MLX_HOSTFILE", "MLX_IBV_DEVICES"))-# Names whose readers disagree about %VAR% and $VAR: huggingface_hub calls-# expandvars on HF_HOME (and on the XDG_CACHE_HOME it defaults from), HF_HUB_CACHE-# and HF_ASSETS_CACHE, and Unsloth calls it on SENTENCE_TRANSFORMERS_HOME, but-# Unsloth's own hf_cache_settings._canonical() does not, so it would read-# %LOCALAPPDATA%\hf as a relative folder. Expanding here before deciding settles-# it: both readers then see one absolute path. Scoped to these names because a-# directory really called "%data%" is legal, and every other name is read as one.+# Readers disagree about %VAR%/$VAR: huggingface_hub expandvars HF_HOME, XDG_CACHE_HOME,+# HF_HUB_CACHE and HF_ASSETS_CACHE and Unsloth does SENTENCE_TRANSFORMERS_HOME, but+# hf_cache_settings._canonical() does not. Expanding here makes both readers see one absolute+# path._EXPANDED_ENV = frozenset(("HF_HOME",@@ -478,9 +443,8 @@ _EXPANDED_ENV = frozenset())-# The pre-quant allowlist skips a bare on/off token precisely so there is no-# "allow all" mode (`diffusion_prequant.py`); anchoring one would turn it into a-# real allowlisted directory.+# The pre-quant allowlist skips a bare on/off token so there is no "allow all" mode+# (diffusion_prequant.py); anchoring one would make it a real allowlisted directory._TOGGLE_ENV = frozenset(("UNSLOTH_ALLOW_LOCAL_PREQUANT_PATH",))_TOGGLE_TOKENS = frozenset(("1", "true", "yes", "on", "0", "false", "no", "off"))@@ -524,16 +488,15 @@ def pin_relative_sys_path(if not isinstance(entry, str):continuetry:- # The empty entry is the working directory by definition; anything- # else has to name something that is really there.+ # The empty entry is the working directory by definition; anything else has to name something really there.if entry.strip() and not exists(entry):continueanchored = _anchor_list_entry("PYTHONPATH", entry, cwd, pathmod, abspath, None, expanduser)except Exception:- # Best effort, unlike the environment: an import root this process- # already holds is not worth refusing the move over.+ # Best effort, unlike the environment: an import root this process already holds is not worth+ # refusing the move over.continueif anchored != entry:syspath[index] = anchored@@ -595,22 +558,17 @@ def _anchor("""original = value = (value or "").strip()if value.startswith("~"):- # Written out rather than skipped: only some readers call expanduser- # first, and llama_cpp.py hands UNSLOTH_LLAMA_CPP_PATH straight to Path(),- # so a move would leave it naming a folder called "~" under the new- # directory. It is what the caller meant either way.+ # Written out rather than skipped: llama_cpp.py hands UNSLOTH_LLAMA_CPP_PATH straight to+ # Path(), so a move would leave it naming a folder called "~".value = (expanduser or pathmod.expanduser)(value)if name in _EXPANDED_ENV and value:- # Written out, so the reader that expands and the one that does not land- # in the same folder. An unset variable is left exactly as written.+ # Written out, so the reader that expands and the one that does not land in the same folder.+ # An unset variable is left as written.expandvars = expandvars or _os.path.expandvarssettled = _expand_settled(value, expandvars)if settled is None:- # One pass does not settle it, so writing the result back would have- # the reader expand it a second time. What the reader does see is one- # pass: if that names a folder on its own the value is safe to leave- # alone, and if it does not, the folder it names depends on where the- # process is standing and the move has to be refused instead.+ # One pass does not settle it, so writing the result back would have the reader expand twice;+ # if one pass does not name a folder on its own, the move has to be refused.once = expandvars(value)if _is_fully_qualified(once, pathmod):return None@@ -619,15 +577,14 @@ def _anchor(if not value:return Noneif _is_fully_qualified(value, pathmod):- # Already names one folder, but still worth writing back if expanding is- # what made it name one: the reader that does not expand cannot see that.+ # Already names one folder, but write back if expanding is what made it name one: the reader+ # that does not expand cannot see that.return value if value != original else Noneif not _names_a_path(name, value):return Noneif pathmod.splitdrive(value)[0] or value.startswith(("\\", "/")):- # "D:cache" is drive D's own current directory and "\cache" the root of- # the current drive, neither of which join() knows, so ask the OS. A- # failure reaches the caller, which then declines to move at all.+ # Ask the OS: "D:cache" is drive D's current directory and "\cache" the current drive's root,+ # neither of which join() knows.return (abspath or pathmod.abspath)(value)return pathmod.join(cwd, value)@@ -647,16 +604,15 @@ def relocation_target(return Noneif home_isdir is None:home_isdir = pathmod.isdir- # A profile that has not mounted yet still has a writable parent, so makedirs- # would build an empty second one that shadows the real one when it arrives.+ # A profile that has not mounted yet still has a writable parent, so makedirs would build an+ # empty second one that shadows the real one.if not home_isdir(home):return Nonework_dir = pathmod.join(home, WORK_DIR_NAME)try:makedirs(work_dir, exist_ok = True)except OSError:- # An unwritable home is a broken profile and Unsloth must write there- # anyway, so stop now rather than failing later.+ # An unwritable home is a broken profile and Unsloth must write there anyway, so stop now.return Nonereturn work_dir@@ -671,13 +627,11 @@ def blocked_message(expanduser = None,):"""The error shown to someone who ran Unsloth from a system folder by hand."""- # allow_public here only: a person can sensibly `cd C:\Users\Public`, but- # relocating there would share one account's state with every other account.+ # allow_public here only: relocating to C:\Users\Public would share one account's state with every other account.home = safe_user_dir(environ, windir, pathmod, sep, expanduser, allow_public = True)if home:- # Quote it, or C:\Users\Jane Doe reaches Set-Location as two arguments.- # PowerShell single quotes are verbatim ('' escapes an apostrophe); cmd- # needs double quotes once extensions are off.+ # Quote it, or C:\Users\Jane Doe reaches Set-Location as two arguments. PowerShell single+ # quotes are verbatim; cmd needs double quotes once extensions are off.home_ps = "'" + home.replace("'", "''") + "'"home_cmd = '"' + home + '"'cd_lines = (@@ -732,8 +686,7 @@ def check_working_directory(try:cwd = getcwd()except OSError:- # The launch directory is gone: say so rather than name a folder they- # were never in.+ # The launch directory is gone: say so rather than name a folder they were never in.return (("Unsloth cannot determine its current folder. It may have been deleted,\n"@@ -748,36 +701,31 @@ def check_working_directory(return None, None, Falseif not relocate or not is_relocatable_invocation(argv, environ):- # `relocate = False` is the imported-as-a-library case: the command- # modules are already loaded, so a move now is too late for the roots they- # resolved at import time.+ # `relocate = False` is the imported-as-a-library case: command modules already resolved their+ # roots at import time.return blocked_message(cwd, argv, environ, windirs, pathmod, sep, expanduser), "red", Truetarget = relocation_target(environ, windirs, pathmod, sep, expanduser, makedirs, home_isdir)unpinnable = None- # The caller's own process state when the guard runs inside a host, so nothing- # stays rewritten unless the move actually happens.+ # The caller's own process state when the guard runs inside a host, so nothing stays rewritten+ # unless the move happens.environ_before = dict(environ)if syspath is None:- # Resolved here rather than inside the pinning, or the console script,- # which passes nothing, would rewrite the real sys.path with no snapshot- # to put back when the move then fails.+ # Resolved here rather than inside the pinning, or the console script (which passes nothing)+ # would rewrite the real sys.path with no snapshot to restore.import sys as _syssyspath = _sys.pathsyspath_before = list(syspath)if target is not None:try:- # Before moving, or a relative override would end up naming a folder- # under the new directory instead of theirs.+ # Before moving, or a relative override would name a folder under the new directory instead of theirs.pin_relative_overrides(environ, cwd, pathmod, abspath, expandvars, expanduser)- # This interpreter read PYTHONPATH before the guard ran and keeps the- # entries as written, resolving a relative one on every import, so it- # needs anchoring here as well as in the environment.+ # This interpreter read PYTHONPATH before the guard ran and resolves relative entries on every+ # import, so it needs anchoring here too.pin_relative_sys_path(cwd, pathmod, syspath, abspath, exists, expanduser)except Exception as error:- # An environment we cannot pin is one we must not move underneath.- # Both reasons are real: a drive with no current directory, and a list- # that no longer fits in a Windows variable once fully qualified.+ # An environment we cannot pin is one we must not move underneath: a drive with no current+ # directory, or a list that no longer fits in a Windows variable.unpinnable = errortarget = Nonemoved = False@@ -795,9 +743,8 @@ def check_working_directory(except OSError:target = Noneif target is None:- # It landed somewhere the CLI still refuses, so go back: the- # values written for the move only mean the same folder from the- # directory they were written in.+ # It landed somewhere the CLI still refuses, so go back: the values written for the move only+ # mean the same folder from where they were written.try:chdir(cwd)except OSError:@@ -812,8 +759,8 @@ def check_working_directory(if syspath_before != syspath:syspath[:] = syspath_beforeif unpinnable is not None:- # Named separately from the profile case below: blaming the user folder- # for an unpinnable override sends them looking in the wrong place.+ # Named separately from the profile case below: blaming the user folder for an unpinnable+ # override sends them looking in the wrong place.return ((f"Unsloth cannot run from {cwd}, and could not move out of it\n"@@ -825,8 +772,8 @@ def check_working_directory(True,)if target is None:- # Fail closed: nowhere usable outside the Windows tree. This text lands in- # the desktop's logs, so it describes that case, not a shell.+ # Fail closed: nowhere usable outside the Windows tree. This text lands in the desktop's logs,+ # so it describes that case, not a shell.return ((f"Unsloth cannot run from {cwd}, and no folder outside {windir} was\n"diff --git a/unsloth_cli/claude_subagent_mcp.py b/unsloth_cli/claude_subagent_mcp.pyindex 5b33429b0f9..6fc92bb8af1 100644--- a/unsloth_cli/claude_subagent_mcp.py+++ b/unsloth_cli/claude_subagent_mcp.py@@ -34,9 +34,7 @@ from unsloth_cli.commands.start import (_MAX_RESULT_CHARACTERS = 100_000_CANCEL_POLL_SECONDS = 0.1_CANCEL_GRACE_SECONDS = 2.0-# A local server that accepts the connection and then never answers leaves the-# child, and the parent waiting on it, blocked forever. Generous enough not to cut-# a long legitimate run short; 0 restores the unbounded wait.+# A server that accepts and never answers would block the child and the parent forever; 0 restores the unbounded wait._DEFAULT_TIMEOUT_SECONDS = 1800.0@@ -87,7 +85,6 @@ def _stop_child(process: subprocess.Popen) -> None:"""Stop the Claude child and any tool processes it started."""if process.poll() is not None:if os.name != "nt":- # Leader exited, but its tool processes may still be running.try:os.killpg(process.pid, signal.SIGTERM)except OSError:@@ -129,7 +126,6 @@ def _stop_child(process: subprocess.Popen) -> None:process.wait()else:if os.name != "nt":- # Leader is gone; kill any surviving group members.try:os.killpg(process.pid, signal.SIGKILL)except OSError:@@ -176,11 +172,8 @@ def run_local_agent("--output-format","json","--no-session-persistence",- # Strip human-blocking tools so the child runs unattended. Only the read-only- # child's writers bite today, since a --print child is never offered the plan- # or prompt tools; those are listed anyway so a version that starts offering- # them cannot stall the subagent. Bash is denied read-only side because plan- # mode gates it through the same local model, which is not a write barrier.+ # Strip human-blocking tools so the child runs unattended; plan/prompt tools are listed+ # pre-emptively, and Bash is denied read-only side because plan mode is not a write barrier."--disallowedTools",("AskUserQuestion,EnterPlanMode,Edit,Write,NotebookEdit,Bash"@@ -215,8 +208,7 @@ def run_local_agent(popen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUPelse:popen_kwargs["start_new_session"] = True- # Same CR/LF hazard as the Codex bridge: --append-system-prompt and the task- # both span lines, so resolve npm shims rather than spawning the .cmd raw.+ # Same CR/LF hazard as the Codex bridge: resolve npm shims rather than spawning the .cmd raw.process = subprocess.Popen(_resolved_launch_command(executable, command[1:], child_env),**popen_kwargs,@@ -366,8 +358,8 @@ def serve(cancel_event.set()def handle_shutdown(_signum: int, _frame: Any) -> None:- # Claude Code sends SIGINT (possibly repeatedly) to cancel a tool call. Only- # the first unwinds stdin; later ones must not interrupt process-tree cleanup.+ # Claude Code may send SIGINT repeatedly; only the first unwinds stdin, later ones must not+ # interrupt process-tree cleanup.first_signal = not shutdown_started.is_set()shutdown_started.set()cancel_active()diff --git a/unsloth_cli/codex_subagent_mcp.py b/unsloth_cli/codex_subagent_mcp.pyindex 16d5d6d8860..6186c14fe5e 100644--- a/unsloth_cli/codex_subagent_mcp.py+++ b/unsloth_cli/codex_subagent_mcp.py@@ -136,8 +136,8 @@ def run_local_agent(task: str, cancel_event: threading.Event | None = None) -> spopen_kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUPelse:popen_kwargs["start_new_session"] = True- # cmd.exe splits CR/LF inside `%*` and the prompt always spans lines, so- # resolve npm shims here rather than spawning the .cmd raw.+ # cmd.exe splits CR/LF inside `%*` and the prompt always spans lines, so resolve npm shims rather+ # than spawning the .cmd raw.launch_command = _resolved_launch_command(executable, command[1:], child_env)process = subprocess.Popen(launch_command, **popen_kwargs)try:diff --git a/unsloth_cli/commands/_password_prompt.py b/unsloth_cli/commands/_password_prompt.pyindex 55f50acbf1f..7ec0e042eef 100644--- a/unsloth_cli/commands/_password_prompt.py+++ b/unsloth_cli/commands/_password_prompt.py@@ -17,12 +17,11 @@ import osimport sysfrom typing import Callable, TextIO-# Keep in sync with studio/backend/models/auth.py ChangePasswordRequest-# (new_password min_length) and studio/backend/auth/storage.py.+# Keep in sync with studio/backend/models/auth.py ChangePasswordRequest and auth/storage.py.+# The bound is `new_password` min_length.MIN_PASSWORD_LENGTH = 8-# Env var that supplies the initial admin password non-interactively (mirror in-# studio/backend/auth/terminal_prompt.py). Keep the name in sync.+# Mirror of studio/backend/auth/terminal_prompt.py; keep the name in sync.SUPPLIED_PASSWORD_ENV = "UNSLOTH_STUDIO_PASSWORD"_BACKSPACE_CHARS = ("\x7f", "\x08")@@ -57,7 +56,7 @@ class _RestoreTtyOnSignals:continuetry:self._previous.append((sig, signal.signal(sig, _restore_and_reraise)))- except (ValueError, OSError): # non-main thread / unsupported+ except (ValueError, OSError):passreturn self@@ -82,32 +81,28 @@ def _read_masked_posix(prompt: str, out: TextIO) -> str:chars: list[str] = []try:with _RestoreTtyOnSignals(fd, old_attrs):- # cbreak + ISIG off (mirrors terminal_prompt.py): with ISIG on,- # Ctrl-Z would suspend mid-read and leave the shell no-echo before- # the finally restores it. Ctrl-C/Ctrl-Z arrive as \x03/\x1a here.+ # cbreak + ISIG off (mirrors terminal_prompt.py): with ISIG on, Ctrl-Z suspends mid-read and leaves+ # the shell no-echo.tty.setcbreak(fd)new_attrs = termios.tcgetattr(fd)new_attrs[3] &= ~termios.ISIGtermios.tcsetattr(fd, termios.TCSADRAIN, new_attrs)- # Decode byte-at-a-time with errors="replace" (mirrors- # terminal_prompt.py): text-mode read(1) can raise UnicodeDecodeError- # on a pasted non-UTF-8 password or yield a lone surrogate that later- # crashes pbkdf2. os.read + incremental decoder maps bad bytes to- # U+FFFD and continues.+ # os.read + incremental decoder with errors="replace": text-mode read(1) can raise or yield a lone+ # surrogate that later crashes pbkdf2.+ # It raises UnicodeDecodeError.decoder = codecs.getincrementaldecoder(sys.stdin.encoding or "utf-8")("replace")submitted = Falsewhile not submitted:raw = os.read(fd, 1)if not raw: # stream ended mid-line: abort, don't submitraise EOFError- # One byte can complete >1 char, so iterate over the decoder's output.for ch in decoder.decode(raw):if ch in _SUBMIT_CHARS:submitted = Truebreakif ch == "\x03": # Ctrl-C (ISIG off: surfaces as a char)raise KeyboardInterrupt- if ch in ("\x04", "\x1a"): # Ctrl-D / Ctrl-Z+ if ch in ("\x04", "\x1a"):if not chars:raise EOFErrorcontinue@@ -117,7 +112,7 @@ def _read_masked_posix(prompt: str, out: TextIO) -> str:out.write("\b \b")out.flush()continue- if ch < " ": # other control characters+ if ch < " ":continuechars.append(ch)out.write("*")@@ -142,11 +137,11 @@ def _read_masked_windows(prompt: str, out: TextIO) -> str:breakif ch == "\x03": # Ctrl-C: getwch swallows the signal, re-raiseraise KeyboardInterrupt- if ch in ("\x04", "\x1a"): # Ctrl-D / Ctrl-Z+ if ch in ("\x04", "\x1a"):if not chars:raise EOFErrorcontinue- if ch in ("\x00", "\xe0"): # function/arrow key: swallow the code+ if ch in ("\x00", "\xe0"):msvcrt.getwch()continueif ch in _BACKSPACE_CHARS:diff --git a/unsloth_cli/commands/chat.py b/unsloth_cli/commands/chat.pyindex f9da50c2d36..2f1bc0866dc 100644--- a/unsloth_cli/commands/chat.py+++ b/unsloth_cli/commands/chat.py@@ -31,10 +31,8 @@ _HELP = (def _you_prompt(colors: bool) -> str:- # The prompt must go through input(), not a separate print — readline- # redraws erase anything they didn't draw, eating the label. GNU readline- # wants colors wrapped in \001/\002; libedit (macOS) prints those- # literally, so it gets raw ANSI.+ # Must go through input(): readline redraws erase text they did not draw. GNU readline wants+ # \001/\002 around colors; libedit (macOS) prints those literally.try:import readlineexcept ImportError:@@ -66,7 +64,6 @@ def _compare_blocked_reason(model_config) -> Optional[str]:def _get_base_load_in_4bit(model_config) -> bool:"""Determine load_in_4bit for base model based on tuned adapter precision."""if not model_config.is_lora or not model_config.path:- # Fallback to default if not a LoRA or no pathreturn Truetry:@@ -86,7 +83,6 @@ def _get_base_load_in_4bit(model_config) -> bool:elif training_method == "qlora":return Trueelif not training_method:- # Fallback: check base model name for -bnb-4bit suffixif model_config.base_model and "-bnb-4bit" not in model_config.base_model.lower():return Falsereturn True@@ -96,9 +92,8 @@ def _get_base_load_in_4bit(model_config) -> bool:def _compare_needs_second_model() -> bool:- # MLX can't toggle the adapter off, so compare loads the base separately.- # detect_hardware() would print into the chat (and import torch), so- # probe its MLX condition quietly: Apple Silicon with mlx installed.+ # MLX cannot toggle the adapter off, so compare loads the base separately; probe MLX quietly since+ # detect_hardware() prints into the chat and imports torch.try:from studio.backend.utils.hardware import hardware as hw@@ -303,9 +298,7 @@ def chat(compare_mode = comparemessages = []- # Compare's base column: server mode keeps the tuned model remote and- # loads the base locally; local MLX (no adapter toggle) does the same;- # local CUDA just toggles the adapter on the one loaded model.+ # Compare's base column: server mode and local MLX load the base separately; local CUDA just toggles the adapter.dual_compare = compare_blocked is None and (server_mode or _compare_needs_second_model())base_backend = None@@ -328,8 +321,7 @@ def chat(markup = False,)try:- # Use the same precision as the tuned model for fair comparison- base_load_opts = dict(load_opts) # Copy original options+ base_load_opts = dict(load_opts)base_load_opts["load_in_4bit"] = _get_base_load_in_4bit(model_config)base_backend = load_chat_backend(base_id, fresh_backend = True, **base_load_opts)except Exception as exc:@@ -446,7 +438,6 @@ def chat(render_columns("base", base_text, f"{name} (tuned)", tuned_text, console = console)- # History continues as the tuned model; base is just the reference.answer = tuned_textelse:if should_print:diff --git a/unsloth_cli/commands/inference.py b/unsloth_cli/commands/inference.pyindex 1d44a795b11..08797f08a88 100644--- a/unsloth_cli/commands/inference.py+++ b/unsloth_cli/commands/inference.py@@ -98,9 +98,7 @@ def inference()raise typer.Exit(code = 1)- # A running Unsloth server keeps the model warm between runs. Under- # mlx.launch, every rank must enter the local MLX path instead of rank 0- # alone talking to a server.+ # Under mlx.launch every rank must enter the local MLX path, not just rank 0 talking to a warm server.load_opts = dict(hf_token = hf_token,max_seq_length = max_seq_length,diff --git a/unsloth_cli/commands/train.py b/unsloth_cli/commands/train.pyindex 4bff2f28764..4925b30e41a 100644--- a/unsloth_cli/commands/train.py+++ b/unsloth_cli/commands/train.py@@ -78,8 +78,7 @@ def train(config_overrides = config_overrides or {}cfg.apply_overrides(**config_overrides)- # CLI/env tokens take precedence; guard against unresolved typer.Option- # (decorator interaction)+ # CLI/env tokens take precedence; guard against unresolved typer.Option.from typer.models import OptionInfoif isinstance(hf_token, OptionInfo):@@ -105,7 +104,6 @@ def train(typer.echo("Error: provide --dataset or --local-dataset (or via --config)", err = True)raise typer.Exit(code = 2)- # A LoRA adapter dir has adapter_config.jsonmodel_path = Path(cfg.model) if cfg.model else Nonemodel_is_lora = (model_path and model_path.is_dir() and (model_path / "adapter_config.json").exists()@@ -122,7 +120,6 @@ def train(trainer = _create_cli_trainer(cfg.model, hf_token)- # Load model (trainer.is_vlm is set after this)if not trainer.load_model(model_name = cfg.model,max_seq_length = cfg.training.max_seq_length,@@ -150,7 +147,7 @@ def train(ds, eval_ds = resulttraining_kwargs = cfg.training_kwargs()- training_kwargs["wandb_token"] = wandb_token # CLI/env takes precedence+ training_kwargs["wandb_token"] = wandb_tokenstarted = trainer.start_training(dataset = ds, eval_dataset = eval_ds, **training_kwargs)if not started:diff --git a/unsloth_cli/options.py b/unsloth_cli/options.pyindex 7a0433e87c1..8a6d4abe75d 100644--- a/unsloth_cli/options.py+++ b/unsloth_cli/options.py@@ -65,7 +65,6 @@ def _collect_config_fields(config_class: type[BaseModel]) -> list[tuple[str, Anyfor name, field_info in config_class.model_fields.items():annotation = field_info.annotation- # Recurse into nested modelsif isinstance(annotation, type) and issubclass(annotation, BaseModel):for nested_name, nested_field in annotation.model_fields.items():if nested_name in seen_names:@@ -95,11 +94,9 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:original_params = list(sig.parameters.values())original_param_names = {p.name for p in original_params}- # Build new parameters: config fields first, then original paramsnew_params = []for field_name, field_info in fields:- # Skip fields already defined in function signature (e.g., with envvar)if field_name in original_param_names:continueannotation = field_info.annotation@@ -107,7 +104,6 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:help_text = field_info.description or ""if _is_list_type(annotation):- # Repeatable option: --flag a --flag b -> ["a", "b"]default = typer.Option(None, flag_name, help = help_text)param = inspect.Parameter(field_name,@@ -141,7 +137,6 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:)new_params.append(param)- # Add original params, excluding config_overrides (will be injected)for param in original_params:if param.name != "config_overrides":new_params.append(param)@@ -155,7 +150,6 @@ def add_options_from_config(config_class: type[BaseModel]) -> Callable:if key in field_names:if kwargs[key] is not None:config_overrides[key] = kwargs[key]- # Only delete if not an explicitly declared parameterif key not in original_param_names:del kwargs[key]diff --git a/unsloth_cli/pi_subagent.ts b/unsloth_cli/pi_subagent.tsindex 9714bf86eb8..2406e496027 100644--- a/unsloth_cli/pi_subagent.ts+++ b/unsloth_cli/pi_subagent.ts@@ -214,8 +214,8 @@ async function runLocalAgent(}if (event.type !== "message_end") return;const message = event.message;- // Pi reports model/API failures as message_end events while still- // exiting 0, so the exit status alone cannot surface them.+ // Pi reports model/API failures as message_end events while still exiting 0, so the exit status+ // alone cannot surface them.if (message?.stopReason === "error" || message?.stopReason === "aborted") {childError =(typeof message.errorMessage === "string" && message.errorMessage) ||diff --git a/unsloth_cli/tests/test_claude_plan_gate.py b/unsloth_cli/tests/test_claude_plan_gate.pyindex 85336c8dfe0..eed54c7bc7c 100644--- a/unsloth_cli/tests/test_claude_plan_gate.py+++ b/unsloth_cli/tests/test_claude_plan_gate.py@@ -106,9 +106,8 @@ def test_plugin_still_writes_the_mcp_server_and_skill(tmp_path):def test_wsl_run_clears_a_gate_left_by_an_earlier_windows_run(tmp_path, monkeypatch):- # The plugin dir survives across runs when persisted, so a gate written by a- # Windows run would otherwise be shipped into the distro with an interpreter- # path it cannot execute.+ # The plugin dir survives across runs when persisted, so a gate written by a Windows run would+ # otherwise be shipped into the distro with an interpreter path it cannot execute.plugin = _plugin(tmp_path)gate = plugin / "hooks" / "plan_gate.py"hooks = plugin / "hooks" / "hooks.json"@@ -123,9 +122,9 @@ def test_wsl_run_clears_a_gate_left_by_an_earlier_windows_run(tmp_path, monkeypadef test_hook_command_survives_a_missing_gate_and_a_path_with_spaces(tmp_path):- # Handing the path straight to the interpreter makes a missing gate exit 2,- # which Claude treats as a blocking error: the editing tool would then be- # denied in every mode, not just plan. Going through runpy makes it exit 1.+ # Handing the path straight to the interpreter makes a missing gate exit 2, which Claude treats as+ # a blocking error: the editing tool would then be denied in every mode, not just plan. Going+ # through runpy makes it exit 1.plugin = _plugin(tmp_path / "dir with space")hook = json.loads((plugin / "hooks" / "hooks.json").read_text())command = hook["hooks"]["PreToolUse"][0]["hooks"][0]["command"]@@ -157,9 +156,9 @@ def test_hook_command_survives_a_missing_gate_and_a_path_with_spaces(tmp_path):@pytest.mark.parametrize("hostile", ["sub$(echo X)", "tick`echo X`", "var$HOME", "pct%TEMP%pct"])def test_gate_survives_shell_metacharacters_in_its_path(tmp_path, hostile):- # The hook command is run by a shell. A temp root holding these expands under- # sh (or cmd, for %VAR%) before Python sees the path, so the gate is not found- # and exits 1, which fails open and silently drops the routing message.+ # The hook command is run by a shell. A temp root holding these expands under sh (or cmd, for+ # %VAR%) before Python sees the path, so the gate is not found and exits 1, which fails open and+ # silently drops the routing message.plugin = _plugin(tmp_path / hostile)command = json.loads((plugin / "hooks" / "hooks.json").read_text())["hooks"]["PreToolUse"][0]["hooks"diff --git a/unsloth_cli/tests/test_claude_subagent_mcp.py b/unsloth_cli/tests/test_claude_subagent_mcp.pyindex 441fd359e47..3bbb204dc36 100644--- a/unsloth_cli/tests/test_claude_subagent_mcp.py+++ b/unsloth_cli/tests/test_claude_subagent_mcp.py@@ -348,8 +348,8 @@ def test_read_only_local_child_uses_plan_mode(monkeypatch, tmp_path):assert command[command.index("--permission-mode") + 1] == "plan"disallowed = command[command.index("--disallowedTools") + 1]assert disallowed == "AskUserQuestion,EnterPlanMode,Edit,Write,NotebookEdit,Bash"- # Bash matters: plan mode routes it through a classifier served by this same- # local model, so without the deny a "read-only" child can still write files.+ # Bash matters: plan mode routes it through a classifier served by this same local model, so+ # without the deny a "read-only" child can still write files.prompt = command[command.index("--append-system-prompt") + 1]assert "read-only local coding subagent" in prompt@@ -481,8 +481,8 @@ def test_result_parser_accepts_diagnostics_before_json():def test_child_is_stopped_when_it_produces_nothing_before_the_deadline(monkeypatch, tmp_path):- # A local server that accepts and never answers used to block the child, and- # the parent waiting on it, indefinitely. Measured past 400s before this.+ # A local server that accepts and never answers used to block the child, and the parent waiting on+ # it, indefinitely. Measured past 400s before this.monkeypatch.setenv("UNSLOTH_CLAUDE_SUBAGENT_TIMEOUT", "0.3")_stub_env(monkeypatch, tmp_path)stopped = []@@ -515,8 +515,8 @@ def test_timeout_can_be_disabled(monkeypatch):def test_local_child_is_spawned_through_the_shim_resolver(monkeypatch, tmp_path):- # Pins the wiring: a Windows .cmd must reach the npm parser, not Popen (#9167).- # The parser behaviour itself is covered in test_start.py.+ # Pins the wiring: a Windows .cmd must reach the npm parser, not Popen (#9167). The parser+ # behaviour itself is covered in test_start.py._stub_env(monkeypatch, tmp_path)monkeypatch.setattr(bridge.shutil, "which", lambda _: r"C:\\nodejs\\claude.cmd")captured = {}diff --git a/unsloth_cli/tests/test_inference_chat.py b/unsloth_cli/tests/test_inference_chat.pyindex bd962d3842a..7c7b1824df9 100644--- a/unsloth_cli/tests/test_inference_chat.py+++ b/unsloth_cli/tests/test_inference_chat.py@@ -117,8 +117,8 @@ def test_visible_text_holds_unclosed_think():def test_visible_text_holds_partial_think_prefix():- # Streams are cumulative, so the opening tag can arrive as "<", "<thi",- # then "<think>". Hold possible tag prefixes until they are disambiguated.+ # Streams are cumulative, so the opening tag can arrive as "<", "<thi", then "<think>". Hold+ # possible tag prefixes until they are disambiguated.assert visible_text("<", show_thinking = False) == ""assert visible_text("<thi", show_thinking = False) == ""assert visible_text("done.<thi", show_thinking = False) == "done."@@ -290,8 +290,7 @@ def test_you_prompt_matches_readline_backend(monkeypatch):assert chatmod._you_prompt(colors = True) == "\n\x1b[1;36mYou: \x1b[0m"assert chatmod._you_prompt(colors = False) == "\nYou: "- # Windows: no readline module at all; the console's own line editing- # handles backspace, so plain ANSI color (no markers) is safe.+ # Windows: no readline module at all; the console's own line editing handles backspace, so plain ANSI color is safe.monkeypatch.setitem(sys.modules, "readline", None)assert chatmod._you_prompt(colors = True) == "\n\x1b[1;36mYou: \x1b[0m"assert chatmod._you_prompt(colors = False) == "\nYou: "@@ -445,13 +444,13 @@ def test_catalog_cached_entries_filter_non_chat_rows(monkeypatch, tmp_path):},# An embedding/CLIP repo carries task None like any chat repo; can_chat separates them.{"repo_id": "org/Embedder", "task": None, "capabilities": {"can_chat": False}},- # An untrusted diffusion repo carries no task either, and its pipeline root has no- # config for can_chat to read, so only its own flag keeps it out of chat.+ # An untrusted diffusion repo carries no task either, and its pipeline root has no config for+ # can_chat to read, so only its own flag keeps it out of chat.{"repo_id": "org/Sdxl", "task": None, "diffusers": True},]- # The real variant lister, not a stub: it decides these labels and picks the load target,- # so a stub tests the plumbing and none of the answer. Pulls neither torch nor fastapi, and- # syspath_prepend is undone after the test, so the suite keeps its no-server-import property.+ # The real variant lister, not a stub: it decides these labels and picks the load target, so a stub+ # tests the plumbing and none of the answer. Pulls neither torch nor fastapi, and syspath_prepend+ # is undone after the test.monkeypatch.syspath_prepend(str(_REPO_ROOT / "studio" / "backend"))monkeypatch.setattr(cat, "_cached_catalog_rows", lambda: (gguf_rows, model_rows))@@ -860,8 +859,8 @@ def test_find_studio_server_none_when_not_running(monkeypatch):def test_find_studio_server_prefers_ipv4_loopback_for_localhost(monkeypatch):- # localhost resolving ::1-first must not hide an Unsloth bound to 127.0.0.1:- # discovery tries each loopback address and returns the one that answers.+ # localhost resolving ::1-first must not hide an Unsloth bound to 127.0.0.1: discovery tries each+ # loopback address and returns the one that answers.import socketimport urllib.request@@ -2319,8 +2318,8 @@ def test_catalog_drops_an_export_with_no_loadable_payload(monkeypatch, tmp_path)(whole / "adapter_model.safetensors").write_bytes(b"\0" * 2048)monkeypatch.setattr(cat, "_path_can_chat", lambda *a, **k: None)- # Warm the real hub.* chain BEFORE shadowing utils.models: _local_dir_holds_a_payload- # imports through it, and a stub package would break that import rather than the test.+ # Warm the real hub.* chain BEFORE shadowing utils.models: _local_dir_holds_a_payload imports+ # through it, and a stub package would break that import rather than the test.assert cat._local_dir_holds_a_payload(whole) is Trueassert cat._local_dir_holds_a_payload(broken) is False@@ -2446,8 +2445,8 @@ QUANT_LAYOUTS = [([("Q4_K_M/Tiny-Q4_K_M.gguf", 16), ("Q8_0/Tiny-Q8_0.gguf", 16)], "Q4_K_M, Q8_0"),# A split quant is ONE thing to pick. The glob listed every shard as its own label.([(f"Tiny-Q4_K_M-0000{n}-of-00003.gguf", 16) for n in (1, 2, 3)], "Q4_K_M"),- # The case the host decides: fnmatch normcases on Windows and not on Linux or macOS, so- # "*.gguf" found this file on one platform only and the cache read differently per machine.+ # The case the host decides: fnmatch normcases on Windows and not on Linux or macOS, so "*.gguf"+ # found this file on one platform only and the cache read differently per machine.([("Tiny-Q8_0.GGUF", 16)], "Q8_0"),([("Tiny-Q4_K_M.gguf", 16), ("mmproj-F16.gguf", 16)], "Q4_K_M"),]diff --git a/unsloth_cli/tests/test_installer_download_markers.py b/unsloth_cli/tests/test_installer_download_markers.pyindex 904bd98b1e8..b5159ed431f 100644--- a/unsloth_cli/tests/test_installer_download_markers.py+++ b/unsloth_cli/tests/test_installer_download_markers.py@@ -29,8 +29,8 @@ _INSTALL_PS1 = _REPO_ROOT / "install.ps1"_THRESHOLD = 52428800 # 50 MiB, the shipped default in both installers.# Shaped like real `uv pip install` output, but the two nvidia sizes are picked to bracket-# _THRESHOLD rather than to match those wheels (both are in fact far larger), so moving the-# default in either direction changes what is marked. transformers lands unannounced.+# _THRESHOLD rather than to match those wheels, so moving the default in either direction changes+# what is marked. transformers lands unannounced._UV_OUTPUT = """Resolved 38 packages in 2.60sDownloading torch (2.4GiB)Downloading nvidia-cudnn-cu12 (674.0MiB)@@ -47,8 +47,8 @@ Installed 38 packages in 9.20s+ torch==2.10.0+cu126"""-# uv does not repeat a completion, but a repeat must not close a download twice: the app-# would see an end for something it never saw start.+# uv does not repeat a completion, but a repeat must not close a download twice: the app would see+# an end for something it never saw start._UV_REPEATED_COMPLETION = """Downloading torch (2.4GiB)Downloaded torchDownloaded torch@@ -171,9 +171,9 @@ def test_the_exit_code_survives_the_added_pipe(exit_code):@pytest.mark.parametrize("verbose", [False, True])def test_a_host_without_awk_still_installs(verbose):- # install.sh supports minimal images that ship no awk, and this pipe now carries every- # install command, so losing awk must cost the markers and nothing else. Without the- # fallback the pipeline closes and the child dies of SIGPIPE, reporting exit 141.+ # install.sh supports minimal images that ship no awk, and this pipe now carries every install+ # command, so losing awk must cost the markers and nothing else. Without the fallback the pipeline+ # closes and the child dies of SIGPIPE, reporting exit 141.with tempfile.TemporaryDirectory() as tmp:stub = Path(tmp) / "bin"stub.mkdir()@@ -187,14 +187,14 @@ def test_a_host_without_awk_still_installs(verbose):assert (ok_rc, rc) == ("0", "42"), "a missing awk turned into SIGPIPE"assert markers == [] and ok_markers == [], "markers cannot be produced without awk"assert _UV_OUTPUT.strip() in shown, "the failure path lost the child's output"- # The sink still has to differ by arm: quiet holds output in the log until something- # fails, verbose passes it straight through.+ # The sink still has to differ by arm: quiet holds output in the log until something fails, verbose+ # passes it straight through.assert (_UV_OUTPUT.strip() in ok_shown) is verbosedef test_a_marker_arrives_while_its_download_is_still_running():- # Every other test reads output after the child exits, so a buffered marker still- # shows up -- just too late. Verbose is the arm with the block-buffering redactor.+ # Every other test reads output after the child exits, so a buffered marker still shows up, just+ # too late. Verbose is the arm with the block-buffering redactor.proc = subprocess.Popen(["/bin/sh", "-c", _sh_harness('printf "Downloading torch (2.4GiB)\\n"; sleep 30')],stdout = subprocess.DEVNULL,@@ -282,8 +282,8 @@ def test_both_installers_emit_the_same_markers(options):def test_both_installers_ship_the_same_threshold():- # Execution only brackets the default between two fixture sizes; the literal pins it- # exactly, and pins the two installers to each other so they cannot drift apart.+ # Execution only brackets the default between two fixture sizes; the literal pins it exactly, and+ # pins the two installers to each other so they cannot drift apart.assert _extract_default() == f': "${{UNSLOTH_DL_MARKER_MIN_BYTES:={_THRESHOLD}}}"'ps1 = _INSTALL_PS1.read_text(encoding = "utf-8")assert f"$script:UvDownloadMarkerMinBytes = {_THRESHOLD}" in ps1diff --git a/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py b/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.pyindex 931739142bc..b66bb5d7a07 100644--- a/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py+++ b/unsloth_cli/tests/test_studio_refresh_shortcuts_installer_source.py@@ -270,8 +270,8 @@ def test_a_windows_tempfile_failure_skips_instead_of_aborting(monkeypatch, tmp_pcommand whose real work is done."""studio = _studio()- # Captured BEFORE any patching: studio.tempfile is the tempfile module itself, so- # patching through it replaces the global.+ # Captured BEFORE any patching: studio.tempfile is the tempfile module itself, so patching through+ # it replaces the global.real_mkstemp = tempfile.mkstempcalls = []@@ -321,8 +321,8 @@ def test_a_windows_tempfile_failure_skips_instead_of_aborting(monkeypatch, tmp_pmonkeypatch.setattr(studio.os, "fdopen", lambda fd, *a, **k: _BadHandle(fd))studio._run_fetched_installer_ps1(b"x", ["--shortcuts-only"], ["powershell.exe"], {})assert not Path(made["path"]).exists(), "the temp script must not be left behind"- # Asserted directly so the descriptor leak is caught on Linux too, rather than- # only surfacing as an unlink failure on Windows.+ # Asserted directly so the descriptor leak is caught on Linux too, rather than only surfacing as an+ # unlink failure on Windows.with pytest.raises(OSError):os.fstat(made["fd"])@@ -367,8 +367,8 @@ def test_a_local_installer_that_cannot_be_launched_falls_back_to_the_network(moncheckout.mkdir()(checkout / "install.sh").write_text("#!/bin/sh\n")monkeypatch.setenv("STUDIO_LOCAL_REPO", str(checkout))- # The tests run from a clone, so _PACKAGE_ROOT would otherwise supply a second- # usable installer and the network would never be the fallback under test.+ # The tests run from a clone, so _PACKAGE_ROOT would otherwise supply a second usable installer and+ # the network would never be the fallback under test.monkeypatch.setattr(studio, "_PACKAGE_ROOT", tmp_path / "no-checkout-here")monkeypatch.setattr(studio.platform, "system", lambda: "Linux")diff --git a/unsloth_cli/tests/test_studio_run_parallel_flag.py b/unsloth_cli/tests/test_studio_run_parallel_flag.pyindex 558bb6203a2..d6e177b5333 100644--- a/unsloth_cli/tests/test_studio_run_parallel_flag.py+++ b/unsloth_cli/tests/test_studio_run_parallel_flag.py@@ -125,10 +125,8 @@ def test_typer_parallel_aliases_are_subset_of_backend_denylist():import inspectimport importlib.util- # Load llama_server_args.py directly so the test doesn't need the- # backend's full runtime chain (fastapi / structlog / loggers /- # utils.hardware) installed -- the invariant is just about the- # _DENYLIST_GROUPS tuple.+ # Load llama_server_args.py directly so the test does not need the backend's full runtime chain+ # installed; the invariant is just about the _DENYLIST_GROUPS tuple.lsa_path = (Path(__file__).resolve().parents[2]/ "studio"@@ -156,14 +154,13 @@ def test_typer_parallel_aliases_are_subset_of_backend_denylist():)-# test_in_venv_path_passes_parallel_to_run_server (below) is the runtime-# equivalent of the retired source-text guard for hardcoded-# `llama_parallel_slots = 4`.+# test_in_venv_path_passes_parallel_to_run_server (below) is the runtime equivalent of the retired+# source-text guard for hardcoded `llama_parallel_slots = 4`.-# Re-exec arg-builder coverage. run() re-execs into the studio venv-# (execvp on POSIX, Popen on Windows). Without explicit forwarding the-# child reverts to typer defaults and silently drops the user's value.+# Re-exec arg-builder coverage. run() re-execs into the studio venv (execvp on POSIX, Popen on+# Windows), and without explicit forwarding the child reverts to typer defaults and silently drops+# the user's value.class _ExecCaptured(SystemExit):@@ -685,8 +682,8 @@ def test_reexec_mixed_parallel_with_passthrough(monkeypatch):"""--parallel + llama-server pass-through flags must all reach the child."""result, captured = _invoke_run(monkeypatch,- # --top-k is now a first-class sampling flag (routed via UNSLOTH_SAMPLING_*), so use- # --seed / --temp here, which remain genuine llama-server pass-through flags.+ # --top-k is now a first-class sampling flag (routed via UNSLOTH_SAMPLING_*), so use --seed /+ # --temp here, which remain genuine llama-server pass-through flags._BASE + ["--parallel", "8", "--seed", "42", "--temp", "0.7"],)assert len(captured) == 1@@ -735,9 +732,8 @@ def test_reexec_forwards_load_in_4bit_in_both_directions(monkeypatch, user_flag,assert other_polarity not in argv, f"unexpected {other_polarity} in child argv; got {argv}"-# Runtime check: fake sys.prefix into the studio venv to bypass-# re-exec, then assert run_server receives --parallel as-# llama_parallel_slots.+# Runtime check: fake sys.prefix into the studio venv to bypass re-exec, then assert run_server+# receives --parallel as llama_parallel_slots.class _RunServerCaptured(SystemExit):@@ -840,9 +836,8 @@ def test_in_venv_path_passes_parallel_to_run_server(run_server(llama_parallel_slots=N), not the old hardcoded 4."""studio_mod = _load_run_command()- # A real directory, not /fake: the launch gate creates STUDIO_HOME and- # locks inside it, so an unwritable home now aborts the run before- # run_server is ever reached and the flag under test goes unchecked.+ # A real directory, not /fake: the launch gate creates STUDIO_HOME and locks inside it, so an+ # unwritable home aborts the run before run_server is reached.fake_venv = tmp_path / "studio" / "venv" / "unsloth_studio"monkeypatch.setattr(sys, "prefix", str(fake_venv))# Pin STUDIO_HOME so sys.prefix.startswith() picks the in-venv branch.@@ -867,9 +862,9 @@ def test_in_venv_path_passes_parallel_to_run_server()fake_backend_run.run_server = fake_run_serverfake_backend_run._resolve_external_ip = lambda: "127.0.0.1"- # run() loads the backend via _load_run_module() (by file path), which- # ignores a sys.modules mock with no matching __file__; inject it as the- # cached run module so the stubbed run_server is used.+ # run() loads the backend via _load_run_module() (by file path), which ignores a sys.modules mock+ # with no matching __file__; inject it as the cached run module so the stubbed run_server is+ # used.monkeypatch.setattr(studio_mod, "_RUN_MODULE", fake_backend_run)import typer as _typer@@ -937,9 +932,8 @@ def test_in_venv_path_passes_api_only_to_run_server("""In-venv path must forward --api-only to run_server(api_only=...)."""studio_mod = _load_run_command()- # A real directory, not /fake: the launch gate creates STUDIO_HOME and- # locks inside it, so an unwritable home now aborts the run before- # run_server is ever reached and the flag under test goes unchecked.+ # A real directory, not /fake: the launch gate creates STUDIO_HOME and locks inside it, so an+ # unwritable home aborts the run before run_server is reached.fake_venv = tmp_path / "studio" / "venv" / "unsloth_studio"monkeypatch.setattr(sys, "prefix", str(fake_venv))monkeypatch.setattr(studio_mod, "STUDIO_HOME", fake_venv.parent)diff --git a/unsloth_cli/tests/test_studio_run_short_alias_clashes.py b/unsloth_cli/tests/test_studio_run_short_alias_clashes.pyindex 4234eb75f40..3065040be24 100644--- a/unsloth_cli/tests/test_studio_run_short_alias_clashes.py+++ b/unsloth_cli/tests/test_studio_run_short_alias_clashes.py@@ -126,8 +126,7 @@ def _invoke(monkeypatch, args):return captured-# (short_flag, value, llama-server long name). All were silently-# mis-parsed pre-cleanup.+# (short_flag, value, llama-server long name). All were silently mis-parsed pre-cleanup._PREVIOUSLY_BROKEN = [("-fa", None, "--flash-attn"),("-fit", None, "--fit"),@@ -174,9 +173,9 @@ def test_dash_hf_documented_alias_still_works(monkeypatch):assert argv[argv.index("--gguf-variant") + 1] == "UD-Q4_K_XL", argv-# Legacy `-m` / `-hfr` / `-f` were typer aliases pre-PR. The-# preprocessor promotes EXACT matches back to their typer params and-# leaves clustered tokens (`-mg`, `-fa`, ...) in the pass-through tail.+# Legacy `-m` / `-hfr` / `-f` were typer aliases pre-PR. The preprocessor promotes EXACT matches+# back to their typer params and leaves clustered tokens (`-mg`, `-fa`, ...) in the pass-through+# tail.@pytest.mark.parametrize(@@ -334,8 +333,8 @@ def test_consume_helper_preserves_value_when_no_match():assert remaining == ["--top-k", "20"]-# `-p` is typer short for --port, so Click clusters `-np8` as `-n -p 8`-# (port=8, parallel dropped). The rewrite splits to `-np 8` pre-parse.+# `-p` is typer short for --port, so Click clusters `-np8` as `-n -p 8` (port=8, parallel dropped).+# The rewrite splits to `-np 8` pre-parse.def test_expand_np_rewrites_attached_form(monkeypatch):@@ -419,8 +418,8 @@ def test_consume_helper_rejects_empty_inline_value():helper(["-m="], ("-m",), None, "--model")-# Gate isolation: importing unsloth_cli from a third-party script must-# leave its sys.argv intact. Pins the narrow basename set.+# Gate isolation: importing unsloth_cli from a third-party script must leave its sys.argv intact.+# Pins the narrow basename set.@pytest.mark.parametrize(diff --git a/unsloth_cli/tests/test_studio_runtime_gate_powershell.py b/unsloth_cli/tests/test_studio_runtime_gate_powershell.pyindex 90c1b69b7ba..d5043efeb5b 100644--- a/unsloth_cli/tests/test_studio_runtime_gate_powershell.py+++ b/unsloth_cli/tests/test_studio_runtime_gate_powershell.py@@ -50,12 +50,11 @@ def test_returns_the_bare_name_as_a_last_resort(monkeypatch, tmp_path):assert resolve_windows_powershell() == "powershell.exe"-# ── the callers ────────────────────────────────────────────────────────────────────-#-# Resolving in the gate alone does not fix #9440: setup() and update() both run the gate and-# then hand off to PowerShell again, so every spawn on that path has to use the resolver or the-# install dies at the next one with the same WinError 2.+# The callers. Resolving in the gate alone does not fix #9440: setup() and update() both run the+# gate and then hand off to PowerShell again, so every spawn on that path has to use the resolver+# or the install dies at the next one with the same WinError 2.+# ── the callers ────────────────────────────────────────────────────────────────────_RESOLVED = ntpath.join(r"C:\Windows", "System32", "WindowsPowerShell", "v1.0", "powershell.exe")diff --git a/unsloth_cli/tests/test_studio_stop.py b/unsloth_cli/tests/test_studio_stop.pyindex 74e34d4fa19..72d936e284c 100644--- a/unsloth_cli/tests/test_studio_stop.py+++ b/unsloth_cli/tests/test_studio_stop.py@@ -96,9 +96,8 @@ def test_stop_does_not_leave_the_older_instance_running(monkeypatch, tmp_path):def test_stop_signals_each_server_once(monkeypatch, tmp_path):- # A server writes its per-port file AND studio.pid. It stays alive while it- # shuts down gracefully, so a second SIGTERM would hit the SIG_DFL the first- # one installs and hard-kill it mid-cleanup.+ # A server writes its per-port file AND studio.pid. It stays alive while it shuts down gracefully,+ # so a second SIGTERM would hit the SIG_DFL the first installs and hard-kill it mid-cleanup.studio_mod = _studio()monkeypatch.setattr(studio_mod, "STUDIO_HOME", tmp_path)monkeypatch.setattr(studio_mod, "_PID_FILE", tmp_path / "studio.pid")@@ -131,8 +130,8 @@ def test_stop_removes_every_stale_file_for_one_pid(monkeypatch, tmp_path):def test_stop_does_not_signal_a_reused_pid(monkeypatch, tmp_path):- # Crash leaves a per-port file behind, the OS hands that PID to something- # else: stop must drop the record, not SIGTERM an unrelated process.+ # Crash leaves a per-port file behind, the OS hands that PID to something else: stop must drop the+ # record, not SIGTERM an unrelated process.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})monkeypatch.setattr(studio_mod, "_pid_is_studio_server", lambda pid, created_times = (): False)_write_pid(tmp_path, "studio-8901-8550.pid", 8550)@@ -145,8 +144,8 @@ def test_stop_does_not_signal_a_reused_pid(monkeypatch, tmp_path):def test_stop_signals_a_live_server_whose_pid_has_a_stale_record(monkeypatch, tmp_path):- # Crash leaves studio-8888-8550.pid, the OS reuses 8550 for a new server on- # another port. The stale timestamp must not veto the live one.+ # Crash leaves studio-8888-8550.pid, the OS reuses 8550 for a new server on another port. The+ # stale timestamp must not veto the live one.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)@@ -169,8 +168,8 @@ def test_stop_signals_a_live_server_whose_pid_has_a_stale_record(monkeypatch, tmdef test_a_bare_run_py_command_line_is_not_rejected(monkeypatch):- # `cd studio/backend && python run.py --port 8901` has no "studio" or "unsloth"- # in argv. Guessing from the command line deleted its record without stopping it.+ # `cd studio/backend && python run.py --port 8901` has no "studio" or "unsloth" in argv; guessing+ # from the command line deleted its record without stopping it.studio_mod = _studio()class _FakeProcess:@@ -189,8 +188,8 @@ def test_a_bare_run_py_command_line_is_not_rejected(monkeypatch):def test_an_untimed_record_is_trusted(monkeypatch):- # A legacy `python run.py --port 8901` has no telltale argv, and the in-venv- # path runs in-process. Guessing from the command line rejected real servers.+ # A legacy `python run.py --port 8901` has no telltale argv and the in-venv path runs in-process;+ # guessing from the command line rejected real servers.studio_mod = _studio()assert studio_mod._pid_is_studio_server(8550) is True@@ -198,9 +197,9 @@ def test_an_untimed_record_is_trusted(monkeypatch):def test_an_unverifiable_record_is_still_stopped(monkeypatch):- # psutil is not a base CLI dependency, so the CLI meets timestamped records it- # cannot check. The old `stop` signalled with no checks at all -- skipping one- # would leave a live server running, the orphan bug this exists to fix.+ # psutil is not a base CLI dependency, so the CLI meets timestamped records it cannot check. The+ # old `stop` signalled with no checks at all, and skipping one would leave a live server+ # running.studio_mod = _studio()monkeypatch.setitem(sys.modules, "psutil", None)@@ -209,8 +208,8 @@ def test_an_unverifiable_record_is_still_stopped(monkeypatch):def test_stop_signals_a_timestamped_record_without_psutil(monkeypatch, tmp_path):- # Multiple servers on different ports: only the newest is also in studio.pid,- # so the earlier ones are timestamp-only and must still be stopped.+ # Multiple servers on different ports: only the newest is also in studio.pid, so the earlier ones+ # are timestamp-only and must still be stopped.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)monkeypatch.setitem(sys.modules, "psutil", None)@@ -224,11 +223,10 @@ def test_stop_signals_a_timestamped_record_without_psutil(monkeypatch, tmp_path)def test_the_untimed_legacy_record_does_not_cancel_a_timed_one(monkeypatch):- # Every current server writes BOTH a timed per-port record and an untimed- # studio.pid, so letting the untimed half win made this check inert exactly- # where it matters: after a crash and a PID reuse, `stop` SIGTERMed whatever- # unrelated process had inherited the PID. An untimed record carries no- # information, so it must not overrule a start time that says "not ours".+ # Every current server writes BOTH a timed per-port record and an untimed studio.pid, so letting+ # the untimed half win made this check inert exactly where it matters: after a crash and a PID+ # reuse, `stop` SIGTERMed whatever unrelated process inherited the PID. An untimed record carries+ # no information, so it must not overrule a start time that says 'not ours'.studio_mod = _studio()class _FakeProcess:@@ -248,8 +246,8 @@ def test_the_untimed_legacy_record_does_not_cancel_a_timed_one(monkeypatch):def test_stop_does_not_signal_a_reused_pid_recorded_in_both_files(monkeypatch, tmp_path):- # End to end for the case above: a crashed server left studio-8901-8550.pid- # and studio.pid, and 8550 now belongs to something else entirely.+ # End to end for the case above: a crashed server left studio-8901-8550.pid and studio.pid, and+ # 8550 now belongs to something else.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)@@ -430,8 +428,8 @@ def test_stop_discards_a_corrupt_pid_file(monkeypatch, tmp_path):def test_stop_keeps_a_record_it_cannot_read(monkeypatch, tmp_path):- # A root-owned record, or one caught mid-write, still belongs to a live- # server. Deleting it is `stop` manufacturing the orphan it exists to fix.+ # A root-owned record, or one caught mid-write, still belongs to a live server. Deleting it is+ # `stop` manufacturing the orphan it exists to fix.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})path = tmp_path / "studio-8901-8550.pid"path.write_text("8550", encoding = "utf-8")@@ -451,8 +449,8 @@ def test_stop_keeps_a_record_it_cannot_read(monkeypatch, tmp_path):def test_stop_does_not_claim_success_when_the_only_record_is_unreadable(monkeypatch, tmp_path):- # A server started under sudo leaves a record we cannot read. Printing "no- # running server" and exiting 0 tells the user the opposite of the truth.+ # A server started under sudo leaves a record we cannot read. Printing "no running server" and+ # exiting 0 tells the user the opposite of the truth.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})path = tmp_path / "studio-8901-8550.pid"path.write_text("8550", encoding = "utf-8")@@ -476,8 +474,7 @@ def test_stop_does_not_claim_success_when_the_only_record_is_unreadable(monkeypadef test_stop_reports_failure_when_one_record_is_unreadable_but_another_stops(monkeypatch, tmp_path):- # Stopping the servers we can see is still a partial result, and exiting 0- # would hide the one we could not.+ # Stopping the servers we can see is still a partial result, and exiting 0 would hide the one we could not.studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550, 8600})_write_pid(tmp_path, "studio-8901-8550.pid", 8550)hidden = tmp_path / "studio-8902-8600.pid"@@ -519,8 +516,8 @@ def test_stop_reaches_every_server_when_one_record_cannot_be_removed(monkeypatchdef test_a_record_whose_pid_is_not_ascii_digits_is_discarded(monkeypatch, tmp_path):- # A superscript two passes isdigit() but int() rejects it, so that gate alone- # let a ValueError escape _read_pid_record and abort the whole command.+ # A superscript two passes isdigit() but int() rejects it, so that gate alone let a ValueError+ # escape _read_pid_record and abort the whole command.studio_mod, _live, _killed = _install(monkeypatch, tmp_path, alive = set())(tmp_path / "studio-8901-1.pid").write_text("²", encoding = "utf-8")diff --git a/unsloth_cli/tests/test_studio_update_local_repo.py b/unsloth_cli/tests/test_studio_update_local_repo.pyindex cb1dd5929cb..7fa810cccb4 100644--- a/unsloth_cli/tests/test_studio_update_local_repo.py+++ b/unsloth_cli/tests/test_studio_update_local_repo.py@@ -96,8 +96,8 @@ def test_the_derived_root_is_used_when_nothing_is_set(monkeypatch):def test_a_pypi_update_never_looks_for_a_checkout(monkeypatch, tmp_path):- # Without --local there is no local repo to find, and a stale- # STUDIO_LOCAL_REPO must not leak into the setup environment.+ # Without --local there is no local repo to find, and a stale STUDIO_LOCAL_REPO must not leak into+ # the setup environment.site = tmp_path / "site-packages"site.mkdir()studio, seen = _neutered(monkeypatch)@@ -109,9 +109,9 @@ def test_a_pypi_update_never_looks_for_a_checkout(monkeypatch, tmp_path):def test_a_relative_override_is_absolutised(monkeypatch, tmp_path):- # setup.sh does `cd "$SCRIPT_DIR"` before install_python_stack.py runs, so- # a relative path handed straight through resolves against studio/ (which- # has no pyproject.toml) and hits the exact uv error the guard replaces.+ # setup.sh does `cd "$SCRIPT_DIR"` before install_python_stack.py runs, so a relative path handed+ # straight through resolves against studio/ (which has no pyproject.toml) and hits the exact uv+ # error the guard replaces.checkout = tmp_path / "unsloth"checkout.mkdir()(checkout / "pyproject.toml").write_text("[project]\nname = 'unsloth'\n")@@ -139,8 +139,8 @@ def test_a_tilde_override_is_expanded(monkeypatch, tmp_path):def test_a_blank_override_falls_back_to_the_derived_root(monkeypatch):- # `STUDIO_LOCAL_REPO= ` (install.sh resets it to empty) must not become- # Path(" ") and fail the guard on a perfectly good checkout.+ # `STUDIO_LOCAL_REPO= ` (install.sh resets it to empty) must not become Path(" ") and fail the+ # guard on a perfectly good checkout.studio, seen = _neutered(monkeypatch)monkeypatch.setenv("STUDIO_LOCAL_REPO", " ")result = CliRunner().invoke(studio.studio_app, ["update", "--local"])@@ -201,9 +201,8 @@ def test_a_pypi_update_passes_no_checkout(monkeypatch):def test_windows_is_shown_a_powershell_assignment(monkeypatch, tmp_path):- # `VAR=value command` is POSIX shell syntax. PowerShell parses the- # assignment as a command name, so the only recovery instruction the guard- # prints was unusable on the platform the guard exists for.+ # `VAR=value command` is POSIX shell syntax. PowerShell parses the assignment as a command name, so+ # the only recovery instruction the guard prints was unusable on the platform it exists for.import platform as _platformsite = tmp_path / "Lib" / "site-packages"diff --git a/unsloth_cli/tests/test_studio_update_stage.py b/unsloth_cli/tests/test_studio_update_stage.pyindex c1c40b67bab..eb9523589d2 100644--- a/unsloth_cli/tests/test_studio_update_stage.py+++ b/unsloth_cli/tests/test_studio_update_stage.py@@ -221,8 +221,8 @@ def test_stage_refuses_when_the_previous_stage_could_not_be_cleared(monkeypatch,_make_venv(tmp_path)stale = _studio_stage.stage_root(tmp_path)(stale / "leftover").mkdir(parents = True)- # discard() swallows its errors, so a locked or undeletable tree would otherwise- # be staged into and shipped as if it were a fresh clone.+ # discard() swallows its errors, so a locked or undeletable tree would otherwise be staged into and+ # shipped as if it were a fresh clone.monkeypatch.setattr(_studio_stage, "discard", lambda root: None)with pytest.raises(_studio_stage.StageError, match = "could not clear"):@@ -357,8 +357,8 @@ def test_stage_relocates_the_launcher_the_update_rewrote(monkeypatch, tmp_path):stage_venv = home / _studio_stage.STAGE_DIR_NAME / _studio_stage.VENV_NAMEdef reinstalling_update(root: Path, args: list[str]) -> int:- # What pip/uv do on every upgrade: rewrite the console script with the- # interpreter named by absolute path, inside the stage.+ # What pip/uv do on every upgrade: rewrite the console script with the interpreter named by+ # absolute path, inside the stage.(root / _studio_stage.VENV_NAME / "bin" / "unsloth").write_text(f"#!{root / _studio_stage.VENV_NAME}/bin/python\nprint('cli')\n", encoding = "utf-8")@@ -371,8 +371,8 @@ def test_stage_relocates_the_launcher_the_update_rewrote(monkeypatch, tmp_path):_studio_stage.stage(home, update_args = [], echo = lambda _: None, run_update = reinstalling_update)launcher = (stage_venv / "bin" / "unsloth").read_text(encoding = "utf-8")- # Activation moves the venv out of .update-stage, so an absolute shebang into- # it would leave the activated backend unable to start at all.+ # Activation moves the venv out of .update-stage, so an absolute shebang into it would leave the+ # activated backend unable to start at all.assert launcher.startswith(_studio_stage.RELOCATABLE_SHEBANG)assert str(_studio_stage.STAGE_DIR_NAME) not in launcher.splitlines()[1]diff --git a/unsloth_cli/tests/test_studio_update_verify.py b/unsloth_cli/tests/test_studio_update_verify.pyindex 0ff9f4ea0c1..b87e5e147ce 100644--- a/unsloth_cli/tests/test_studio_update_verify.py+++ b/unsloth_cli/tests/test_studio_update_verify.py@@ -76,8 +76,8 @@ def site(tmp_path, monkeypatch):d = tmp_path / "site-packages"d.mkdir()monkeypatch.syspath_prepend(str(d))- # sys.path alone is not enough: the real environment's distributions would- # still be discovered and could contribute findings of their own.+ # sys.path alone is not enough: the real environment's distributions would still be discovered+ # and could contribute findings of their own.import importlib.metadata as mdreal = md.distributions@@ -167,28 +167,25 @@ def test_a_deleted_file_is_reported(site):def test_deletion_is_seen_whatever_Distribution_files_does(site):- # Distribution.files is not a usable basis for this check, and it is not- # consistent either: newer CPython filters out entries whose file is gone- # (so a deletion becomes invisible), older CPython lists them but with a- # path that does not exist. Which one you get depends on the interpreter,- # and this project supports >= 3.9, so pinning one behaviour would make the- # test fail on the other. RECORD is parsed directly instead, which reports- # the deletion on every version.+ # Distribution.files is not a usable basis for this check and is not consistent either: newer+ # CPython filters out entries whose file is gone, older CPython lists them with a path that+ # does not exist. This project supports >= 3.9, so RECORD is parsed directly instead and the+ # deletion is reported on every version.import importlib.metadata as md_make_dist(site, "gamma", {"gamma/a.py": b"a\n", "gamma/b.py": b"bb\n"})(site / "gamma" / "b.py").unlink()stale = [f for f in (md.distribution("gamma").files or []) if str(f) == "gamma/b.py"]- # Either it was dropped, or it is listed and locate() does not resolve.- # Both mean files() cannot tell you the file is gone.+ # Either it was dropped, or it is listed and locate() does not resolve; both mean files()+ # cannot tell you the file is gone.assert not stale or not stale[0].locate().exists()assert any("gamma/b.py is missing" in f for f in _deps().damaged_installed_files())def test_a_file_larger_than_recorded_is_not_damage(site):- # Two distributions claiming one path: descript-audio-codec ships a- # top-level tests/__init__.py that another package overwrites. Flagging that- # would block updates on a perfectly healthy install.+ # Two distributions claiming one path: descript-audio-codec ships a top-level+ # tests/__init__.py that another package overwrites, and flagging that would block updates on+ # a healthy install._make_dist(site,"delta",@@ -199,11 +196,10 @@ def test_a_file_larger_than_recorded_is_not_damage(site):def test_a_shared_file_shorter_than_recorded_is_not_damage(site):- # The mirror of the case above. Two distributions claiming one path is a- # packaging collision, and whichever copy landed is the one on disk, so its- # size says nothing about either RECORD -- in either direction. Only the- # larger direction was excluded, so a collision that overwrote with a- # shorter file was reported as corruption and blocked every update.+ # The mirror of the case above: whichever copy landed is the one on disk, so its size says+ # nothing about either RECORD in either direction. Only the larger direction was excluded, so+ # a collision that overwrote with a shorter file was reported as corruption and blocked every+ # update._make_dist(site, "iota", {"shared/__init__.py": b"short\n"}, record_sizes = {"shared/__init__.py": 900})@@ -221,10 +217,9 @@ def test_a_singly_owned_short_file_is_still_damage(site):def test_the_scan_is_limited_to_this_interpreters_site_packages(monkeypatch, tmp_path):- # distributions() searches every sys.path entry, so a damaged distribution- # reachable only through an inherited PYTHONPATH failed every update while- # sitting outside the installation, where neither printed repair command can- # reach it. Only --no-verify broke the loop.+ # distributions() searches every sys.path entry, so a damaged distribution reachable only+ # through an inherited PYTHONPATH failed every update while sitting outside the installation,+ # where neither printed repair command can reach it. Only --no-verify broke the loop.external = tmp_path / "elsewhere"(external / "ext-1.0.dist-info").mkdir(parents = True)(external / "ext").mkdir()@@ -248,8 +243,8 @@ def test_the_scan_is_limited_to_this_interpreters_site_packages(monkeypatch, tmpdef test_a_deleted_shared_file_is_still_reported(site):- # Multiple ownership makes the recorded SIZES ambiguous; it cannot explain- # the file being gone. Skipping shared paths outright hid real deletions.+ # Multiple ownership makes the recorded SIZES ambiguous; it cannot explain the file being gone,+ # and skipping shared paths outright hid real deletions._make_dist(site, "mu", {"shared/x.py": b"hello\n"}, record_sizes = {"shared/x.py": 10})_make_dist(site, "nu", {}, record_sizes = {})(site / "nu-1.0.dist-info" / "RECORD").write_text(@@ -262,8 +257,8 @@ def test_a_deleted_shared_file_is_still_reported(site):def test_a_row_without_a_recorded_size_is_still_checked(site):- # The size field is optional and real wheels leave it blank. Dropping those- # rows meant a deleted file was never reported.+ # The size field is optional and real wheels leave it blank; dropping those rows meant a+ # deleted file was never reported._make_dist(site, "xi", {"xi/__init__.py": b"y\n"})(site / "xi-1.0.dist-info" / "RECORD").write_text("xi/__init__.py,,\n")(site / "xi" / "__init__.py").unlink()@@ -272,8 +267,8 @@ def test_a_row_without_a_recorded_size_is_still_checked(site):def test_a_directory_standing_in_for_a_module_is_damage(site):- # An empty directory is commonly 4096 bytes on POSIX, so it sails past the- # shrinkage test while importing as something other than the recorded module.+ # An empty directory is commonly 4096 bytes on POSIX, so it sails past the shrinkage test while+ # importing as something other than the recorded module._make_dist(site, "omicron", {}, record_sizes = {})(site / "omicron-1.0.dist-info" / "RECORD").write_text("omicron/mod.py,sha256=x,10\n")(site / "omicron" / "mod.py").mkdir(parents = True)@@ -282,8 +277,8 @@ def test_a_directory_standing_in_for_a_module_is_damage(site):def test_installer_owned_metadata_is_ignored(site):- # .dist-info files are rewritten in place and drift from the size recorded- # inside themselves; two real distributions did exactly that.+ # .dist-info files are rewritten in place and drift from the size recorded inside themselves;+ # two real distributions did exactly that._make_dist(site, "epsilon", {"epsilon/__init__.py": b"e\n"})info = site / "epsilon-1.0.dist-info"(info / "RECORD").write_text(@@ -308,11 +303,10 @@ def test_findings_are_capped(site):def test_findings_are_capped_when_the_files_are_deleted(site):- # Truncation and deletion take different branches, and only truncation was- # covered. A wiped package -- `rm -rf` on a venv's torch, the shape a user- # actually hits -- takes the deletion branch, so an uncapped one floods the- # caller with a line per RECORD entry (~11.8k for torch), each of which the- # desktop updater turns into its own IPC event.+ # Truncation and deletion take different branches and only truncation was covered. A wiped+ # package (`rm -rf` on a venv's torch, the shape users hit) takes the deletion branch, so an+ # uncapped one floods the caller with a line per RECORD entry, about 11.8k for torch, each of+ # which the desktop updater turns into its own IPC event.files = {f"theta/m{i}.py": b"x" * 500 for i in range(40)}_make_dist(site, "theta", files)for rel in files:@@ -416,8 +410,8 @@ def test_a_damaged_tree_exits_nonzero_and_names_the_files(monkeypatch, capsys):def test_a_foreign_cli_stays_quiet(monkeypatch):- # A pip-installed CLI can drive an update into a venv it does not live in;- # its own file list would describe the wrong tree, so it must not accuse.+ # A pip-installed CLI can drive an update into a venv it does not live in; its own file list+ # would describe the wrong tree, so it must not accuse.studio = _studio()monkeypatch.setattr(studio._studio_deps, "running_outside_managed_venv", lambda *a: True)@@ -429,11 +423,10 @@ def test_a_foreign_cli_stays_quiet(monkeypatch):def test_a_system_python_is_not_treated_as_the_managed_venv(monkeypatch, tmp_path):- # Colab has no Unsloth venv: studio/setup.sh installs the backend into the- # system Python on purpose. Distro-packaged RECORDs there list files the- # distro never installed (PEP 627), so running the file check would accuse- # the distro of damaging Unsloth. Reproduced on Ubuntu system Python, which- # reports an apt-owned `markdown-it-py: ../scripts/markdown-it is missing`.+ # Colab has no Unsloth venv: studio/setup.sh installs the backend into the system Python on+ # purpose, and distro-packaged RECORDs there list files the distro never installed (PEP 627),+ # so the file check would accuse the distro of damaging Unsloth. Reproduced on Ubuntu system+ # Python, which reports an apt-owned `markdown-it-py: ../scripts/markdown-it is missing`.prefix = tmp_path / "usr"prefix.mkdir()monkeypatch.setattr(sys, "prefix", str(prefix))@@ -474,10 +467,9 @@ def test_update_exposes_verify_defaulting_on():def test_the_verify_help_does_not_promise_an_import_check():- # The scan compares RECORD entries against the filesystem and imports- # nothing, so it cannot see same-size corruption or an intact but- # incompatible package. Saying it checks that the backend still imports- # promises a stronger guarantee than it delivers.+ # The scan compares RECORD entries against the filesystem and imports nothing, so it cannot see+ # same-size corruption or an intact but incompatible package. Saying it checks that the+ # backend still imports promises more than it delivers.import inspectopt = inspect.signature(_studio().update).parameters["verify"].default@@ -524,9 +516,9 @@ def test_no_verify_skips_the_check(monkeypatch):def test_a_tauri_update_is_verified_too(monkeypatch):- # The Tauri path returns before the shortcut refresh, so a check placed- # after that return would silently not run for desktop-initiated updates,- # the one flow where the user never sees a terminal.+ # The Tauri path returns before the shortcut refresh, so a check placed after that return would+ # silently not run for desktop-initiated updates, the one flow where the user never sees a+ # terminal.verified = []monkeypatch.setenv("UNSLOTH_TAURI_UPDATE", "1")result = _run_update(monkeypatch, [], verified)@@ -542,10 +534,9 @@ def test_a_tauri_update_is_verified_too(monkeypatch):],)def test_a_custom_root_is_carried_into_the_reinstall_command(monkeypatch, capsys, system, expected):- # The CLI shim is a bare symlink and _ensure_studio_env_exported only sets- # os.environ for this process, so the shell that runs the printed command- # has no UNSLOTH_STUDIO_HOME. Unqualified, it would build a fresh- # ~/.unsloth/studio and leave the damaged custom root broken.+ # The CLI shim is a bare symlink and _ensure_studio_env_exported only sets os.environ for this+ # process, so the shell running the printed command has no UNSLOTH_STUDIO_HOME; unqualified,+ # it would build a fresh ~/.unsloth/studio and leave the damaged custom root broken.import platform as _platformimport typer@@ -587,9 +578,9 @@ def test_a_root_with_spaces_is_quoted(monkeypatch, capsys):],)def test_a_no_torch_install_keeps_that_mode_in_the_reinstall(monkeypatch, capsys, system, expected):- # install.sh derives SKIP_TORCH from its flag or UNSLOTH_NO_TORCH only, so- # following the plain command on a GGUF-only install pulls the whole PyTorch- # stack -- multiple GB the user deliberately opted out of.+ # install.sh derives SKIP_TORCH from its flag or UNSLOTH_NO_TORCH only, so following the plain+ # command on a GGUF-only install pulls the whole PyTorch stack, multiple GB the user opted out+ # of.import platform as _platformimport typer@@ -598,9 +589,9 @@ def test_a_no_torch_install_keeps_that_mode_in_the_reinstall(monkeypatch, capsysmonkeypatch.setattr(studio._studio_deps, "damaged_installed_files", lambda *a, **k: ["x: y is missing"])- # The stub records what root it was asked for: the manifest and marker live- # in the venv, and an earlier attempt at this passed STUDIO_HOME, which is- # one directory too high, so it read None and never fired in production.+ # The stub records what root it was asked for: the manifest and marker live in the venv, and an+ # earlier attempt passed STUDIO_HOME, one directory too high, so it read None and never fired+ # in production.seen = {}def _module(*a, **k):@@ -621,9 +612,9 @@ def test_a_no_torch_install_keeps_that_mode_in_the_reinstall(monkeypatch, capsys@pytest.mark.parametrize("recorded", [False, None])def test_an_unrecorded_or_torch_install_does_not_gain_the_flag(monkeypatch, capsys, recorded):- # recorded_no_torch() returns None when nothing recorded the mode, and its- # contract is that None is not False. Adding the flag on a guess would leave- # a torch install without torch, so the flag is added only on an explicit True.+ # recorded_no_torch() returns None when nothing recorded the mode, and its contract is that+ # None is not False: adding the flag on a guess would leave a torch install without torch, so+ # it is added only on an explicit True.import platform as _platformimport typer@@ -662,13 +653,11 @@ def test_the_default_root_keeps_the_plain_command(monkeypatch, capsys):def test_the_message_covers_packages_the_installer_will_not_repair(monkeypatch, capsys):- # install_python_stack installs the current requirement sets and prunes- # nothing, and the installer never recreates the venv, so damage in an- # orphan from an older release survives the reinstall it recommends and- # would report the same failure forever. The scan is deliberately not- # scoped to Unsloth's dependency closure: under-including there would let- # real damage through, which is the failure this whole check exists to- # catch. So the message has to carry the fallback instead.+ # install_python_stack installs the current requirement sets and prunes nothing, and the+ # installer never recreates the venv, so damage in an orphan from an older release survives the+ # reinstall it recommends and would report the same failure forever. The scan is deliberately+ # not scoped to Unsloth's dependency closure, since under-including would let real damage+ # through, so the message carries the fallback instead.import typerstudio = _studio()@@ -683,11 +672,11 @@ def test_the_message_covers_packages_the_installer_will_not_repair(monkeypatch,err = capsys.readouterr().errassert "still listed after that" in errassert "--force-reinstall" in err- # Without --no-deps, pip resolves the damaged package's graph and- # --force-reinstall can swap the pinned CUDA/ROCm torch build.+ # Without --no-deps, pip resolves the damaged package's graph and --force-reinstall can swap+ # the pinned CUDA/ROCm torch build.assert "--no-deps" in err- # A bare name would let --force-reinstall upgrade the orphan rather than- # repair it, which --no-deps does not prevent.+ # A bare name would let --force-reinstall upgrade the orphan rather than repair it, which+ # --no-deps does not prevent.assert "<package>==<installed version>" in errassert "--no-verify" in err@@ -700,8 +689,7 @@ def test_the_message_covers_packages_the_installer_will_not_repair(monkeypatch,],)def test_the_repair_command_quotes_the_interpreter(monkeypatch, capsys, system, exe, expected):- # Custom roots with spaces are supported, so an unquoted sys.executable- # would split into several shell tokens and the command would not run.+ # Custom roots with spaces are supported, so an unquoted sys.executable would split into several shell tokens.import platform as _platformimport typer@@ -723,10 +711,10 @@ def test_the_repair_command_quotes_the_interpreter(monkeypatch, capsys, system,def test_a_shared_top_level_test_tree_is_not_damage(site):- # Reported as `einx: test/conftest.py is missing`. einx and torchao both- # ship it, and install_python_stack.py force-reinstalls torchao every- # update, so pip removes the file and the pinned torchao does not ship it.- # Nothing imports another project's fixtures, and no reinstall repairs it.+ # Reported as `einx: test/conftest.py is missing`. einx and torchao both ship it and+ # install_python_stack.py force-reinstalls torchao every update, so pip removes the file and+ # the pinned torchao does not ship it. Nothing imports another project's fixtures, and no+ # reinstall repairs it._make_dist(site, "einx", {"einx/__init__.py": b"e\n"})(site / "einx-1.0.dist-info" / "RECORD").write_text("einx/__init__.py,sha256=x,2\ntest/conftest.py,sha256=x,20650\n"@@ -735,9 +723,9 @@ def test_a_shared_top_level_test_tree_is_not_damage(site):def test_an_installer_rewritten_lockfile_is_not_damage(site):- # Reported as `package-lock.json is 27225 bytes, expected 28473`.- # setup.ps1/setup.sh run `npm install` inside the installed tree, and npm- # dedupes hoisted entries under legacy-peer-deps, shrinking the file.+ # Reported as `package-lock.json is 27225 bytes, expected 28473`: setup.ps1/setup.sh run `npm+ # install` inside the installed tree, and npm dedupes hoisted entries under legacy-peer-deps,+ # shrinking the file.lock = "studio/backend/core/data_recipe/oxc-validator/package-lock.json"_make_dist(site,@@ -749,8 +737,8 @@ def test_an_installer_rewritten_lockfile_is_not_damage(site):def test_a_deleted_installer_rewritten_file_is_still_damage(site):- # Only the SIZE of these drifts, because npm rewrites the lockfile in place.- # It never deletes it, so a missing one is real damage and must be reported.+ # Only the SIZE of these drifts, because npm rewrites the lockfile in place. It never deletes+ # it, so a missing one is real damage and must be reported.lock = "studio/backend/core/data_recipe/oxc-validator/package-lock.json"_make_dist(site, "unsloth", {"unsloth/__init__.py": b"u\n", lock: b"L" * 100})(site / lock).unlink()@@ -759,8 +747,8 @@ def test_a_deleted_installer_rewritten_file_is_still_damage(site):def test_a_shared_top_level_scripts_tree_is_not_damage(site):- # unsloth_zoo ships a top-level scripts/, the same squatted-namespace shape- # as einx's test/. It has no __init__.py, so nothing imports it.+ # unsloth_zoo ships a top-level scripts/, the same squatted-namespace shape as einx's test/. It+ # has no __init__.py, so nothing imports it._make_dist(site, "upsilon", {"upsilon/__init__.py": b"u\n"})(site / "upsilon-1.0.dist-info" / "RECORD").write_text("upsilon/__init__.py,sha256=x,2\nscripts/helper.py,sha256=x,99\n"@@ -769,8 +757,8 @@ def test_a_shared_top_level_scripts_tree_is_not_damage(site):def test_a_package_owned_tests_subdirectory_is_still_checked(site):- # Only the shared top-level namespace is exempt; a tests/ tree inside a- # package is that package's alone, so a deletion there is real.+ # Only the shared top-level namespace is exempt; a tests/ tree inside a package is that+ # package's alone, so a deletion there is real._make_dist(site, "rho", {"rho/tests/helper.py": b"h\n"})(site / "rho" / "tests" / "helper.py").unlink()found = _deps().damaged_installed_files()@@ -800,8 +788,8 @@ def test_runtime_damage_still_fails_when_ignored_rows_are_present(site):def test_ignored_rows_do_not_consume_the_finding_budget(site):- # Filtering happens while RECORD is read, so harmless rows cannot crowd a- # real one off a capped list. Unfiltered, these 40 fill limit = 3.+ # Filtering happens while RECORD is read, so harmless rows cannot crowd a real one off a capped+ # list; unfiltered, these 40 fill limit = 3.files = {f"test/t{i}.py": b"x" for i in range(40)}files["tau/__init__.py"] = b"t\n"_make_dist(site, "tau", files)@@ -812,11 +800,10 @@ def test_ignored_rows_do_not_consume_the_finding_budget(site):def test_our_own_shared_top_level_trees_are_exempt_too(site):- # Reported as `unsloth_zoo: tests/conftest.py is 8107 bytes, expected 11429`- # on an install that was already current, so the update failed on every- # retry. 11429 is the size in the 2026.8.5 wheel, which shipped a top-level- # tests/ that any other wheel using that name overwrites. Squatting the- # namespace does not make it ours, and nothing imports it at runtime.+ # Reported as `unsloth_zoo: tests/conftest.py is 8107 bytes, expected 11429` on an install that+ # was already current, so the update failed on every retry. 11429 is the size in the 2026.8.5+ # wheel, which shipped a top-level tests/ that any other wheel using that name overwrites;+ # squatting the namespace does not make it ours, and nothing imports it at runtime.conftest = "tests/conftest.py"_make_dist(site,@@ -828,9 +815,9 @@ def test_our_own_shared_top_level_trees_are_exempt_too(site):def test_two_distributions_claiming_one_shared_path(site):- # The real shape: a third party also ships tests/conftest.py and lands last,- # so the bytes on disk match its RECORD and are shorter than unsloth_zoo's.- # Neither the drift nor the deletion can affect startup.+ # The real shape: a third party also ships tests/conftest.py and lands last, so the bytes on+ # disk match its RECORD and are shorter than unsloth_zoo's. Neither the drift nor the deletion+ # can affect startup.rel = "tests/conftest.py"_make_dist(site, "unsloth_zoo", {rel: b"u" * 11429})_make_dist(site, "upsilon", {rel: b"c" * 8107})@@ -841,8 +828,8 @@ def test_two_distributions_claiming_one_shared_path(site):def test_our_own_shared_top_level_trees_may_also_vanish(site):- # Same path, deleted rather than overwritten: the einx shape, but claimed by- # a distribution of ours. No reinstall repairs it either.+ # Same path, deleted rather than overwritten: the einx shape, but claimed by a distribution of+ # ours. No reinstall repairs it either._make_dist(site, "unsloth_zoo", {"unsloth_zoo/__init__.py": b"z\n"})(site / "unsloth_zoo-1.0.dist-info" / "RECORD").write_text("unsloth_zoo/__init__.py,sha256=x,2\nscripts/helper.py,sha256=x,99\n"@@ -851,8 +838,8 @@ def test_our_own_shared_top_level_trees_may_also_vanish(site):def test_our_own_runtime_trees_are_still_checked(site):- # The exemption is scoped to the shared roots. Everything Unsloth actually- # imports lives outside them and must still fail an update when damaged.+ # The exemption is scoped to the shared roots; everything Unsloth actually imports lives+ # outside them and must still fail an update when damaged._make_dist(site,"unsloth_zoo",