Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Restore facts the comment trim cut from the studiobench harness (#10331)
Daniel Han2acc200360de
diff --git a/tests/studio/studiobench/__main__.py b/tests/studio/studiobench/__main__.pyindex c0935538f70..67710276a68 100644--- a/tests/studio/studiobench/__main__.py+++ b/tests/studio/studiobench/__main__.py@@ -793,9 +793,11 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_linit_scripts = []# ONE PROVIDER PER ORIGIN: the provider lives in the backend and localStorage is per-origin, so- # an attached null control is two sides on one Unsloth. `register_provider` is idempotent by- # DISPLAY NAME, so registering per side deletes the id the first side's seed script captured.- # `is_null_control` is what recognises it.+ # an attached null control (`--attach U --attach-b U`) is two sides on one Unsloth.+ # `register_provider` is idempotent by DISPLAY NAME, so registering per side deletes the id the+ # first side's seed script captured, and that script keeps selecting the dead id on every+ # navigation it wins: the order init scripts run in is not defined and `StudioAuth.rotate`+ # re-adds them mid-run. `is_null_control` is what recognises it.registered: dict = {}for index, side in enumerate(sides):side_install = installs[index][0]@@ -1021,7 +1023,8 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_lif args.tier == "fast":# Said in the log AND recorded. A fast-tier reading is a DIRECTION, not a number: one rung, a 47 s- # film, a wider detection floor and no null control, so the analysis layer must refuse to pool it.+ # film, however few repetitions were asked for, a wider detection floor, and no null control of+ # its own unless one is run alongside. So the analysis layer must refuse to pool it._log("")_log(" FAST TIER: for iteration while you are changing something, not for reporting.")_log(@@ -1200,7 +1203,9 @@ def _run_holding_out_dir(args, ab_ref, specs, arm_labels, windowed, paths, out_l"base_ref": sides[0]["ref"],"treatment_ref": sides[1]["ref"],# WHICH SERVER THE TREATMENT WAS, when the caller attached one: without it a resume compares only- # the `--ab` label, which says nothing about the build that answered. Empty when we installed it.+ # the `--ab` label, which says nothing about the build that answered. `run_meta` records the+ # base the same way in `studio_ref`; see `requested_identity`. Empty when this run installed+ # the treatment itself, since then the ref above IS its identity."treatment_url": "" if sides[1]["owns"] else sides[1]["base_url"],# The treatment's half of `studio_commit`. Same reason, same emptiness rule."treatment_commit": sides[1].get("commit") or "",diff --git a/tests/studio/studiobench/analysis/parity.py b/tests/studio/studiobench/analysis/parity.pyindex d53023aefbd..6fa2e9ed307 100644--- a/tests/studio/studiobench/analysis/parity.py+++ b/tests/studio/studiobench/analysis/parity.py@@ -401,7 +401,8 @@ def compare_styles(base: dict, treat: dict) -> tuple[str, str]:# produces. NOT_COMPARABLE AND NOT MATCH: the probe reads ONE aggregate digest, so a genuine CSS# difference elsewhere is inside the same number.# Over up to `STYLE_CAP` elements.- # `streaming` and `queued_idle` are read off the run state, not the composer.+ # `streaming` and `queued_idle` are a second reading of the same composer subtree and not an+ # independent signal; see `_run_state_disagrees` for what that does and does not cover.if (bs.get("elements") != ts.get("elements") or bs.get("digest") != ts.get("digest")) and (generation_disagrees(base, treat) and _run_state_disagrees(base, treat)):@@ -450,10 +451,15 @@ def _run_state_disagrees(base: dict, treat: dict) -> bool:"""Do the arms disagree about the run state, read OFF the run state rather than the composer?`generation_disagrees` reads `composer_control`, which is the composer. Using it alone to- excuse a composer difference proves the premise with the conclusion, so this is the second,- independent half: `streaming` is `isRunning()` and `queued_idle` is the queue waiting to be- dispatched, both taken from the thread's own run state and neither derivable from which button- was drawn.+ excuse a composer difference proves the premise with the conclusion, so this is a second read:+ `streaming` is `isRunning()` and `queued_idle` is the queue waiting to be dispatched.++ NOT INDEPENDENT OF THE COMPOSER, and the comment on the caller says what that costs.+ `dom.isRunning()` is `Boolean(stopButton() || queueButton())` and `scene/parity.js` derives+ `queued_idle` from that value, `stopQueuedButton()` and the prompt queue surface, so all of it+ comes from the same run-state slot the token names. It separates a control dropped or renamed+ out of `RUN_STATE_CONTROLS`, which leaves both flags equal, from a genuine Stop-against-Send+ pair; it cannot separate the latter from a treatment that renders the wrong control.Captures older than these fields report neither, and two `False`s then read as agreement --which is the conservative direction here, since it makes the pair a reported difference rather@@ -649,14 +655,30 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:# regression inside the streaming message lands as NOT COMPARABLE, and a REORDER past another# message of the same role is demoted from DIFFER (10 of 11 injected differences still DIFFER).streaming = in_flight(base, treat)- # THE COMPOSER IS NOT A RENDERING DIFFERENCE. One arm finished and one still writing has its- # MESSAGES withheld correctly, but the dock is inside `.aui-thread-root`, so `digest_scaffold`- # carries Stop on one arm and Send on the other and `_any_moved` reported DIFFER while every- # settled row was byte-identical. THE NULL BATTERY CANNOT SEE THIS -- one build against itself- # has both arms generating and the bias cancels. WITHHELD RATHER THAN IGNORED: if a message or- # overlay also moved this never runs. AND THE RUN STATE HAS TO SAY SO INDEPENDENTLY, or the- # suppression argues in a circle.# ── THE COMPOSER IS NOT A RENDERING DIFFERENCE ──────────────────────────────────────────────+ #+ # One arm finished and one still writing has its MESSAGES withheld correctly, but the dock is+ # inside `.aui-thread-root`, so `digest_scaffold` carries Stop on one arm and Send on the other+ # and `_any_moved` reported DIFFER on the single claim `thread scaffolding outside any message+ # (373->381c)` while every settled row was byte-identical. THE NULL BATTERY CANNOT SEE THIS,+ # which is why it survived a 15-of-15-to-0 null: one build against itself has both arms+ # generating, so the bias is symmetric and cancels. WITHHELD RATHER THAN IGNORED: if a message+ # or overlay also moved this never runs.+ #+ # AND THE RUN STATE HAS TO CORROBORATE, or the suppression argues in a circle:+ # `generation_disagrees` reads `composer_control`, the token naming which control was rendered,+ # so the composer would be excusing itself. A treatment that DROPS the Send button, renames it+ # or selects the wrong control reaches this branch with every message and overlay agreeing, and+ # a refusal here is a green run (`report` files NOT COMPARABLE under `blind` and exits on+ # `stable_bad or one_sided`). `_run_state_disagrees` reads `streaming` and `queued_idle`, and+ # THAT IS A SECOND READING OF THE SAME SUBTREE RATHER THAN AN INDEPENDENT ONE: `dom.isRunning()`+ # is `Boolean(stopButton() || queueButton())`, and `scene/parity.js` builds `queued_idle` from+ # that value, `stopQueuedButton()` and the prompt queue surface. WHAT THE SECOND READING BUYS,+ # from reading the two predicates: a control DROPPED or RENAMED out of `RUN_STATE_CONTROLS`+ # moves the token while leaving both flags equal, so this branch does not fire. WHAT IT DOES+ # NOT RULE OUT: a treatment that renders the WRONG run-state control, since Stop on a settled+ # thread moves `composer_control` and `streaming` together and is refused here as NOT+ # COMPARABLE. That case needs a run-state signal read outside the composer subtree.if (generation_disagrees(base, treat)and _run_state_disagrees(base, treat)diff --git a/tests/studio/studiobench/fixture/corpus.py b/tests/studio/studiobench/fixture/corpus.pyindex 03ad263c82a..4702400568f 100644--- a/tests/studio/studiobench/fixture/corpus.py+++ b/tests/studio/studiobench/fixture/corpus.py@@ -1138,7 +1138,10 @@ def plan_rung(# than a tolerance: clipping at a whole BLOCK boundary loses at most one block and is what# keeps a prefix from ending inside a fence, while exceeding the unit loses everything above# it and is unbounded, and a percentage bound cannot tell those apart. The DEFAULT path is- # exempt on purpose, since `STREAM_TAIL_CHARS` is a ceiling the small rungs are under.+ # exempt on purpose, since `STREAM_TAIL_CHARS` is a ceiling the small rungs are legitimately+ # under: on that path `tail_target` is capped at `target_chars`, so a rung whose whole budget+ # is under one tail asks for less than one whatever the ratio, and that shortfall is the rung+ # being small rather than the corpus failing to answer.if stream_tail_chars is not None and tail_target >= source.chars:raise _tail_not_deliverable(rung,diff --git a/tests/studio/studiobench/instruments/cpuprofile.py b/tests/studio/studiobench/instruments/cpuprofile.pyindex f883737f126..37acf2b5a91 100644--- a/tests/studio/studiobench/instruments/cpuprofile.py+++ b/tests/studio/studiobench/instruments/cpuprofile.py@@ -225,7 +225,9 @@ def compare_with_trace_profile(# since a cross-check that can never fail reads as corroboration; and this instrument is only# meaningful at level 1, where the trace carries NO profiler categories and it gives stacks# without the ProfileChunk volume that dominates an L2 buffer (a real L1 capture had 1748 of 1748-# standalone samples inside the trace's own `RunTask` span). At level 2 and above it stands down+# standalone samples inside the trace's own `RunTask` span, so the standalone profile covers the+# same interval the trace does; it still carries no task tree, and `_summarise` ranks samples over+# the whole profile interval rather than per task window). At level 2 and above it stands down# and says so.import time # noqa: E402diff --git a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.pyindex d3d5a8c0f7a..66878afd32e 100644--- a/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py+++ b/tests/studio/studiobench/instruments/selftest/test_studiobench_streamcost_integrity.py@@ -124,9 +124,9 @@ def test_a_clean_stream_is_scoreable_and_counts_every_character(page):"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value: the id says which decoder the two numbers above are about, and the integer- # depends on how many decoders the page has built.- # Same footing as `carried_flushes` beside it.+ # Not pinned to a value, on the same footing as `carried_flushes` beside it: the id says which+ # decoder the two numbers above are about, and the exact integer depends on how many decoders+ # the page has built before this assertion."decoder_id": got["decoder_id"],}, got@@ -233,8 +233,9 @@ def test_the_counter_survives_a_split_inside_the_data_prefix(page):"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value: the id says which decoder the two numbers above are about.- # Same footing as `carried_flushes` beside it.+ # Not pinned to a value, on the same footing as `carried_flushes` beside it: the id says which+ # decoder the two numbers above are about, and the exact integer depends on how many decoders+ # the page has built before this assertion."decoder_id": got["decoder_id"],}, got@@ -273,8 +274,9 @@ def test_unrelated_text_ending_in_a_marker_letter_does_not_corrupt_the_next_fram"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value: the id says which decoder the two numbers above are about.- # Same footing as `carried_flushes` beside it.+ # Not pinned to a value, on the same footing as `carried_flushes` beside it: the id says which+ # decoder the two numbers above are about, and the exact integer depends on how many decoders+ # the page has built before this assertion."decoder_id": got["decoder_id"],}, got@@ -433,8 +435,9 @@ def test_an_aborted_frame_does_not_follow_the_stream_that_replaces_it(page):"failures": 0,"pending_chars": 0,"carried_flushes": got["carried_flushes"],- # Not pinned to a value: the id says which decoder the two numbers above are about.- # Same footing as `carried_flushes` beside it.+ # Not pinned to a value, on the same footing as `carried_flushes` beside it: the id says which+ # decoder the two numbers above are about, and the exact integer depends on how many decoders+ # the page has built before this assertion."decoder_id": got["decoder_id"],}, gotdiff --git a/tests/studio/studiobench/instruments/tracing.py b/tests/studio/studiobench/instruments/tracing.pyindex 23717693e7d..c23eb2a000f 100644--- a/tests/studio/studiobench/instruments/tracing.py+++ b/tests/studio/studiobench/instruments/tracing.py@@ -43,9 +43,10 @@ from typing import Any, Callable, Sequencefrom ..analysis import CellFailure-# L0 is the only level headline numbers may come from: nothing is attached beyond the renderer's own metrics counters.# --------------------------------------------------------------------- ladder+# L0 is the only level headline numbers may come from: nothing is attached to the renderer beyond+# the metrics counters it already maintains for itself.L0 = "L0"# L1 adds the timeline trace (task boundaries, frames, layout, user timing). No CPU profiler,# so no stacks and no naming, but the cheapest level giving a real task tree.diff --git a/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py b/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.pyindex 1f8aae053cd..3914a77fcf3 100644--- a/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py+++ b/tests/studio/studiobench/runtime/selftest/test_studiobench_run_acquisition.py@@ -433,14 +433,15 @@ def test_a_plain_run_and_a_plain_resume_are_unaffected(studio):)-# The external probe on the record. `SBENCH_EXTRA_INIT_SCRIPT` is an ENVIRONMENT variable, so it-# outlives the command that wanted it: a resume under a shell that still has it set runs the rungs-# still owed with the probe in the page and appends a probed `run_meta`, and `refuse_if_probed`-# reads every `run_meta` in a file, so cells recorded cleanly before it stop being scorable too,-# permanently. So `run_meta` has to carry the probe and `--resume` has to compare it.+# ── the external probe on the record ────────────────────────────────────────────────────────+#+# `SBENCH_EXTRA_INIT_SCRIPT` is an ENVIRONMENT variable, not a flag, so it outlives the command+# that wanted it: a resume under a shell that still has it set runs the rungs still owed with the+# probe in the page and appends a probed `run_meta`, and `refuse_if_probed` reads every `run_meta`+# in a file, so cells recorded cleanly before it stop being scorable too, permanently, because a+# payload is append-only. So `run_meta` has to carry the probe and `--resume` has to compare it.-# ── the external probe on the record ────────────────────────────────────────────────────────class _ProbedBundle(_Bundle):"""A probe run attaches console and `pageerror` listeners, which a `None` page cannot take."""diff --git a/tests/studio/studiobench/runtime/session.py b/tests/studio/studiobench/runtime/session.pyindex a5e1062066c..76404a1f169 100644--- a/tests/studio/studiobench/runtime/session.py+++ b/tests/studio/studiobench/runtime/session.py@@ -1031,8 +1031,10 @@ class CellRunner:# The harness layer's attestation, load-bearing rather than decorative:# `scoring/schema._walk_for_bare_zeros` rejects a bare zero with no sibling `*_attempted` flag,# and this block has legitimate zeros -- a thread with no mounted code blocks records- # `code_token_spans: 0`, and Chromium coarsens `performance.now()` to 100 us.- # Which is what `forced_layout_ms` is built from.+ # `code_token_spans: 0`, and `blur_inpage_ms` and `forced_layout_ms` come from+ # `performance.now()`, which Chromium coarsens to 100 us outside a cross-origin-isolated+ # context, so an operation shorter than that reads exactly 0. Without this flag `--report`+ # refuses the whole payload of a completed probe run."click_attribution_attempted": True,"first_touch_ms": decay[0],"settled_touch_ms": min(decay[1:]),diff --git a/tests/studio/studiobench/scene/actions.py b/tests/studio/studiobench/scene/actions.pyindex 70b0135e7b8..c69cbfe8003 100644--- a/tests/studio/studiobench/scene/actions.py+++ b/tests/studio/studiobench/scene/actions.py@@ -1638,8 +1638,12 @@ def image_upload(ctx: ActionContext) -> ActionResult:plus = Noneif plus is None:# WHY, not just THAT: a bare "not visible" conflates a button that is absent, one that is covered- # and a locator that disagrees with the page, and has already cost three wrong hypotheses.- # Carrying the probe state into the row means the next run answers it.+ # and a locator that disagrees with the page, and has already cost three wrong hypotheses. A+ # direct probe once found the control at 36x36, fully opaque and hit-testable, on a fresh chat,+ # after a settings round trip and under a 20,000-character composer fill, so those states alone+ # did not explain it then; a fixture that loads no model has no attachments button at all, and+ # nothing here rules out a change in the current build. Read the diagnostic below rather than+ # this note. Carrying the probe state into the row means the next run answers it.return not_run("no visible attachments button on the composer: "+ json.dumps(_ev(ctx, IMAGE_BUTTON_DIAGNOSTIC) or {})diff --git a/tests/studio/studiobench/scene/schedule.py b/tests/studio/studiobench/scene/schedule.pyindex 20bb0c5b3a9..8448a93cdf1 100644--- a/tests/studio/studiobench/scene/schedule.py+++ b/tests/studio/studiobench/scene/schedule.py@@ -138,9 +138,10 @@ QUICK = Scene(("reasoning_toggle", 26_500, 4_500),# 1,500 rather than 4,000, because of the gap AFTER it rather than the cost of the send:# `send_turn` is a sub-100 ms action, so a 4,000 ms window is permission to begin the follow-up- # four seconds late without recording a miss, and every one of those seconds comes off- # `message_menu`'s window. Measured from the latest legal send, this film left 3,500 ms for a- # 4,562 ms drain; the fast film had the same shape and CI failed on it.+ # four seconds late without recording a miss. The drain runs from when the send actually fires,+ # so every one of those seconds comes off `message_menu`'s window: measured from the latest+ # legal send, this film left 3,500 ms for a 4,562 ms drain; the fast film had the same shape+ # and CI failed on it.("send_turn", 31_500, 1_500),("message_menu", 36_000, 3_000),("copy_markdown", 39_500, 2_500),diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.pyindex 20cfb4c7ea9..dc08ea82b1f 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_copy_confirmed_live.py@@ -226,7 +226,8 @@ def test_the_guard_is_on_the_clipboard_and_not_on_the_engine_name():assert "sentinel" in src# EXECUTABLE LINES ONLY. The engine names belong in the comment explaining which engine was# observed failing and with what numbers; what must not exist is a BRANCH on one. The word- # "engine" is allowed because it appears in the refusal MESSAGE; a specific engine's NAME is not.+ # "engine" is allowed because it appears in the refusal MESSAGE; a specific engine's NAME is not,+ # because a name is what a branch would need.code = "\n".join(line for line in src.splitlines() if not line.lstrip().startswith("#"))for name in ("webkit", "WebKit", "firefox", "Firefox", "browser_name", "browser_type"):assert name not in code, f"the guard branches on {name!r} rather than on the clipboard"diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.pyindex 9fe2ff7bcd6..b8228e72a18 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_queued_idle_live.py@@ -298,14 +298,14 @@ def test_the_shipped_composer_still_renders_the_two_queue_buttons():assert 'aria-label="Stop queued message"' in src+# ── what the blind-probe refusal may NOT take out with it ────────────+#: An overlay is walked from `document`, OUTSIDE `.aui-thread-root`, so its digest carries neither#: the streamed message nor the composer, which makes it readable on a pair whose stream could not#: be placed.-# ── what the blind-probe refusal may NOT take out with it ────────────-_MENU = '<div role="menu"><div class="item">Rename</div></div>'_MENU_CHANGED = '<div role="menu"><div class="item">Rename thread</div></div>'-#:The composer of a thread that is NOT generating. `_STOP_BUTTON` is the same composer generating.+#: The composer of a thread that is NOT generating. `_STOP_BUTTON` is the same composer generating._SEND_BUTTON = ('<button class="aui-composer-send" aria-label="Send message">''<span class="aui-sr-only">Send message</span></button>'diff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py b/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.pyindex 455a38233c9..2bd4a79cbe9 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_stop_slot_budget.py@@ -77,13 +77,13 @@ from studiobench.scene.schedule import FAST, QUICK, STANDARD # noqa: E402#: charged separately below, so about 4 ms is the round trip itself.ROUND_TRIP_MS = 4.0-#:Enter pressed to `isRunning()` answering true: a POST to the relay and the first SSE frame back.+#: Enter pressed to `isRunning()` answering true: a POST to the relay and the first SSE frame back.START_MS = 120.0-#:Stop clicked to `isRunning()` answering false: the abort round trip.+#: Stop clicked to `isRunning()` answering false: the abort round trip.STOP_MS = 90.0-#:The delete inside `STOP_CLEANUP_JS`: the click, the removal, and the two `__sbNextPaint` frames the loop waits on.+#: The delete inside `STOP_CLEANUP_JS`: the click, the removal, and the two `__sbNextPaint` frames#: the loop waits on.CLEANUP_MS = 90.0@@ -116,7 +116,8 @@ class _Keyboard:return# SENT, NOT STARTED, and the difference is the whole of the P1 above: the app takes the text out of# the composer and puts the user turn and its reply into the thread NOW, while the reply begins- # generating `start_ms` later and the action has to poll for it.+ # generating `start_ms` later and the action has to poll for it - the cost the first version of+ # this shim handed out free.self._page.composer = ""self._page.messages += 2self._page.sent_at_ms = self._page.elapsed_msdiff --git a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.pyindex dac07a4b42b..047253268c2 100644--- a/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py+++ b/tests/studio/studiobench/scene/selftest/test_studiobench_visible_blind_probe_live.py@@ -252,14 +252,16 @@ def test_a_lost_conversation_is_still_a_finding_while_a_reply_runs(browser):assert "DIFFERENT MESSAGES on screen" in got["reason"]-# What the refusal may NOT take out with it. The blind-probe refusal is about rows whose meaning-# depends on where the stream had got to; two kinds provably do not: a row both arms call the-# user's, and a row whose ROLE changed. Both used to leave here as NOT COMPARABLE with an empty-# `moved`, and `visible_report` buckets a refusal as blind and never consults it for the exit-# code, so the run went green on them.+# ── what the refusal may NOT take out with it ────────────────────────+#+# The blind-probe refusal is about rows whose meaning depends on where the stream had got to. Two+# kinds of row provably do not: a row both arms call the user's, because a reply is written into an+# assistant message; and a row whose ROLE changed, because a role is captured beside the digest and+# how far a reply has arrived says nothing about whose it is. Both used to leave here as NOT+# COMPARABLE with an empty `moved`, and `visible_report` buckets a refusal as blind and never+# consults it for the exit code, so the run went green on them.-# ── what the refusal may NOT take out with it ────────────────────────def test_a_changed_user_row_survives_the_blind_refusal(browser):base = _capture(browser, **_SETTLED)treat = _capture(browser, **dict(_BLIND, user_body = "the user message, rewritten"))diff --git a/tests/studio/studiobench/sweep/floor_table.py b/tests/studio/studiobench/sweep/floor_table.pyindex 3d814067dc7..9f307dd682b 100644--- a/tests/studio/studiobench/sweep/floor_table.py+++ b/tests/studio/studiobench/sweep/floor_table.py@@ -780,9 +780,9 @@ def render(# null survived. Censoring removes the slow ones and `spread_pct` is `max - min` over the# survivors, so it can only narrow: on a real null at 100K plus 500K,# `reasoning_toggle.close_ms` gave an applied floor of 17.1% from four surviving pairs while- # the censored repetitions of that same null paired as far apart as 1.50 on identical builds,- # and a real result of -24.8% printed `faster` against it.- # They ran 1429-2149 ms against the survivors' 498-681 ms.+ # the censored repetitions of that same null ran 1429-2149 ms against those survivors'+ # 498-681 ms and paired as far apart as 1.50 on identical builds. A real result of -24.8%+ # printed `faster` against that 17.1%.floor_stat = None if floors is None else floors.get(metric)floor_partial = floor_stat is not None and floor_stat.get("poolable") is False# MARKED APART FROM `[*]`, because the two say different things: `[*]` means this row's owndiff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.pyindex 5e850e50467..cace3b0422f 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_null_audit.py@@ -782,13 +782,15 @@ def test_an_interrupted_retry_does_not_inherit_the_completion_it_superseded(tmp_assert ("r100K", "settings") not in unstable, unstable-# The audit block sat below the structural early return while reading nothing from the plan,-# so a payload with no structural entries returned first and it never ran; and `--mode-# visible` / `--mode behaviour` hard-set `structural` to empty, so those two could never audit-# at all.+# ── the audit has to run in the modes that ask for it ────────────────+#+# The audit block sat below the structural early return while reading nothing from the plan, so a+# payload with no structural entries returned first and the audit never ran. `--mode visible` and+# `--mode behaviour` hard-set `structural` to an empty set for every pattern, so those two could+# never audit at all: the command exited 0 out of the ordinary visible report, having silently+# skipped the option whose promise is to FAIL unless the null decided what the result needs.-# ── the audit has to run in the modes that ask for it ────────────────def test_the_cli_audits_a_single_repetition_under_forced_visible_mode(tmp_path, capsys):"""THE DEFECT. Same payload and same expectation as the auto-mode failure test above, with`--mode visible` added: the audit must still run and still fail."""diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.pyindex 801e3c54c2b..6637535f47e 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_parity_shots.py@@ -227,8 +227,9 @@ def test_shot_index_reads_the_payload_rather_than_globbing(tmp_path):def test_a_one_repetition_flake_is_not_illustrated_at_the_verdicts_threshold(tmp_path, capsys):- # `settings` differs on both passes and fails the job; `thread_reopen` differs on one and is- # explicitly uncorroborated. Shipping both leaves the reader unable to tell which is which.+ # `settings` differs on both passes and is what fails the job; `thread_reopen` differs on one and+ # is explicitly uncorroborated. Shipping both leaves the reader unable to tell which is which,+ # which buries the finding the artifact exists to show.shots = tmp_path / "shots"rows = [{"row_type": "run_meta", "tier": "fast"}]for rep in ("rep0", "rep1"):diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.pyindex e9ef3c0d7ea..210d7cbfbbc 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_sweep.py@@ -201,7 +201,7 @@ def test_a_count_is_harvested_under_a_name_that_marks_it_as_an_invariant(tmp_patdef test_a_count_that_fell_reads_as_a_loss_and_never_as_faster(tmp_path):# The regression this catches: virtualization truncates select-all from 400k chars to 3k.- # Every timing improves, expect_ok stays true, and only this row can say so.+ # Every timing improves, `expect_ok` stays true because chars > 0, and only this row can say so.stats = F.summarise([count_payload(tmp_path, "r", [(400000.0, 3000.0)] * 4)])floor = {"delta_pct": 0.0, "spread_pct": 1.0}_f, v = F.verdict_for(stats[COUNT_METRIC], floor, F.is_count_metric(COUNT_METRIC))@@ -412,7 +412,8 @@ def test_an_incomplete_cell_is_not_measured(tmp_path):def test_a_shard_pairs_within_itself_and_never_across(tmp_path):- # Both shards number repetitions from rep0; pairing on the repetition alone would cross them over.+ # Both shards number repetitions from rep0; pairing on the repetition alone would cross them over+ # and silently compare one session's base with another's treatment.a = payload(tmp_path / "a", "s0", [(1000.0, 500.0)])b = payload(tmp_path / "b", "s0", [(2000.0, 1000.0)])pooled, _ = F.load([a, b])@@ -683,7 +684,7 @@ def timed_action(cid: str, sid: str, ms: float) -> dict:def resumed_payload(tmp_path: Path, name: str, retry_session: str) -> Path:- """One arm completed, its partner died, and `--resume` re-ran the dead arm.+ """One arm completed, its partner died, and only the dead arm was re-run: a PARTIAL resume.`retry_session` is the session the retry was recorded under. The real `--resume` mints a newone; passing the original back is the control that shows the refusal keys on the session and@@ -722,9 +723,14 @@ def resumed_payload(tmp_path: Path, name: str, retry_session: str) -> Path:def test_an_arm_resumed_into_a_new_session_is_not_paired_with_the_old_one(tmp_path):- # `--resume` re-runs the dead arm under a NEW session id in the same shard; keyed on the- # repetition alone the two pair and the 8% drift is charged to the re-run arm.- # `scoring/ab.py` refuses exactly this.+ # A LEGACY PARTIAL-RESUME PAYLOAD: one arm completed under the first session id and only its+ # dead partner re-run under a NEW one, in the SAME shard directory. Today's `--resume` does not+ # write this shape, since `__main__._resume_set` passes the completed set through+ # `ab.skippable_cells`, which returns an EMPTY skip set while any A/B work remains, so both arms+ # are re-run in the new session. It reaches the sweep from an older writer or an externally+ # produced payload. Keyed on the repetition alone the two rows pair and the whole 8% session+ # drift is charged to the re-run arm. `scoring/ab.py` refuses exactly this comparison, and the+ # sweep's own pairing has to refuse it too.path = resumed_payload(tmp_path, "resumed", "s2")assert F.cell_metrics(F.read_rows(path))["r100K.treatment.rep0"] == {"message_menu.open_close_ms": 108.0diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.pyindex 5b4c0cecf08..069705777ee 100644--- a/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_windowed_parity.py@@ -1401,17 +1401,17 @@ def test_a_capture_that_saw_no_thread_at_all_falls_back_on_the_declaration(tmp_passert all(mode == U.WINDOWED for mode, _why in U.decide_modes([shard]).values())-# A cell that failed its own completeness gate carries no UI verdict. `probe_thread_completeness`-# runs before the film and `record_completeness_gate` writes the verdict against the cell, so a-# windowed arm that kept its first and last page and lost the middle says so in its own payload,-# and `report/payload.py::excluded_from_rows` drops it from the PERFORMANCE score. `ui_parity.py`-# read no gate row except the windowed declaration, so the same cell's eighteen action rows were-# still scored, and the visible region is a window on the END of the thread, which such a store-# still fills, so the pairs matched and `--mode auto` exited 0 over a payload that had already-# recorded the loss.+# ── a cell that failed its own completeness gate carries no UI verdict ──+#+# `probe_thread_completeness` runs before the film and `record_completeness_gate` writes the+# verdict against the cell, so a windowed arm that kept its first page and its last one and lost+# everything between them says so in its own payload, and `report/payload.py::excluded_from_rows`+# drops that cell from the PERFORMANCE score. `ui_parity.py` read no gate row except the windowed+# declaration, so the same cell's eighteen action rows were still scored for UI parity, and the+# visible region is a window on the END of the thread, which such a store still fills, so the+# pairs matched and `--mode auto` exited 0 over a payload that had already recorded the loss.-# ── a cell that failed its own completeness gate carries no UI verdict ──def _completeness_gate(cell_id,passed,