Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studiobench: compare a code fence one arm scrolled past on its text (#11741)
Daniel Han45f8fa65ecf0
diff --git a/tests/studio/studiobench/analysis/parity.py b/tests/studio/studiobench/analysis/parity.pyindex 6fa2e9ed307..b28f440d73e 100644--- a/tests/studio/studiobench/analysis/parity.py+++ b/tests/studio/studiobench/analysis/parity.py@@ -155,6 +155,94 @@ def in_flight(base: Optional[dict], treat: Optional[dict]) -> set[int]:return out+def _latched_fences(capture: dict) -> Optional[int]:+ """How many code fences this capture found LATCHED, or None when it read no fences at all."""+ total: Optional[int] = None+ for m in capture.get("messages") or []:+ fences = m.get("fences") if isinstance(m, dict) else None+ if isinstance(fences, list):+ total = (total or 0) + sum(+ 1 for f in fences if isinstance(f, dict) and f.get("latched")+ )+ return total+++def fence_latch_residue(+ base: Optional[dict],+ treat: Optional[dict],+ skip: Optional[set[int]] = None,+) -> list[int]:+ """Messages whose digests differ ONLY in which of their code fences had been scrolled past.++ A CODE FENCE'S HIGHLIGHT STATE IS SCROLL HISTORY. `code-fence-defer.tsx` renders a fence the+ reader has not come near as a plain shell (`data-unsloth-fence-deferred="true"`) and upgrades it+ to token spans the first time it comes within a viewport, one way only, for the life of the+ mount. So the same fence serialises as a shell on an arm whose viewport never passed it and as+ spans on one whose did, on ONE build. On the r100K fast film `reasoning_toggle` leaves the+ viewport either near the tail or ~20,000px higher, a coin flip per cell on the same build, and+ the higher landing latches every fence in msg11/13/15 for the rest of the cell. A null control+ whose four cells all happened to land low measured every later action stable, and backend-only+ pull requests failed on `msg15(assistant):1847269->2150610c`.++ THE RULE, per message, and every clause has to hold: the same role; the same number of fences;+ the message with every fence replaced by a marker (`digest_unfenced`) identical, so anything+ OUTSIDE a fence is still compared exactly; a fence latched on both arms, or on neither, identical+ in full, so a highlighting or shell regression on a fence both arms reached is still a+ difference; and a fence latched on ONE arm only identical in its language and its TEXT, which+ is read as lines in both forms, so spacing, indentation, string contents and where the lines+ break all still count. At least one fence must be in that last case, or the difference is not a+ latch.++ WHAT THIS GIVES UP, said plainly: the token markup of a fence that only ONE arm had latched is not+ compared, because it has no counterpart on the other arm to compare with. The same fence+ latched on both arms in another cell, or at the bottom of the thread where every cell latches it+ at mount, still carries that comparison.++ THE ONE REGRESSION THIS COULD OTHERWISE SWALLOW is a build whose fences never upgrade at all:+ every one of its fences reads as "not latched" against a base that latched some. So when one arm+ latched NOTHING anywhere in the thread and the other latched something, nothing is excused.++ A capture recorded before the fence fields existed carries none, and is scored exactly as before.+ """+ if not isinstance(base, dict) or not isinstance(treat, dict):+ return []+ bl, tl = _latched_fences(base), _latched_fences(treat)+ if bl is None or tl is None or (bl == 0) != (tl == 0):+ return []+ skip = skip or set()+ bm, tm = _messages(base), _messages(treat)+ out: list[int] = []+ for i in sorted(set(bm) & set(tm)):+ if i in skip:+ continue+ b, t = bm[i], tm[i]+ if b.get("digest") == t.get("digest") or b.get("role") != t.get("role"):+ continue+ bf, tf = b.get("fences"), t.get("fences")+ if not isinstance(bf, list) or not isinstance(tf, list) or not bf or len(bf) != len(tf):+ continue+ if b.get("digest_unfenced") is None or b.get("digest_unfenced") != t.get("digest_unfenced"):+ continue+ one_arm = False+ explained = True+ for x, y in zip(bf, tf):+ if not isinstance(x, dict) or not isinstance(y, dict) or x.get("lang") != y.get("lang"):+ explained = False+ break+ if bool(x.get("latched")) == bool(y.get("latched")):+ if x.get("digest") != y.get("digest"):+ explained = False+ break+ elif x.get("text") is None or x.get("text") != y.get("text"):+ explained = False+ break+ else:+ one_arm = True+ if explained and one_arm:+ out.append(i)+ return out++def _scaffold(capture: dict) -> tuple[Optional[str], Optional[int]]:"""The thread with every message elided, falling back to the whole-thread digest.@@ -512,6 +600,9 @@ def settled_messages_moved(base: dict, treat: dict) -> list[str]:"""bm, tm = _messages(base), _messages(treat)streaming = in_flight(base, treat)+ # A fence one arm had scrolled past and the other had not is scroll history, not a rendering:+ # see `fence_latch_residue`.+ latched = set(fence_latch_residue(base, treat, streaming))out: list[str] = []for i in sorted(set(bm) & set(tm)):b, t = bm[i], tm[i]@@ -520,7 +611,7 @@ def settled_messages_moved(base: dict, treat: dict) -> list[str]:continue# Flagged in flight by the arm that COULD place its stream; its digest is a point in a stream on# that arm whatever role it carries, so it is withheld like any other.- if i in streaming:+ if i in streaming or i in latched:continueif b.get("role") == "user" and b.get("digest") != t.get("digest"):out.append(f"msg{i}(user):{b.get('chars')}->{t.get('chars')}c")@@ -655,6 +746,11 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:# regression inside the streaming message lands as NOT COMPARABLE, and a REORDER past another# message of the same role is demoted from DIFFER (10 of 11 injected differences still DIFFER).streaming = in_flight(base, treat)+ # FENCES ONE ARM HAD SCROLLED PAST AND THE OTHER HAD NOT, compared on their text rather than their+ # token markup (see `fence_latch_residue`). NOT a refusal: everything these messages carry was+ # compared, and a message is only here when all of it agreed but the latch, so it can reach MATCH.+ latched = set(fence_latch_residue(base, treat, streaming))+ skip = streaming | latched# ── THE COMPOSER IS NOT A RENDERING DIFFERENCE ──────────────────────────────────────────────## One arm finished and one still writing has its MESSAGES withheld correctly, but the dock is@@ -684,7 +780,7 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:and _run_state_disagrees(base, treat)and scaffold_moved(base, treat)and not overlays_moved(base, treat)- and not _messages_moved(base, treat, streaming)+ and not _messages_moved(base, treat, skip)):bc, tc = _scaffold(base)[1], _scaffold(treat)[1]return {@@ -702,10 +798,11 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:),"moved": [],"in_flight": sorted(streaming),+ "fence_latch": sorted(latched),"style_verdict": style_verdict,"style_reason": style_reason,}- if not _any_moved(base, treat, streaming):+ if not _any_moved(base, treat, skip):bm, tm = _messages(base), _messages(treat)unsettled = sorted(i@@ -735,6 +832,7 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:),"moved": [],"in_flight": sorted(streaming),+ "fence_latch": sorted(latched),"not_digested": unsettled,"style_verdict": style_verdict,"style_reason": style_reason,@@ -750,14 +848,16 @@ def compare(base: Optional[dict], treat: Optional[dict]) -> dict:"reason": "","moved": [],"in_flight": sorted(streaming),+ "fence_latch": sorted(latched),"style_verdict": style_verdict,"style_reason": style_reason,}return {"verdict": DIFFER,"reason": "",- "moved": localise(base, treat, streaming),+ "moved": localise(base, treat, skip),"in_flight": sorted(streaming),+ "fence_latch": sorted(latched),"style_verdict": style_verdict,"style_reason": style_reason,}diff --git a/tests/studio/studiobench/scene/parity.js b/tests/studio/studiobench/scene/parity.jsindex 486ead53cf7..8030e5d0725 100644--- a/tests/studio/studiobench/scene/parity.js+++ b/tests/studio/studiobench/scene/parity.js@@ -93,13 +93,62 @@return normText(raw);};+ // A CODE FENCE'S HIGHLIGHT STATE IS SCROLL HISTORY, NOT RENDERING. `code-fence-defer.tsx` (#9462,+ // default since #9567) renders an unreached fence as a plain shell carrying+ // `data-unsloth-fence-deferred="true"` and upgrades it to token spans the first time it comes within+ // one viewport of the reader, one way only, for the rest of its mount. So two runs of ONE build that+ // scrolled differently serialise the same fence as a shell on one arm and as spans on the other.+ // Observed on the r100K fast film after `reasoning_toggle`: the viewport either stays near the tail+ // (scrollTop ~50,740) or lands ~20,000px higher (~28,000-34,000), in 78 of 172 recorded cells, on+ // one build. The higher landing latches the fences of msg11/13/15, their signatures grow from+ // 174,956/65,402/325,362 to 283,962/373,775/~2,000,000 characters, and every later action in the+ // cell carries that until `thread_reopen` remounts the thread. A null control whose cells all+ // happened to stay low then calls those actions stable, and a backend-only pull request fails.+ // So each message also carries its fences, read in the SAME walk: whether each was latched, its+ // full digest, its language, and a digest of its TEXT alone, plus a digest of the message with+ // every fence replaced by a marker. The text is read as LINES, since that is all the two forms+ // disagree on: the shell is one text node split by newlines, the highlighted fence one element+ // per line with none between them (a blank line holding a lone "\n"). Both are brought to lines+ // joined by one newline, trailing blank lines dropped, and every other character is kept as+ // written, so `x = 1` against `x=1`, an indentation change, a space inside a string, a line break+ // moved within the code and a changed duration or id literal all still differ. `analysis/parity.fence_latch_residue` compares a fence+ // latched on both arms, or on neither, in full, and one latched on ONE arm on its language and+ // text.+ const FENCE_ATTR = "data-streamdown";+ const FENCE_ROOT = "code-block";+ const FENCE_BODY = "code-block-body";+ const FENCE_DEFERRED_ATTR = "data-unsloth-fence-deferred";+ const FENCE_MARKER = "<!fence>";++ // Marks where the walk met an element directly inside a fence body's <code>: one per line of a+ // highlighted fence. The shell's <code> holds only text, so it never carries one.+ const FENCE_LINE = "\u0001";+ const fenceText = (raw) => {+ const s = raw || "";+ const lines = s.includes(FENCE_LINE)+ ? s+ .split(FENCE_LINE)+ .filter((line, k) => k > 0 || line !== "")+ .map((line) => line.replace(/\r?\n/g, ""))+ : s.split(/\r?\n/);+ // Not normText: its placeholders are for UI prose, and inside code `"295ms"`, a timestamp or an id literal is+ // the content itself.+ return lines.join("\n").replace(/\n+$/, "");+ };+// `dropAttrs` is a Set of attributes this digest does not compare AT ALL, unlike VOLATILE_ATTRS// which keeps the presence. `elide` is a Set of ELEMENTS whose subtree is not serialised: a marker// carrying the tag and `data-role` goes in their place, so presence, position and role still// compare and a vanished message still moves the digest. Both off by default.- const signature = (root, dropAttrs, elide) => {- if (!root) return "";+ // `fences`, when an array, receives one `{start, end, latched, text}` per OUTERMOST code fence met,+ // `start`/`end` indexing the returned parts; it changes nothing about the parts themselves, so the+ // digest a caller already compares is byte-identical with or without it.+ const walkParts = (root, dropAttrs, elide, fences) => {+ if (!root) return [];const parts = [];+ // The fence being walked, if any: nested fences are part of their outer fence's reading.+ let fence = null;+ let inBody = false;const walk = (el, depth) => {// A depth cap is a TRUNCATION, so the marker is left in the signature and anything deeper reads// as "not walked" rather than as absent.@@ -112,6 +161,27 @@);return;}+ // Opened BEFORE the tag is written, so `start` covers the fence's own element.+ let opened = false;+ let bodyHere = false;+ if (fences && el.getAttribute) {+ const role = el.getAttribute(FENCE_ATTR);+ if (fence === null && role === FENCE_ROOT) {+ fence = {+ start: parts.length,+ latched: el.getAttribute(FENCE_DEFERRED_ATTR) !== "true",+ // On the fence root in both forms, so a language change is seen without the markup.+ lang: el.getAttribute("data-language"),+ all: "",+ body: null,+ };+ opened = true;+ } else if (fence !== null && !inBody && role === FENCE_BODY) {+ if (fence.body === null) fence.body = "";+ inBody = true;+ bodyHere = true;+ }+ }parts.push("<" + el.tagName.toLowerCase());const names = [];for (const attr of el.attributes) {@@ -142,17 +212,70 @@};for (const child of el.childNodes) {if (child.nodeType === 3) {- run += child.nodeValue == null ? "" : child.nodeValue;+ const v = child.nodeValue == null ? "" : child.nodeValue;+ run += v;+ if (fence !== null) {+ fence.all += v;+ if (inBody) fence.body += v;+ }} else if (child.nodeType === 1) {flush();+ if (fence !== null && inBody && el.tagName === "CODE") fence.body += FENCE_LINE;walk(child, depth + 1);}}flush();parts.push("</" + el.tagName.toLowerCase() + ">");+ if (bodyHere) inBody = false;+ if (opened) {+ // THE BODY'S TEXT WHEN THERE IS A BODY: the header carries the language label on both forms,+ // but a highlighted fence also mounts a copy and download bar the shell does not.+ fences.push({+ start: fence.start,+ end: parts.length,+ latched: fence.latched,+ lang: fence.lang,+ text: fenceText(fence.body === null ? fence.all : fence.body),+ });+ fence = null;+ }};walk(root, 0);- return parts.join("");+ return parts;+ };++ const signature = (root, dropAttrs, elide) => walkParts(root, dropAttrs, elide).join("");++ // ONE WALK, THREE READINGS of a message: the signature every caller already compares, unchanged;+ // each fence's latch state, full digest and text digest; and the message with every fence+ // replaced by one marker, so what surrounds the fences is still compared exactly.+ // `fences` is absent from the result when the message has none, which keeps a fence-free payload+ // byte-identical to what this instrument wrote before.+ const messageReading = (el) => {+ const marks = [];+ const parts = walkParts(el, undefined, undefined, marks);+ const sig = parts.join("");+ const out = { sig };+ if (marks.length) {+ const rest = [];+ let at = 0;+ const fences = [];+ for (const m of marks) {+ for (let k = at; k < m.start; k++) rest.push(parts[k]);+ rest.push(FENCE_MARKER);+ at = m.end;+ fences.push({+ latched: m.latched,+ lang: m.lang,+ digest: hash(parts.slice(m.start, m.end).join("")),+ text: hash(m.text),+ });+ }+ for (let k = at; k < parts.length; k++) rest.push(parts[k]);+ out.fences = fences;+ out.digest_unfenced = hash(rest.join(""));+ }+ return out;};// The bounded computed-style probe: three properties on at most `STYLE_CAP` elements, digested@@ -511,6 +634,7 @@normText,normUrl,signature,+ messageReading,hash,// One digest for the whole thread plus one PER MESSAGE, so a mismatch says which message moved@@ -571,13 +695,20 @@const messages = [];const in_flight = [];for (let i = 0; i < nodes.length; i++) {- const sig = signature(nodes[i]);+ const reading = messageReading(nodes[i]);+ const sig = reading.sig;const row = {i,role: nodes[i].getAttribute("data-role") || "?",digest: hash(sig),chars: sig.length,};+ // See FENCE_ATTR above: what lets the comparison tell a fence the reader scrolled past on one+ // arm from a fence that renders differently.+ if (reading.fences) {+ row.fences = reading.fences;+ row.digest_unfenced = reading.digest_unfenced;+ }if (inFlight.has(nodes[i])) {row.in_flight = true;in_flight.push(i);diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_fence_latch_parity.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_fence_latch_parity.pynew file mode 100644index 00000000000..ca28cc0f51a--- /dev/null+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_fence_latch_parity.py@@ -0,0 +1,581 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""A code fence the reader scrolled past on one arm is not a UI change.++THE FALSE ALARM THIS HOLDS. `code-fence-defer.tsx` renders a code fence nobody has come near as a+plain shell and upgrades it to token spans, one way only, the first time it comes within a viewport.+On the r100K fast film `reasoning_toggle` leaves the viewport either near the tail or ~20,000px+higher, at random per cell on ONE build, and the higher landing latches every fence in msg11/13/15+for the rest of the cell. Their serialisations then differ by up to 1.7 million characters between+two runs of the same build. A null control whose four cells all happened to land low called every+later action stable, and two backend-only pull requests failed the gate on 2026-09-23:++ #11727 run 35913031644 model_change / reasoning_toggle / select_all_copy / settings,+ msg15(assistant):1847269->2150610c, null control quiet+ #11724 run 35913021155 image_upload, msg11(assistant):174956->283962c, the null control+ could not decide image_upload (one arm missed its slot)++`testdata/` holds both runs' payloads as recorded by CI, trimmed to the fields the verdict reads+(the verdict text is byte-identical on the trimmed and the full files). They predate the per-fence+readings, so the first tests replay the failure as it happened. The rest add the readings that+`scene/parity.js` now takes, modelled on the mechanism: in the messages whose size moved with the+census's highlight-span count (msg11/13/15/17), each distinct digest is a distinct set of latched+fences over identical fence TEXT and an identical message outside the fences. That model is the+claim under test; the live evidence for it is a null run recorded with the new capture, and the+DOM-level half is `test_the_capture_reads_a_shell_and_its_highlighted_fence_as_one_text` below.++Every "clears" test is paired with one that injects a genuine difference into the same recorded+payload and requires it to still fail, because a fix for a false alarm that also silences real+changes is the worse bug.+"""++from __future__ import annotations++import gzip+import json+import shutil+import subprocess+import sys+from pathlib import Path++import pytest++sys.path.insert(0, str(Path(__file__).resolve().parents[4]))++from tests.studio.studiobench.analysis import parity as P # noqa: E402+from tests.studio.studiobench.sweep import ui_parity as U # noqa: E402++TESTDATA = Path(__file__).resolve().parent / "testdata"+PARITY_JS = Path(__file__).resolve().parents[2] / "scene" / "parity.js"++#: The messages whose serialised size tracked `census.highlight_spans` in the recorded cells.+LATCHING = (11, 13, 15, 17)+#: Fences per latching message in the model. Fence 0 is the one every cell latches at mount.+FENCES = 12+++def _load(name: str) -> list[dict]:+ with gzip.open(TESTDATA / name, "rt", encoding = "utf-8") as fh:+ return [json.loads(line) for line in fh if line.strip()]+++def _write(rows: list[dict], path: Path) -> Path:+ path.mkdir(parents = True, exist_ok = True)+ with open(path / "payload.jsonl", "w", encoding = "utf-8") as fh:+ for row in rows:+ fh.write(json.dumps(row) + "\n")+ return path+++def _captures(rows: list[dict]):+ for row in rows:+ if row.get("row_type") == "action" and isinstance(row.get("parity"), dict):+ yield row, row["parity"]+++def annotate(*payloads: list[dict]) -> None:+ """Add the fence readings `scene/parity.js` now records, IN PLACE, across one run's payloads.++ Built over BOTH payloads of a run together, so a digest seen in the null control and in the+ result maps to the same latch set in each.+ """+ seen: dict[int, list[str]] = {i: [] for i in LATCHING}+ for rows in payloads:+ for _row, cap in _captures(rows):+ for m in cap.get("messages") or []:+ i = m.get("i")+ if i in seen and m.get("digest") not in seen[i]:+ seen[i].append(m["digest"])+ for rows in payloads:+ for _row, cap in _captures(rows):+ for m in cap.get("messages") or []:+ i = m.get("i")+ if i not in seen:+ continue+ j = seen[i].index(m["digest"])+ assert j < 2 ** (FENCES - 1), "more distinct digests than latch sets"+ m["fences"] = [+ {+ "latched": latched,+ "digest": f"f{i}.{k}.{'hl' if latched else 'shell'}",+ "text": f"t{i}.{k}",+ }+ for k in range(FENCES)+ for latched in [k == 0 or bool((j >> (k - 1)) & 1)]+ ]+ m["digest_unfenced"] = f"u{i}"+++def _run(tmp_path, capsys, result, null) -> tuple[int, str]:+ """The workflow's verdict step, flag for flag: (exit code, what it printed)."""+ rdir = _write(result, tmp_path / "parity-result")+ ndir = _write(null, tmp_path / "parity-null-control")+ rc = U.main(["--min-reps", "2", "--min-compared", "16", "--null", str(ndir), str(rdir)])+ return rc, capsys.readouterr().out+++def _stable_section(out: str) -> str:+ """The lines under the heading the exit code is taken from."""+ head = "UI PARITY DIFFERENCES ON STABLE ACTIONS"+ if head not in out:+ return ""+ block = out.split(head, 1)[1]+ lines = []+ for line in block.splitlines()[1:]:+ if not line.startswith(" "):+ break+ lines.append(line)+ return "\n".join(lines)+++def _runs():+ return [+ pytest.param(+ "pr11727",+ {"model_change", "reasoning_toggle", "select_all_copy", "settings"},+ id = "11727-quiet-null",+ ),+ pytest.param("pr11724", {"image_upload"}, id = "11724-undecided-null"),+ ]+++def _recorded(tag: str) -> tuple[list[dict], list[dict]]:+ return _load(f"{tag}_result.payload.jsonl.gz"), _load(f"{tag}_null.payload.jsonl.gz")+++# ── the failure, replayed from what CI recorded ──────────────────────────────────────────────++[email protected]("tag,flagged", _runs())+def test_the_recorded_run_reproduces_the_false_alarm(tmp_path, capsys, tag, flagged):+ """As recorded, without fence readings, the verdict fails exactly as CI did."""+ result, null = _recorded(tag)+ rc, out = _run(tmp_path, capsys, result, null)+ assert rc == 1, out+ section = _stable_section(out)+ for action in flagged:+ assert action in section, (action, section)+ # And every one of them is an assistant message in the latching set, which is the whole finding.+ for line in section.splitlines():+ for claim in line.split(": ", 1)[1].split(", "):+ assert any(claim.startswith(f"msg{i}(assistant)") for i in LATCHING), line++[email protected]("tag,flagged", _runs())+def test_fence_readings_clear_the_false_alarm(tmp_path, capsys, tag, flagged):+ """With the latch recorded, every one of those differences is a latch and the run passes."""+ result, null = _recorded(tag)+ annotate(result, null)+ rc, out = _run(tmp_path, capsys, result, null)+ assert rc == 0, out+ assert _stable_section(out) == ""+ # Said, and not paid for in coverage: a latch-only pair is a comparison, not a refusal, so the+ # --min-compared floor (exit 3) is not what turned this green.+ assert "of which fence-latch only" in out++[email protected]("tag,flagged", _runs())+def test_the_workflow_null_audit_still_passes_with_fence_readings(tmp_path, capsys, tag, flagged):+ result, null = _recorded(tag)+ annotate(result, null)+ rdir = _write(result, tmp_path / "parity-result")+ ndir = _write(null, tmp_path / "parity-null-control")+ rc = U.main(+ [+ "--audit-null",+ "--allow-undecided",+ "image_upload",+ "--compared-in",+ str(rdir),+ "--min-reps",+ "2",+ str(ndir),+ ]+ )+ assert rc == 0, capsys.readouterr().out+++# ── the other direction: a real change on the same recorded payload still fails ────────────────+++def _treatment_rows(rows: list[dict], action: str):+ for row, cap in _captures(rows):+ if row.get("action") == action and ".treatment." in row.get("cell_id", ""):+ yield row, cap+++def _msg(cap: dict, i: int) -> dict:+ return next(m for m in cap["messages"] if m["i"] == i)++[email protected]("tag,flagged", _runs())+def test_a_real_message_change_on_a_stable_action_still_fails(tmp_path, capsys, tag, flagged):+ """A message with no fences, rendered differently by head in both repetitions of `settings`."""+ result, null = _recorded(tag)+ annotate(result, null)+ hit = 0+ for _row, cap in _treatment_rows(result, "settings"):+ m = _msg(cap, 3)+ m["digest"] = "0badc0de"+ m["chars"] += 17+ hit += 1+ assert hit == 2+ rc, out = _run(tmp_path, capsys, result, null)+ assert rc == 1, out+ assert "settings" in _stable_section(out) and "msg3(assistant)" in _stable_section(out)++[email protected]("tag,flagged", _runs())+def test_a_change_outside_the_fences_of_a_latched_message_still_fails(+ tmp_path, capsys, tag, flagged+):+ """msg15 differs by latch AND by something outside its fences: the latch cannot excuse it."""+ result, null = _recorded(tag)+ annotate(result, null)+ for _row, cap in _treatment_rows(result, "settings"):+ m = _msg(cap, 15)+ m["digest_unfenced"] = "u15-changed"+ m["digest"] = m["digest"] + "x"+ rc, out = _run(tmp_path, capsys, result, null)+ assert rc == 1, out+ assert "msg15(assistant)" in _stable_section(out)++[email protected]("tag,flagged", _runs())+def test_a_highlighting_change_on_a_fence_both_arms_latched_still_fails(+ tmp_path, capsys, tag, flagged+):+ """Fence 0 is latched on every arm, so its token markup is compared in full."""+ result, null = _recorded(tag)+ annotate(result, null)+ for _row, cap in _treatment_rows(result, "settings"):+ m = _msg(cap, 17)+ m["fences"][0]["digest"] = "f17.0.hl-changed"+ m["digest"] = m["digest"] + "x"+ rc, out = _run(tmp_path, capsys, result, null)+ assert rc == 1, out+ assert "msg17(assistant)" in _stable_section(out)++[email protected]("tag,flagged", _runs())+def test_a_build_whose_fences_never_upgrade_is_not_excused(tmp_path, capsys, tag, flagged):+ """Every fence a shell on head: the one regression the text comparison alone would swallow."""+ result, null = _recorded(tag)+ annotate(result, null)+ for row, cap in _captures(result):+ if ".treatment." not in row.get("cell_id", ""):+ continue+ for m in cap["messages"]:+ if "fences" in m:+ m["fences"] = [+ {**f, "latched": False, "digest": f"f{m['i']}.{k}.shell"}+ for k, f in enumerate(m["fences"])+ ]+ m["digest"] = f"never-{m['i']}"+ rc, out = _run(tmp_path, capsys, result, null)+ assert rc == 1, out+ assert "msg17(assistant)" in _stable_section(out)+++# ── the rule itself ──────────────────────────────────────────────────────────────────────────+++def _cap(+ fences: list[dict],+ unfenced: str = "u",+ digest: str = "d",+ role: str = "assistant",+):+ msg = {"i": 0, "role": role, "digest": digest, "chars": 10}+ if fences is not None:+ msg["fences"] = fences+ msg["digest_unfenced"] = unfenced+ return {+ "parity_attempted": True,+ "root_kind": "thread",+ "digest": digest,+ "digest_scaffold": "s",+ "chars_scaffold": 5,+ "messages": [msg],+ "overlays": [],+ "styles": {"digest": "st", "elements": 3, "capped": False},+ }+++def _f(+ latched: bool,+ text: str = "t",+ digest: str | None = None,+ lang: str = "python",+) -> dict:+ return {+ "latched": latched,+ "lang": lang,+ "text": text,+ "digest": digest or ("hl" if latched else "sh"),+ }+++def test_a_latch_only_difference_is_a_match_not_a_refusal():+ base = _cap([_f(True), _f(False)], digest = "a")+ treat = _cap([_f(True), _f(True)], digest = "b")+ got = P.compare(base, treat)+ assert got["verdict"] == P.MATCH, got+ assert got["fence_latch"] == [0]+++ "treat_fences,unfenced,why",+ [+ ([_f(True), _f(True, text = "other")], "u", "the one-sided fence's text changed"),+ ([_f(True, digest = "hl2"), _f(True)], "u", "a fence latched on both arms changed"),+ ([_f(True)], "u", "a fence vanished"),+ ([_f(True), _f(True)], "u2", "something outside the fences changed"),+ ([_f(True), _f(False, digest = "sh2")], "u", "no fence changed latch; the shell did"),+ (+ [_f(True), _f(True, lang = "javascript")],+ "u",+ "the one-sided fence's language changed",+ ),+ ],+)+def test_anything_but_the_latch_still_differs(treat_fences, unfenced, why):+ base = _cap([_f(True), _f(False)], digest = "a")+ treat = _cap(treat_fences, unfenced = unfenced, digest = "b")+ got = P.compare(base, treat)+ assert got["verdict"] == P.DIFFER, why+ assert got["fence_latch"] == [], why+++def test_one_arm_latching_nothing_anywhere_is_never_excused():+ base = _cap([_f(True), _f(False)], digest = "a")+ treat = _cap([_f(False), _f(False)], digest = "b")+ assert P.fence_latch_residue(base, treat) == []+ assert P.compare(base, treat)["verdict"] == P.DIFFER+++def test_a_payload_recorded_before_the_fence_readings_is_scored_as_before():+ base, treat = _cap(None, digest = "a"), _cap(None, digest = "b")+ assert P.fence_latch_residue(base, treat) == []+ assert P.compare(base, treat)["verdict"] == P.DIFFER+ # And a message whose digest agrees is never reported, whatever its fences say.+ same = _cap([_f(True)], digest = "a")+ assert P.fence_latch_residue(same, _cap([_f(False)], digest = "a")) == []+++def test_the_derived_null_counts_a_latch_only_pair_as_stable():+ """`derive_unstable` reads `compare`, so a latch-only null pair is an observation of stability."""+ base = _cap([_f(True), _f(False)], digest = "a")+ treat = _cap([_f(True), _f(True)], digest = "b")+ row = P.derive_unstable([("settings", P.compare(base, treat))] * 2)["settings"]+ assert row["unstable"] is False and row["observations"] == 2+++# ── the capture, the real parity.js under node ───────────────────────────────────────────────++HARNESS_JS = r"""+const fs = require("fs");+const src = fs.readFileSync(process.argv[2], "utf8");+const window = {};+const document = { body: { tagName: "BODY", attributes: [], childNodes: [],+ getAttribute: () => null },+ querySelectorAll: () => [] };+window.getComputedStyle = () => ({ getPropertyValue: () => "" });+(new Function("window", "document", src))(window, document);+const build = (spec) => {+ if (typeof spec === "string") return { nodeType: 3, nodeValue: spec };+ const attrs = spec.attrs || {};+ return {+ nodeType: 1,+ tagName: (spec.tag || "div").toUpperCase(),+ attributes: Object.keys(attrs).map((name) => ({ name })),+ getAttribute: (name) => (name in attrs ? attrs[name] : null),+ childNodes: (spec.children || []).map(build),+ };+};+const P = window.__sb.parity;+const specs = JSON.parse(fs.readFileSync(process.argv[3], "utf8"));+const out = specs.map((s) => {+ const el = build(s);+ const r = P.messageReading(el);+ return { fences: r.fences || null, digest_unfenced: r.digest_unfenced || null,+ digest: P.hash(r.sig), same_as_signature: r.sig === P.signature(build(s)) };+});+process.stdout.write(JSON.stringify(out));+"""++CODE = "def f(x):\n return x + 1\n\nprint(f(2))"+++def _shell(code: str) -> dict:+ """What `code-fence-defer.tsx`'s FenceShell renders for a fence nobody has reached."""+ return {+ "attrs": {+ "data-streamdown": "code-block",+ "data-unsloth-fence-deferred": "true",+ "data-language": "python",+ "class": "my-4 flex",+ },+ "children": [+ {+ "attrs": {"data-streamdown": "code-block-header", "data-language": "python"},+ "children": [{"tag": "span", "children": ["python"]}],+ },+ {+ "attrs": {"data-streamdown": "code-block-body", "data-language": "python"},+ "children": [{"tag": "pre", "children": [{"tag": "code", "children": [code]}]}],+ },+ ],+ }+++def _highlighted(code: str, lang: str = "python") -> dict:+ """streamdown's highlighted fence: one span per line, NO newline between lines, a span per token."""+ lines = []+ for line in code.split("\n"):+ tokens = [t for t in line.replace(" ", "\0 \0").split("\0") if t] or ["\n"]+ lines.append(+ {+ "tag": "span",+ "attrs": {"class": "block"},+ "children": [+ {"tag": "span", "attrs": {"style": "--sdm-c:#fff"}, "children": [t]}+ for t in tokens+ ],+ }+ )+ return {+ "attrs": {+ "data-streamdown": "code-block",+ "data-language": lang,+ "class": "my-4 flex",+ "style": "content-visibility:auto",+ },+ "children": [+ {+ "attrs": {"data-streamdown": "code-block-header", "data-language": lang},+ "children": [{"tag": "span", "children": [lang]}],+ },+ {+ "attrs": {"data-streamdown": "code-block-actions"},+ "children": [{"tag": "button", "attrs": {"title": "Copy Code"}}],+ },+ {+ "attrs": {"data-streamdown": "code-block-body", "data-language": lang},+ "children": [{"tag": "pre", "children": [{"tag": "code", "children": lines}]}],+ },+ ],+ }+++def _message(fence: dict, prose: str = "Here is the function.") -> dict:+ return {+ "attrs": {"data-role": "assistant"},+ "children": [{"tag": "p", "children": [prose]}, fence, {"tag": "p", "children": ["Done."]}],+ }+++def _node_readings(tmp_path: Path, specs: list[dict]) -> list[dict]:+ exe = shutil.which("node") or shutil.which("nodejs")+ if not exe:+ pytest.skip("node is not installed, so the shipped parity.js could not be evaluated")+ harness = tmp_path / "harness.js"+ harness.write_text(HARNESS_JS, encoding = "utf-8")+ spec_file = tmp_path / "specs.json"+ spec_file.write_text(json.dumps(specs), encoding = "utf-8")+ got = subprocess.run(+ [exe, str(harness), str(PARITY_JS), str(spec_file)],+ capture_output = True,+ text = True,+ timeout = 60,+ check = True,+ )+ return json.loads(got.stdout)+++def test_the_capture_reads_a_shell_and_its_highlighted_fence_as_one_text(tmp_path):+ shell, lit, edited, reworded, bare = _node_readings(+ tmp_path,+ [+ _message(_shell(CODE)),+ _message(_highlighted(CODE)),+ _message(_highlighted(CODE.replace("x + 1", "x + 2"))),+ _message(_shell(CODE), prose = "Here is a different sentence."),+ {+ "attrs": {"data-role": "assistant"},+ "children": [{"tag": "p", "children": ["no code"]}],+ },+ ],+ )+ # The false alarm, reproduced at the DOM: one fence, two serialisations.+ assert shell["digest"] != lit["digest"]+ assert shell["fences"][0]["latched"] is False and lit["fences"][0]["latched"] is True+ # ...and the reading that resolves it: same text, same message around it.+ assert shell["fences"][0]["text"] == lit["fences"][0]["text"]+ assert shell["digest_unfenced"] == lit["digest_unfenced"]+ # The other direction: a changed line of code and a changed sentence are both still seen.+ assert edited["fences"][0]["text"] != lit["fences"][0]["text"]+ assert reworded["digest_unfenced"] != shell["digest_unfenced"]+ # The digest every existing caller compares is unchanged, and a fence-free message carries no+ # new fields at all.+ assert all(r["same_as_signature"] for r in (shell, lit, edited, reworded, bare))+ assert bare["fences"] is None and bare["digest_unfenced"] is None+++def test_the_capture_keeps_spacing_indentation_and_language_in_a_fence(tmp_path):+ """Only line breaks separate the two forms, so only line breaks are dropped from the text."""+ lit, spaced, dedented, quoted, other_lang = _node_readings(+ tmp_path,+ [+ _message(_highlighted(CODE)),+ _message(_shell(CODE.replace("x + 1", "x+1"))),+ _message(_shell(CODE.replace(" return", " return"))),+ _message(_shell(CODE.replace("f(2)", "f( 2)"))),+ _message(_highlighted(CODE, lang = "javascript")),+ ],+ )+ for got, why in (+ (spaced, "spaces around an operator"),+ (dedented, "an indentation change"),+ (quoted, "a space inside the call"),+ ):+ assert got["fences"][0]["latched"] is False and lit["fences"][0]["latched"] is True+ assert got["fences"][0]["text"] != lit["fences"][0]["text"], why+ assert other_lang["fences"][0]["text"] == lit["fences"][0]["text"]+ assert other_lang["fences"][0]["lang"] == "javascript" and lit["fences"][0]["lang"] == "python"+++def test_the_capture_keeps_where_a_fence_breaks_its_lines(tmp_path):+ """Lines, not a run of characters: a line break moved within the code is a different fence."""+ lit, moved, merged, trailing, blank_lit, blank_shell = _node_readings(+ tmp_path,+ [+ _message(_highlighted("a\nbc")),+ _message(_shell("ab\nc")),+ _message(_shell("abc")),+ _message(_shell("a\nbc\n\n")),+ _message(_highlighted("a\n\n\nbc")),+ _message(_shell("a\n\n\nbc")),+ ],+ )+ assert moved["fences"][0]["text"] != lit["fences"][0]["text"], "a line break moved"+ assert merged["fences"][0]["text"] != lit["fences"][0]["text"], "two lines joined"+ # The shell trims trailing newlines; that is not a difference.+ assert trailing["fences"][0]["text"] == lit["fences"][0]["text"]+ # Blank lines render as a line holding a lone newline, and count once, as in the shell.+ assert blank_lit["fences"][0]["text"] == blank_shell["fences"][0]["text"]+ assert blank_lit["fences"][0]["text"] != lit["fences"][0]["text"]+++def test_the_capture_keeps_literals_that_look_volatile_in_prose(tmp_path):+ """The time and id placeholders are for UI prose; inside code a literal is the content."""+ lit, duration, clock = _node_readings(+ tmp_path,+ [+ _message(_highlighted('timeout = "295ms"\nat = "10:30"')),+ _message(_shell('timeout = "310ms"\nat = "10:30"')),+ _message(_shell('timeout = "295ms"\nat = "11:45"')),+ ],+ )+ assert duration["fences"][0]["text"] != lit["fences"][0]["text"], "a duration literal changed"+ assert clock["fences"][0]["text"] != lit["fences"][0]["text"], "a time literal changed"diff --git a/tests/studio/studiobench/sweep/selftest/testdata/pr11724_null.payload.jsonl.gz b/tests/studio/studiobench/sweep/selftest/testdata/pr11724_null.payload.jsonl.gznew file mode 100644index 00000000000..5b2e8cab26cBinary files /dev/null and b/tests/studio/studiobench/sweep/selftest/testdata/pr11724_null.payload.jsonl.gz differdiff --git a/tests/studio/studiobench/sweep/selftest/testdata/pr11724_result.payload.jsonl.gz b/tests/studio/studiobench/sweep/selftest/testdata/pr11724_result.payload.jsonl.gznew file mode 100644index 00000000000..ffc299cab76Binary files /dev/null and b/tests/studio/studiobench/sweep/selftest/testdata/pr11724_result.payload.jsonl.gz differdiff --git a/tests/studio/studiobench/sweep/selftest/testdata/pr11727_null.payload.jsonl.gz b/tests/studio/studiobench/sweep/selftest/testdata/pr11727_null.payload.jsonl.gznew file mode 100644index 00000000000..5e499c0d12fBinary files /dev/null and b/tests/studio/studiobench/sweep/selftest/testdata/pr11727_null.payload.jsonl.gz differdiff --git a/tests/studio/studiobench/sweep/selftest/testdata/pr11727_result.payload.jsonl.gz b/tests/studio/studiobench/sweep/selftest/testdata/pr11727_result.payload.jsonl.gznew file mode 100644index 00000000000..716729c4446Binary files /dev/null and b/tests/studio/studiobench/sweep/selftest/testdata/pr11727_result.payload.jsonl.gz differdiff --git a/tests/studio/studiobench/sweep/ui_parity.py b/tests/studio/studiobench/sweep/ui_parity.pyindex 590a0a6478b..5b44b5ac0cc 100644--- a/tests/studio/studiobench/sweep/ui_parity.py+++ b/tests/studio/studiobench/sweep/ui_parity.py@@ -1699,6 +1699,16 @@ def report(f"cell rows, so completion could not be checked")print(f" matched: {matched}")+ # SAID, not silent: a pair that matched only once fence latch state was set aside is a weaker+ # match than one whose every byte agreed, and a reader should be able to count them.+ latch_normalised = sum(+ 1 for _a, _s, _c, r in results if r.get("verdict") == P.MATCH and r.get("fence_latch")+ )+ if latch_normalised:+ print(+ f" of which fence-latch only: {latch_normalised} (the only difference was which code "+ f"fences each arm had scrolled past; those fences were compared on their text)"+ )print(f" stable actions differing: {len(stable_bad)}"+ (f" (in >= {min_reps} repetitions)" if min_reps > 1 else "")