Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

43.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Two captures are MATCH, DIFFER or NOT COMPARABLE; the last is never folded into MATCH."""

from __future__ import annotations

import collections
from typing import Any, Iterable, Optional

# The digest walks only the thread: MATCH means no structure change, not an unchanged UI.
MATCH = "match"
DIFFER = "differ"
NOT_COMPARABLE = "not_comparable"
NOT_EXERCISED = "not_exercised"
# NOT_APPLICABLE: the question is wrong for this pair (an arm meant to put less DOM on screen).
# Distinct so `derive_unstable` counts neither as evidence.
NOT_APPLICABLE = "not_applicable"

# A key on a NOT_COMPARABLE result, not a verdict: all agreed except a mid-tail reply. Only
# `derive_unstable` reads it.
SETTLED_MATCH = "settled_match"

# Actions that legitimately differ between two runs of one build. The mechanism is the value so a
# test can require one; `derive_unstable` cross-checks each.
UNSTABLE_ACTIONS: dict[str, str] = {
    "stop_generation": "stops a live stream, so how many characters arrived before the stop is a race with the "
    "network and differs run to run on one build. NOT reproduced by the 100K fast-tier null "
    "control (0 of 4), so this entry rests on the mechanism rather than on a measurement",
    "send_turn": "starts a new turn, so how far it has streamed when the digest is taken is wall clock. "
    "Measured differing on 2 of 4 base-vs-base pairs, with assistant_chars differing too",
    "scroll_during_generation": "the digest is taken after a scroll gesture against a growing thread; where it comes to "
    "rest depends on the autoscroll observer, which is the mechanism under study. Measured "
    "differing on 3 of 4 base-vs-base pairs",
    "scroll_after": "same gesture against a settled thread; the resting offset still depends on the observer. "
    "NOT reproduced by the 100K fast-tier null control (0 of 4); kept on the mechanism alone",
    # These four were found by the null control.
    "keystroke": "types characters into the composer over wall clock, and the composer's text is in the "
    "DOM, so how many keystrokes had landed by the capture deadline is a race. Measured 4 of "
    "4 differing, by 8 signature characters, with assistant_chars identical",
    "composer_fill": "fills the composer with a fixed string, but whether the send button has re-enabled by "
    "capture time is a race. A bare `disabled` serialises as exactly ten characters and the "
    "observed delta was exactly ten, in both directions across repetitions",
    "copy_markdown": "on the packed films this slot opens while the turn started by the preceding send_turn is "
    "still streaming, so the last assistant message is mid-tail. Measured 3 of 4 differing, "
    "with assistant_chars differing on every one",
    "message_menu": "same cause as copy_markdown: the slot lands inside the previous send_turn's stream. "
    "Measured 3 of 4 differing, with assistant_chars differing",
    "select_text": "same cause again, and the worst of the three at 4 of 4 differing; the selection is taken "
    "over a message whose tail is still arriving",
}


# Racy actions key on the not_run reason; a treatment removing a control must still be caught.
RACY_EXECUTION: dict[str, tuple[str, tuple[str, ...]]] = {
    "stop_generation": (
        "needs a live stream to stop, and returns not_run when nothing was generating and a new "
        "turn did not start within 8s (scene/actions.py:493). Whether a stream was in flight on "
        "this arm at this moment is a race with the model, not a property of the build. Its "
        "OTHER not_run, the stop button being absent, is the build and is not exempt",
        ("nothing was generating",),
    ),
    "scroll_during_generation": (
        "returns not_run with 'nothing was generating, so this is not a scroll during "
        "generation' (scene/actions.py:334), and that is its only not_run path",
        ("nothing was generating",),
    ),
    "send_turn": (
        "returns not_run when 'a reply was still streaming, so this send would have been queued' "
        "(scene/actions.py:993), which is the previous turn's stream overrunning rather than "
        "this build being unable to send. Its composer-absent and queue-exhausted paths are not "
        "exempt",
        ("a reply was still streaming",),
    ),
}


def racy_execution(action: str, reason: str) -> bool:
    """Is THIS not-run a stream-timing race, rather than merely on a racy action?"""
    entry = RACY_EXECUTION.get(action)
    if entry is None:
        return False
    return any(marker in (reason or "") for marker in entry[1])


def _messages(capture: dict) -> dict[int, dict]:
    return {m["i"]: m for m in (capture.get("messages") or []) if "i" in m}


def in_flight(base: Optional[dict], treat: Optional[dict]) -> set[int]:
    """Union, not intersection: a message still being written on either arm has no moment to compare."""
    out: set[int] = set()
    for side in (base, treat):
        if isinstance(side, dict):
            for i in side.get("in_flight") or []:
                if isinstance(i, int):
                    out.add(i)
    return out


def _latched_fences(capture: dict) -> Optional[int]:
    """How many code fences this capture found LATCHED, or None when it read no fences at all."""
    total: Optional[int] = None
    for m in capture.get("messages") or []:
        fences = m.get("fences") if isinstance(m, dict) else None
        if isinstance(fences, list):
            total = (total or 0) + sum(
                1 for f in fences if isinstance(f, dict) and f.get("latched")
            )
    return total


def fence_latch_residue(
    base: Optional[dict],
    treat: Optional[dict],
    skip: Optional[set[int]] = None,
) -> list[int]:
    """Excuses fence highlight differences from scroll history, unless one arm never upgraded any fence."""
    if not isinstance(base, dict) or not isinstance(treat, dict):
        return []
    bl, tl = _latched_fences(base), _latched_fences(treat)
    if bl is None or tl is None or (bl == 0) != (tl == 0):
        return []
    skip = skip or set()
    bm, tm = _messages(base), _messages(treat)
    out: list[int] = []
    for i in sorted(set(bm) & set(tm)):
        if i in skip:
            continue
        b, t = bm[i], tm[i]
        if b.get("digest") == t.get("digest") or b.get("role") != t.get("role"):
            continue
        bf, tf = b.get("fences"), t.get("fences")
        if not isinstance(bf, list) or not isinstance(tf, list) or not bf or len(bf) != len(tf):
            continue
        if b.get("digest_unfenced") is None or b.get("digest_unfenced") != t.get("digest_unfenced"):
            continue
        one_arm = False
        explained = True
        for x, y in zip(bf, tf):
            if not isinstance(x, dict) or not isinstance(y, dict) or x.get("lang") != y.get("lang"):
                explained = False
                break
            if bool(x.get("latched")) == bool(y.get("latched")):
                if x.get("digest") != y.get("digest"):
                    explained = False
                    break
            elif x.get("text") is None or x.get("text") != y.get("text"):
                explained = False
                break
            else:
                one_arm = True
        if explained and one_arm:
            out.append(i)
    return out


def _scaffold(capture: dict) -> tuple[Optional[str], Optional[int]]:
    """Thread digest with every message elided; falls back to the whole-thread digest for old captures."""
    if "digest_scaffold" in capture:
        return capture.get("digest_scaffold"), capture.get("chars_scaffold")
    return capture.get("digest"), capture.get("chars")


def _overlays(capture: dict) -> list[dict]:
    return list(capture.get("overlays") or [])


def overlays_moved(base: dict, treat: dict) -> list[str]:
    """Overlays sit outside the thread root, so they are checked even when the stream is unplaced."""
    bo, to = _overlays(base), _overlays(treat)
    if len(bo) != len(to):
        return [f"overlays {len(bo)}->{len(to)}"]
    return [
        f"overlay{k}[{b.get('sel')}]:{b.get('chars')}->{t.get('chars')}c"
        for k, (b, t) in enumerate(zip(bo, to))
        if b.get("digest") != t.get("digest")
    ]


def windowed_mount(capture: Optional[dict]) -> bool:
    """Read from the capture's own counts, not a caller flag, so no arm is scored full by omission."""
    if not isinstance(capture, dict):
        return False
    mounted, total = capture.get("mounted_messages"), capture.get("thread_total")
    if not isinstance(mounted, int) or not isinstance(total, int):
        return False
    return total > mounted


def applicability(base: Optional[dict], treat: Optional[dict]) -> Optional[str]:
    """Explains why the digest is the wrong question for a windowed arm; an answer, not a failure."""
    if not isinstance(base, dict) or not isinstance(treat, dict):
        return None
    bw, tw = windowed_mount(base), windowed_mount(treat)
    if bw or tw:
        which = "both arms" if bw and tw else ("the base arm" if bw else "the treatment arm")
        return (
            f"{which} mounts a WINDOW of the thread "
            f"(base {base.get('mounted_messages')} of {base.get('thread_total')}, treatment "
            f"{treat.get('mounted_messages')} of {treat.get('thread_total')} messages mounted). "
            "A structural DOM digest compares what is on screen, and this arm changes what is on "
            "screen by design, so the comparison cannot distinguish the intended change from an "
            "unintended one. Score this pair on behavioural invariants instead"
        )
    return None


def mount_count_mismatch(base: Optional[dict], treat: Optional[dict]) -> Optional[str]:
    """Non-windowed arms with unequal mounted counts have lost messages: a difference, never a refusal."""
    if not isinstance(base, dict) or not isinstance(treat, dict):
        return None
    bm, tm = base.get("mounted_messages"), treat.get("mounted_messages")
    if not isinstance(bm, int) or not isinstance(tm, int) or bm == tm:
        return None
    return (
        f"the arms mounted different numbers of messages ({bm} vs {tm}) and NEITHER arm is "
        "windowing, so this is not a difference of what is kept in the DOM on purpose -- one side "
        "is rendering messages the other is not. The per-message rows are keyed by position in "
        "the mounted list, so they are not quoted here: with different lengths every row would "
        "read as moved"
    )


def comparability(base: Optional[dict], treat: Optional[dict]) -> Optional[str]:
    """Why these two captures may NOT be compared, or None if they may be."""
    for label, side in (("base", base), ("treatment", treat)):
        if not isinstance(side, dict):
            return f"{label} recorded no parity capture at all"
        if not side.get("parity_attempted"):
            return (
                f"{label} could not be captured: " f"{side.get('reason') or 'no reason recorded'}"
            )
    assert base is not None and treat is not None
    # Missing on both sides is an old payload; on one side it is two instrument versions.
    bk, tk = base.get("root_kind"), treat.get("root_kind")
    if (bk is None) != (tk is None):
        return (
            "the two arms were captured by different versions of the parity instrument "
            f"(root_kind base={bk!r} treatment={tk!r})"
        )
    if bk is not None and bk != tk:
        return (
            f"the arms digested different roots (base={bk}, treatment={tk}); a body-root "
            "capture includes the sidebar and header and is not comparable with a thread one"
        )
    return None


def streaming_probe(base: dict, treat: dict) -> Optional[str]:
    """Refuses the stream split when a reply ran but no message published a streaming state."""
    for label, side in (("base", base), ("treatment", treat)):
        if isinstance(side, dict) and side.get("in_flight_unplaced"):
            return (
                f"a reply was running on the {label} arm and no message published a streaming "
                "state, so the streamed message could not be identified and its digest cannot be "
                "held apart from the settled thread. This is a probe that needs fixing (the "
                "`data-status` / `aria-busy` hooks in scene/dom.js), not a settled thread"
            )
    return None


def localise(
    base: dict,
    treat: dict,
    skip: Optional[set[int]] = None,
) -> list[str]:
    """Lists differences but skips in-flight messages: a mid-stream digest is a point, not a rendering."""
    skip = skip or set()
    moved: list[str] = []
    bm, tm = _messages(base), _messages(treat)
    for i in sorted(set(bm) | set(tm)):
        if i in skip and i in bm and i in tm:
            continue
        if i not in bm:
            moved.append(f"msg{i}({tm[i].get('role', '?')}):only treatment")
        elif i not in tm:
            moved.append(f"msg{i}({bm[i].get('role', '?')}):only base")
        elif bm[i].get("digest") != tm[i].get("digest"):
            moved.append(
                f"msg{i}({bm[i].get('role', '?')}):" f"{bm[i].get('chars')}->{tm[i].get('chars')}c"
            )

    moved.extend(overlays_moved(base, treat))

    if not moved:
        bc, tc = _scaffold(base)[1], _scaffold(treat)[1]
        moved.append(f"thread scaffolding outside any message ({bc}->{tc}c)")
    return moved


def compare_styles(base: dict, treat: dict) -> tuple[str, str]:
    """Reported apart: it alone sees stylesheet changes, and is most likely caught mid-transition."""
    bs, ts = base.get("styles"), treat.get("styles")
    if not isinstance(bs, dict) or not isinstance(ts, dict):
        return NOT_COMPARABLE, "one or both arms carry no style probe"
    if bs.get("capped") or ts.get("capped"):
        return NOT_COMPARABLE, (
            f"the probe hit its element cap "
            f"(base={bs.get('elements')}, treatment={ts.get('elements')})"
        )
    # Positive control: two empty scans have equal counts and digests and would MATCH on nothing.
    if not bs.get("elements") or not ts.get("elements"):
        return NOT_COMPARABLE, (
            f"the style probe matched no elements (base={bs.get('elements')}, "
            f"treatment={ts.get('elements')}), so it observed nothing on at least one arm. Its "
            "selector list is written against Unsloth's markup and does not survive a rename; "
            "this is a probe that needs fixing, not two arms that agree"
        )
    # STYLE_SELECTORS includes Send and Stop, so differing run state moves the digest on identical
    # threads. NOT_COMPARABLE rather than MATCH, since one aggregate digest covers everything.
    if (bs.get("elements") != ts.get("elements") or bs.get("digest") != ts.get("digest")) and (
        generation_disagrees(base, treat) and _run_state_disagrees(base, treat)
    ):
        return NOT_COMPARABLE, (
            "the two arms rendered different composer run-state controls "
            f"({base.get('composer_control')!r} against {treat.get('composer_control')!r}) and the "
            "run state itself says why, so the style probe walked a different selector for that "
            "one button -- its signature carries the selector that matched. The difference is the "
            "control swap and not a stylesheet change. The digest is ONE aggregate over every "
            "matched element, so a real CSS difference elsewhere cannot be separated from it here; "
            "this is a reading withheld, not a pass"
        )
    if bs.get("elements") != ts.get("elements"):
        return DIFFER, (
            f"the probe matched a different number of elements "
            f"({bs.get('elements')} vs {ts.get('elements')})"
        )
    if bs.get("digest") == ts.get("digest"):
        return MATCH, ""
    return DIFFER, f"display/visibility/pointer-events differ over {bs.get('elements')} elements"


def generation_disagrees(base: dict, treat: dict) -> bool:
    """Reads composer_control, since Stop and Queue both report isRunning yet render different controls."""
    bc, tc = base.get("composer_control"), treat.get("composer_control")
    if bc is not None and tc is not None:
        return bc != tc
    return bool(base.get("streaming")) != bool(treat.get("streaming"))


def _run_state_disagrees(base: dict, treat: dict) -> bool:
    """A second read via streaming and queued_idle; it cannot catch a wrong control."""
    return bool(base.get("streaming")) != bool(treat.get("streaming")) or bool(
        base.get("queued_idle")
    ) != bool(treat.get("queued_idle"))


def _messages_moved(
    base: dict,
    treat: dict,
    skip: Optional[set[int]] = None,
) -> bool:
    """The message half of `_any_moved`, on its own."""
    skip = skip or set()
    bm, tm = _messages(base), _messages(treat)
    # Mid-reply excuses a digest, never an absence.
    if set(bm) != set(tm):
        return True
    return any(bm[i].get("digest") != tm[i].get("digest") for i in bm if i not in skip)


def scaffold_moved(base: dict, treat: dict) -> bool:
    """Did the thread scaffolding -- viewport, composer, empty state -- move?"""
    return _scaffold(base)[0] != _scaffold(treat)[0]


def settled_messages_moved(base: dict, treat: dict) -> list[str]:
    """Moved rows provably not the streamed reply: user rows both arms agree on, or role-changed rows."""
    bm, tm = _messages(base), _messages(treat)
    streaming = in_flight(base, treat)
    # A fence scrolled past on one arm only is scroll history, not rendering; see fence_latch_residue.
    latched = set(fence_latch_residue(base, treat, streaming))
    out: list[str] = []
    for i in sorted(set(bm) & set(tm)):
        b, t = bm[i], tm[i]
        if b.get("role") != t.get("role"):
            out.append(f"msg{i}:role {b.get('role')}->{t.get('role')}")
            continue
        if i in streaming or i in latched:
            continue
        if b.get("role") == "user" and b.get("digest") != t.get("digest"):
            out.append(f"msg{i}(user):{b.get('chars')}->{t.get('chars')}c")
    # No assistant-row rule here: renaming the `data-status` hook moves every assistant digest too.
    return out


def _any_moved(
    base: dict,
    treat: dict,
    skip: Optional[set[int]] = None,
) -> bool:
    """Checks overlay digests too: a whole-thread digest alone let a changed menu still report MATCH."""
    skip = skip or set()
    # The scaffold, not the whole-thread digest, which differs on essentially every in-flight pair.
    if _scaffold(base)[0] != _scaffold(treat)[0]:
        return True
    bo, to = _overlays(base), _overlays(treat)
    if [(o.get("sel"), o.get("digest")) for o in bo] != [
        (o.get("sel"), o.get("digest")) for o in to
    ]:
        return True
    return _messages_moved(base, treat, skip)


def compare(base: Optional[dict], treat: Optional[dict]) -> dict:
    """One base/treatment pair -> {verdict, reason, moved, style_verdict, style_reason}."""
    why = comparability(base, treat)
    if why is not None:
        return {
            "verdict": NOT_COMPARABLE,
            "reason": why,
            "moved": [],
            "style_verdict": NOT_COMPARABLE,
            "style_reason": why,
        }
    assert base is not None and treat is not None
    not_applicable = applicability(base, treat)
    if not_applicable is not None:
        return {
            "verdict": NOT_APPLICABLE,
            "reason": not_applicable,
            "moved": [],
            "style_verdict": NOT_APPLICABLE,
            "style_reason": not_applicable,
        }
    lost = mount_count_mismatch(base, treat)
    if lost is not None:
        style_verdict, style_reason = compare_styles(base, treat)
        return {
            "verdict": DIFFER,
            "reason": lost,
            "moved": [],
            "style_verdict": style_verdict,
            "style_reason": style_reason,
        }
    style_verdict, style_reason = compare_styles(base, treat)
    blind = streaming_probe(base, treat)
    if blind is not None:
        # Overlays are walked from `document`, so they are a finding regardless of the stream. The scaffold
        # is not consulted here: the composer is inside it and Stop vs Send moves it.
        independent = overlays_moved(base, treat)
        # Settled user rows and role changes cannot be the reply being written.
        independent = independent + settled_messages_moved(base, treat)
        # The scaffold only when both arms agree on running, corroborated off the run state:
        # `generation_disagrees` reads the composer itself.
        if scaffold_moved(base, treat) and not (
            generation_disagrees(base, treat) and _run_state_disagrees(base, treat)
        ):
            bc, tc = _scaffold(base)[1], _scaffold(treat)[1]
            independent = independent + [f"thread scaffolding outside any message ({bc}->{tc}c)"]
        if independent:
            return {
                "verdict": DIFFER,
                "reason": (
                    "the streamed message could not be identified on one arm, but surfaces that "
                    "CANNOT be the reply being written differ: an overlay is walked outside the "
                    "thread root, so its digest carries neither the stream nor the composer, and a "
                    "row both arms call the user's, or a row whose role changed, is not the "
                    f"message a stream writes into. {blind}"
                ),
                "moved": independent,
                "style_verdict": style_verdict,
                "style_reason": style_reason,
            }
        return {
            "verdict": NOT_COMPARABLE,
            "reason": blind,
            "moved": [],
            "style_verdict": style_verdict,
            "style_reason": style_reason,
        }
    # In-flight messages are refused, not normalised. Order matters: settled differ is DIFFER; all
    # agree is MATCH; only in-flight differs is NOT COMPARABLE.
    streaming = in_flight(base, treat)
    # Not a refusal: these messages agreed on everything but the latch.
    latched = set(fence_latch_residue(base, treat, streaming))
    skip = streaming | latched
    # Composer sits in .aui-thread-root, so Stop vs Send moves the scaffold; run state must corroborate.
    if (
        generation_disagrees(base, treat)
        and _run_state_disagrees(base, treat)
        and scaffold_moved(base, treat)
        and not overlays_moved(base, treat)
        and not _messages_moved(base, treat, skip)
    ):
        bc, tc = _scaffold(base)[1], _scaffold(treat)[1]
        return {
            "verdict": NOT_COMPARABLE,
            "reason": (
                "the only difference is the thread scaffolding "
                f"({bc}->{tc}c), and one arm was running a reply while the other was not. The "
                "composer dock is inside the thread root and renders Stop or Queue while a reply "
                "is running and Send when it is not, so the scaffolding differs BECAUSE the two "
                "arms were at different points in the same turn. Every message and every overlay "
                "agreed. The scaffold is read as ONE AGGREGATE digest -- viewport, composer dock "
                "and empty state together -- so this cannot separate the composer swap from a "
                "change elsewhere in the scaffold that happened at the same time, and it does not "
                "claim to have: that is part of why it refuses. Nothing here is a pass"
            ),
            "moved": [],
            "in_flight": sorted(streaming),
            "fence_latch": sorted(latched),
            "style_verdict": style_verdict,
            "style_reason": style_reason,
        }
    if not _any_moved(base, treat, skip):
        bm, tm = _messages(base), _messages(treat)
        unsettled = sorted(
            i
            for i in streaming
            if i in bm and i in tm and bm[i].get("digest") != tm[i].get("digest")
        )
        if unsettled:
            names = ", ".join(
                f"msg{i}({bm[i].get('role', '?')}):{bm[i].get('chars')}->{tm[i].get('chars')}c"
                for i in unsettled[:4]
            )
            # `_any_moved` cannot see a role change on an in-flight row.
            roles_agree = all(bm[i].get("role") == tm[i].get("role") for i in set(bm) & set(tm))
            out = {
                "verdict": NOT_COMPARABLE,
                "reason": (
                    f"the settled thread is identical on both arms and the only difference is in "
                    f"{len(unsettled)} message(s) that were STILL BEING WRITTEN when the digest "
                    f"was taken ({names}). The two arms are two cells against one pacer with their "
                    "own send clicks and their own paint clocks, so that digest names a point in a "
                    "stream rather than a rendering, and its size carries no information either: "
                    "the renderer repairs the half-arrived construct, so the serialisation is not "
                    "monotonic in how much text has landed. Nothing here is a pass"
                ),
                "moved": [],
                "in_flight": sorted(streaming),
                "fence_latch": sorted(latched),
                "not_digested": unsettled,
                "style_verdict": style_verdict,
                "style_reason": style_reason,
            }
            if roles_agree:
                # The only refusal that also carries a positive reading (see `SETTLED_MATCH`).
                out[SETTLED_MATCH] = True
            return out
        return {
            "verdict": MATCH,
            "reason": "",
            "moved": [],
            "in_flight": sorted(streaming),
            "fence_latch": sorted(latched),
            "style_verdict": style_verdict,
            "style_reason": style_reason,
        }
    return {
        "verdict": DIFFER,
        "reason": "",
        "moved": localise(base, treat, skip),
        "in_flight": sorted(streaming),
        "fence_latch": sorted(latched),
        "style_verdict": style_verdict,
        "style_reason": style_reason,
    }


def execution_verdict(base_row: Optional[dict], treat_row: Optional[dict]) -> Optional[dict]:
    """Verdict the digest can never give when one arm did not run; shared so the paths cannot drift."""
    ran: dict[str, bool] = {}
    for label, row in (("base", base_row), ("treatment", treat_row)):
        if not isinstance(row, dict):
            return {
                "verdict": NOT_COMPARABLE,
                "reason": f"the {label} arm has no row for this action",
                "moved": [],
                "one_sided": "",
                "idle_reason": "",
                "style_verdict": NOT_COMPARABLE,
                "style_reason": "",
            }
        ran[label] = bool(row.get("ran"))
    if all(ran.values()):
        return None
    live = [label for label in ("base", "treatment") if ran[label]]
    label = "base" if not ran["base"] else "treatment"
    row = base_row if label == "base" else treat_row
    assert isinstance(row, dict)
    reason = row.get("reason") or "no reason recorded"
    # A missed slot is not a build difference, even on one arm; a precondition failure is.
    missed_slot = bool(row.get("slot_missed"))
    detail = (
        f"the action did not run on the {label} arm ({reason}), so any "
        "agreement here is about a surface nothing touched"
    )
    if live and not missed_slot:
        detail = (
            f"the action RAN on the {live[0]} arm and could not be performed on the "
            f"{label} arm ({reason}), so the two builds did not behave the same way"
        )
    if missed_slot:
        live = []
    return {
        "verdict": NOT_EXERCISED,
        "moved": [],
        "one_sided": live[0] if live else "",
        # The raw not_run string: the exemption matches what the action recorded.
        "idle_reason": reason,
        "reason": detail,
        "style_verdict": NOT_EXERCISED,
        "style_reason": "",
    }


def expect_regression(base_row: Optional[dict], treat_row: Optional[dict]) -> tuple[str, str]:
    """One arm failing its own expect_ok while the other passes; ran alone cannot see that."""
    rows_ = (("base", base_row), ("treatment", treat_row))
    if not all(isinstance(row, dict) for _label, row in rows_):
        return "", ""
    failed = [label for label, row in rows_ if row.get("expect_ok") is False]  # type: ignore[union-attr]
    passed = [label for label, row in rows_ if row.get("expect_ok") is True]  # type: ignore[union-attr]
    if not (len(failed) == 1 and passed):
        return "", ""
    who = failed[0]
    row = base_row if who == "base" else treat_row
    assert isinstance(row, dict)
    return who, (
        f"the action ran on both arms and its own assertion failed on the {who} arm "
        f"({row.get('reason') or 'no reason recorded'}), so the two builds did not behave "
        "the same way"
    )


def compare_rows(base_row: Optional[dict], treat_row: Optional[dict]) -> dict:
    """An action that ran on neither arm is not a pass; one arm unable to perform it is a difference."""
    idle = execution_verdict(base_row, treat_row)
    if idle is not None:
        return idle
    assert base_row is not None and treat_row is not None
    out = compare(base_row.get("parity"), treat_row.get("parity"))
    out["expect_regressed"], out["expect_reason"] = expect_regression(base_row, treat_row)
    return out


def derive_unstable(
    pairs: Iterable[tuple[str, dict]], *, min_observations: int = 2
) -> dict[str, dict]:
    """Derives empirical instability from a null control; min_observations stops a flake hiding a signal."""
    seen: dict[str, int] = collections.Counter()
    differ: dict[str, int] = collections.Counter()
    blind: dict[str, int] = collections.Counter()
    settled: dict[str, int] = collections.Counter()
    for action, result in pairs:
        verdict = result.get("verdict")
        if verdict not in (MATCH, DIFFER):
            if verdict == NOT_COMPARABLE and result.get(SETTLED_MATCH):
                # An observation of non-difference, tallied separately.
                seen[action] += 1
                settled[action] += 1
                continue
            # Neither may count as an observation of stability.
            blind[action] += 1
            continue
        seen[action] += 1
        if verdict == DIFFER:
            differ[action] += 1
    out: dict[str, dict] = {}
    for action in sorted(set(seen) | set(blind)):
        n, d = seen[action], differ[action]
        out[action] = {
            "observations": n,
            "differed": d,
            "not_comparable": blind[action],
            SETTLED_MATCH: settled[action],
            # Complete comparisons only: a settled-match refusal must not mint an exemption.
            "unstable": bool(d and (n - settled[action]) >= min_observations),
            # Same count `unstable` used, or one DIFFER beside one settled match reads as stable.
            "undetermined": (n - settled[action] if d else n) < min_observations,
        }
    return out


def cross_check(derived: dict[str, dict], declared: Iterable[str]) -> dict[str, list[str]]:
    """Reports mismatches both ways, derived vs declared unstable lists; each is a finding."""
    declared = set(declared)
    unstable = {a for a, r in derived.items() if r["unstable"]}
    undetermined = {a for a, r in derived.items() if r["undetermined"]}
    return {
        "declared_stable_in_practice": sorted(
            a for a in declared if a in derived and a not in unstable and a not in undetermined
        ),
        "unstable_but_not_declared": sorted(unstable - declared),
        "declared_but_never_observed": sorted(a for a in declared if a not in derived),
        "undetermined": sorted(undetermined),
    }


def mutation_detected(before: dict, after: dict) -> dict:
    """Uses production compare(), so the mutation check tests the real comparison, not a copy."""
    result = compare(before, after)
    result["detected"] = result["verdict"] == DIFFER
    return result


def summarise(results: Iterable[dict[str, Any]]) -> dict[str, int]:
    """Tally of verdicts, with NOT_COMPARABLE counted as its own outcome rather than as a pass."""
    tally: dict[str, int] = collections.Counter()
    for r in results:
        tally[r.get("verdict", NOT_COMPARABLE)] += 1
    return dict(tally)


# Policy: preserve UI/UX idempotency except for an on-the-record performance win, off-screen-only
# differences, and select-all shortfalls when the copy stays complete.

# The claim each verdict makes: the digest never covers sidebar, geometry or custom properties.
CLAIM_STRUCTURAL = (
    "thread-structure parity: the thread root and the declared overlay selectors serialise "
    "identically on both arms, on screen and off. It does NOT cover the sidebar, computed layout "
    "or geometry, or CSS custom properties, so it is not a statement that the UI is unchanged: "
    "run against a real sidebar-drag change this digest reported 0 of 34 differing pairs, and so "
    "did its own null control"
)
CLAIM_VISIBLE = (
    "visible-region parity: every message the viewport showed at any point during the action is "
    "present on both arms and identical between them, and every difference lies off screen"
)
CLAIM_BEHAVIOURAL = (
    "behavioural parity: the scroll extent matches and the invariants a windowed mount breaks "
    "first still hold, including that each arm's clipboard covers the thread's own visible text "
    "BY LENGTH. Says NOTHING about how anything looks, and nothing about WHICH characters were "
    "copied"
)

# Printed beside each verdict. Only `--mode behaviour` grants the copy exemption, scored by length
# against the thread with the live coverage band.
POLICY = (
    "UI and UX idempotency is required, with three exemptions: a deliberate difference accepted "
    "for a dramatic performance improvement, a difference that exists only OFF SCREEN, and a "
    "select-all that does not select all PROVIDED the copy it produces stays complete"
)

POLICY_BY_MODE = {
    "structural": (
        f"{POLICY}. This mode judges the FIRST requirement over the thread root and the declared "
        "overlays only. It cannot grant the off-screen exemption, because it does not know what "
        "was on screen: an arm that renders only the visible region reads as a difference here "
        "and needs --mode visible to get a verdict rather than a refusal"
    ),
    "visible": (
        f"{POLICY}. This mode is the one that can GRANT the off-screen exemption: it compares "
        "only what the viewport actually showed, so an off-screen-only difference passes on its "
        "merits rather than being argued about. The exemption changes what counts as a pass; it "
        "does not remove the floor, so read the null control printed on the same scale"
    ),
    "behaviour": (
        f"{POLICY}. This mode judges neither requirement directly. It checks invariants a "
        "windowed mount breaks first -- scroll extent, what the clipboard carries, whether the "
        "thread survives being reopened -- so a pass here is evidence about BEHAVIOUR and says "
        "nothing about appearance. It cannot grant the performance or off-screen exemptions, and "
        "it is the only mode that speaks to the select-all one, on the half of that exemption "
        "that is REQUIRED: whether the copy stays complete. It measures that BY LENGTH, scoring "
        "each arm's clipboard against the thread's own visible text and requiring coverage in "
        "{min}-{max}. It does not compare the copied characters and cannot: the base arm's "
        "clipboard is the DOM's rendered text and a store-based copy is markdown source, so a "
        "character comparison of two correct serialisations fails. Without a readable "
        "select_all_copy it records the exemption rather than granting it"
    ),
}


def behaviour_policy(min_coverage: float, max_coverage: float) -> str:
    """Interpolates the enforced coverage band, so the printed policy cannot drift from the gate."""
    return POLICY_BY_MODE["behaviour"].format(min = min_coverage, max = max_coverage)


def compare_visible(base: Optional[dict], treat: Optional[dict]) -> dict:
    """Scores the visible region only, naming moved ordinals by thread position, not mounted index."""
    for label, side in (("base", base), ("treatment", treat)):
        if not isinstance(side, dict) or not side.get("visible_attempted"):
            why = (side or {}).get("reason") or "no visible-region capture"
            return {
                "verdict": NOT_COMPARABLE,
                "reason": f"the {label} arm produced no visible-region capture: {why}",
                "moved": [],
                "claim": CLAIM_VISIBLE,
            }
    assert base is not None and treat is not None

    # Positive control: an empty visibility scan would otherwise return MATCH.
    bn, tn = len(base.get("ever_visible") or []), len(treat.get("ever_visible") or [])
    if bn == 0 or tn == 0:
        return {
            "verdict": NOT_COMPARABLE,
            "reason": (
                f"the visibility scan matched no messages (base={bn}, treatment={tn}). Nothing "
                "was observed on at least one arm, so there is nothing to agree about: this is a "
                "probe that needs fixing, not two arms that match"
            ),
            "moved": [],
            "claim": CLAIM_VISIBLE,
        }

    # Before comparing: rows sharing an `aria-posinset` collapse into one digest.
    collisions = {
        label: int(side.get("ordinal_collisions") or 0)
        for label, side in (("base", base), ("treatment", treat))
    }
    # One arm ending empty is not suppressible by a collision, which can merge but never empty a map.
    lost_the_thread = (len(base.get("messages") or {}) == 0) != (
        len(treat.get("messages") or {}) == 0
    )
    if any(collisions.values()) and not lost_the_thread:
        which = ", ".join(
            f"{label} {n} at ordinal(s) "
            f"{sorted((base if label == 'base' else treat).get('collided_ordinals') or [])[:8]}"
            for label, n in collisions.items()
            if n
        )
        return {
            "verdict": NOT_COMPARABLE,
            "reason": (
                "two or more mounted rows published the SAME thread position during this action, "
                f"so one of them is missing from everything compared below ({which}). The digest "
                "map is keyed by that position and the visible set is a set of them, so neither "
                "carries the extra row"
            ),
            "moved": [],
            "claim": CLAIM_VISIBLE,
            "not_digested": [],
        }

    bev, tev = set(base.get("ever_visible") or []), set(treat.get("ever_visible") or [])
    if bev != tev:
        only_b, only_t = sorted(bev - tev), sorted(tev - bev)
        return {
            "verdict": DIFFER,
            "reason": (
                "the two arms put DIFFERENT MESSAGES on screen during this action, which is a "
                "visible difference and not an off-screen one "
                f"(only base: {only_b[:8]}, only treatment: {only_t[:8]})"
            ),
            "moved": [f"ordinal {o}" for o in (only_b + only_t)[:8]],
            "claim": CLAIM_VISIBLE,
        }

    bmsg, tmsg = base.get("messages") or {}, treat.get("messages") or {}
    # Unmounted ordinals are reported, never counted as agreement.
    uncomparable = sorted(int(o) for o in map(str, sorted(bev)) if o not in bmsg or o not in tmsg)
    moved = []
    # Rows that cannot be a point in a stream survive the blind-probe refusal.
    settled: list[str] = []
    for ordinal in sorted(bev):
        key = str(ordinal)
        b, t = bmsg.get(key), tmsg.get(key)
        if b is None or t is None:
            continue
        # A role change is reported even in flight.
        if b.get("role") != t.get("role"):
            claim = f"ordinal {ordinal}:role {b.get('role')}->{t.get('role')}"
            moved.append(claim)
            settled.append(claim)
            continue
        if b.get("digest") != t.get("digest"):
            if b.get("in_flight") or t.get("in_flight"):
                # Still streaming on one arm: residue that refuses the verdict, as in `compare`.
                uncomparable.append(ordinal)
                continue
            claim = f"ordinal {ordinal}({b.get('role')}):{b.get('chars')}->{t.get('chars')}c"
            moved.append(claim)
            # A user row is never the reply being written.
            if b.get("role") == "user":
                settled.append(claim)
    uncomparable = sorted(set(uncomparable))
    # One viewport ending empty is a real difference (seen on `model_change` at 100K).
    b_left, t_left = len(bmsg), len(tmsg)
    if (b_left == 0) != (t_left == 0):
        empty, full = ("treatment", "base") if t_left == 0 else ("base", "treatment")
        return {
            "verdict": DIFFER,
            "reason": (
                f"the {empty} arm's viewport ended this action EMPTY while the {full} arm still "
                f"showed {max(b_left, t_left)} of the {len(bev)} message(s) that had been visible "
                "during it. Both arms displayed the same messages earlier in the action, so this "
                "is not a windowing difference: one arm lost the thread"
            ),
            "moved": [f"ordinal {o}" for o in sorted(bev)[:8]],
            "claim": CLAIM_VISIBLE,
            # Not suppressible by the noise floor: losing the whole thread is a different statement.
            "severe": True,
        }
    # A build that renames the `data-status` / `aria-busy` hooks reports every row settled.
    blind = streaming_probe(base, treat)
    if blind is not None:
        # Same rule as `compare`: settled rows survive the refusal.
        if settled:
            return {
                "verdict": DIFFER,
                "reason": (
                    f"{len(settled)} visible message(s) rendered differently on rows that cannot "
                    "be the reply being written (a user row, or a row whose role changed), so "
                    f"they are reported even though the stream could not be placed. {blind}"
                ),
                "moved": settled,
                "claim": CLAIM_VISIBLE,
                "not_digested": uncomparable,
            }
        return {
            "verdict": NOT_COMPARABLE,
            "reason": blind,
            "moved": [],
            "claim": CLAIM_VISIBLE,
            "not_digested": uncomparable,
        }
    if moved:
        return {
            "verdict": DIFFER,
            "reason": f"{len(moved)} visible message(s) rendered differently",
            "moved": moved,
            "claim": CLAIM_VISIBLE,
            "not_digested": uncomparable,
        }
    if uncomparable:
        # Any residue refuses the verdict: the claim quantifies over every message the viewport showed.
        digested = len(bev) - len(uncomparable)
        return {
            "verdict": NOT_COMPARABLE,
            "reason": (
                f"{len(uncomparable)} of the {len(bev)} message(s) that were visible during this "
                "action had been unmounted again by the time the capture ran, so they could not be "
                f"digested (ordinals {uncomparable[:8]}). The {digested} that could be digested "
                "agreed, which is not the claim this mode makes"
            ),
            "moved": [],
            "claim": CLAIM_VISIBLE,
            "not_digested": uncomparable,
        }
    return {
        "verdict": MATCH,
        "reason": "",
        "moved": [],
        "claim": CLAIM_VISIBLE,
        "not_digested": [],
    }