Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studiobench UI parity: report a UI change the PR declares instead of failing (#13583)

Daniel Han481c979eb09c
diff --git a/.github/workflows/studiobench-ui-parity.yml b/.github/workflows/studiobench-ui-parity.yml
index 50cf0ed1c8b..18e76800461 100644
--- a/.github/workflows/studiobench-ui-parity.yml
+++ b/.github/workflows/studiobench-ui-parity.yml
@@ -623,6 +623,9 @@ jobs:
name: Parity verdict, scored against the null
needs: arms
runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ pull-requests: read # the PR's labels as they are now; see "Read the PR's labels"
timeout-minutes: 10
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -781,7 +784,32 @@ jobs:
# The cost is stated in `corroborated()`: a genuine change that can only
# render in one repetition is demoted to UNCORROBORATED and printed rather
# than counted.
+ #
+ # `affects UI#` declares a UI change: rendering differences are reported, behaviour differences still fail.
+ # Live labels, not `github.event.pull_request.labels` (stale on a re-run); a failed read is undeclared.
+ - name: Read the PR's labels
+ id: labels
+ env:
+ GH_TOKEN: ${{ github.token }}
+ PR: ${{ github.event.pull_request.number }}
+ REPO: ${{ github.repository }}
+ run: |
+ set -uo pipefail
+ declared=false
+ if names="$(gh api "repos/$REPO/pulls/$PR" --jq '.labels[].name')"; then
+ if printf '%s\n' "$names" | grep -qxF 'affects UI#'; then
+ declared=true
+ fi
+ else
+ echo "::warning::could not read this PR's labels; scoring it as an undeclared UI change."
+ fi
+ echo "declared UI change: $declared"
+ echo "declared=$declared" >> "$GITHUB_OUTPUT"
+
- name: The verdict
+ id: verdict
+ env:
+ UI_CHANGE_DECLARED: ${{ steps.labels.outputs.declared }}
run: |
# `set +e` is not redundant. GitHub runs every `run:` block under
# `bash -e {0}`, so a non-zero exit aborts the step before the line
@@ -791,7 +819,12 @@ jobs:
# measurement was fine.
set +e
set -uo pipefail
+ declare_flag=()
+ if [ "$UI_CHANGE_DECLARED" = true ]; then
+ declare_flag=(--declared-ui-change)
+ fi
python -m tests.studio.studiobench.sweep.ui_parity \
+ "${declare_flag[@]}" \
--min-reps 2 \
--min-compared 16 \
--null outputs/parity-null-control \
@@ -805,6 +838,11 @@ jobs:
grep -E 'action pairs|matched:|stable actions differing|unstable actions|NOT COMPARABLE|NOT EXERCISED|No stable action' verdict.txt || true
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
+ if [ "$rc" = 0 ] && grep -q 'DECLARED UI CHANGE' verdict.txt; then
+ echo "::notice::The UI changed, as this PR's 'affects UI#' label declares. The before/after pairs are in the studiobench-parity-evidence artifact."
+ echo "UI change declared by the 'affects UI#' label: reported, not failed." >> "$GITHUB_STEP_SUMMARY"
+ echo "declared_diff=true" >> "$GITHUB_OUTPUT"
+ fi
exit "$rc"
# ONLY ON FAILURE, which is what keeps a green run cheap: a passing run publishes no
@@ -819,7 +857,9 @@ jobs:
# the more comfortable design and a dishonest one: stream progress differs run to run, so
# the second run would very often show two identical pages for a difference that was real.
- name: Draw the before/after pairs for what turned it red
- if: failure()
+ if: failure() || steps.verdict.outputs.declared_diff == 'true'
+ env:
+ DECLARED_DIFF: ${{ steps.verdict.outputs.declared_diff }}
run: |
set -uo pipefail
pip install --quiet 'pillow<12'
@@ -837,9 +877,14 @@ jobs:
echo "Composites are BEFORE (merge base) | AFTER (head), padded rather than scaled,"
echo "each half labelled with its arm, action, cell and scrollTop. A pair whose two"
echo "scroll offsets disagree is marked as not a like-for-like comparison."
+ # A declared change is green only because these pictures stand in for the failure.
+ if [ "$DECLARED_DIFF" = true ] && ! find outputs/parity-evidence -name '*__composite.png' -print -quit | grep -q .; then
+ echo "::error::the declared UI change produced no before/after composite to review."
+ exit 1
+ fi
- name: Upload the before/after evidence
- if: failure()
+ if: failure() || steps.verdict.outputs.declared_diff == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: studiobench-parity-evidence
diff --git a/tests/studio/studiobench/sweep/parity_shots.py b/tests/studio/studiobench/sweep/parity_shots.py
index d082f7b741f..5cc753a0b59 100644
--- a/tests/studio/studiobench/sweep/parity_shots.py
+++ b/tests/studio/studiobench/sweep/parity_shots.py
@@ -44,13 +44,18 @@ from tests.studio.studiobench.analysis import parity as P # noqa: E402
from tests.studio.studiobench.scoring.from_payload import latest_attempt_rows # noqa: E402
from tests.studio.studiobench.sweep.ui_parity import ( # noqa: E402
compare_all,
+ WINDOWED,
+ compare_all_with,
confine_to_runner,
corroborated,
+ decide_modes,
in_arm_repeatability,
is_unstable,
+ rung_of_cell,
shards_of,
swapped_between_arms,
unstable_set,
+ visible_unstable_set,
)
BANNER = 46 # px of label strip above each half
@@ -175,6 +180,20 @@ def differing_actions(
firm_expect, _weak_expect = corroborated(expect_bad, min_reps)
for action, shard, cell, moved, *_dir in firm + firm_one_sided + firm_expect:
out.append({"action": action, "shard": shard, "cell": cell, "moved": moved})
+ # The verdict's own visible-mode pairs and floor, so a declared change found only there gets a picture.
+ windowed = {key for key, (mode, _why) in decide_modes(result_paths).items() if mode == WINDOWED}
+ if windowed:
+ floor = visible_unstable_set(null_paths or None)
+ visible, _tally = compare_all_with(result_paths, P.compare_visible, "visible", windowed)
+ for action, shard, cell, r in visible:
+ if (
+ r.get("_ran")
+ and r["verdict"] == P.DIFFER
+ and (r.get("severe") or (rung_of_cell(cell), action) not in floor)
+ ):
+ out.append(
+ {"action": action, "shard": shard, "cell": cell, "moved": r.get("moved", [])}
+ )
return out
diff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_declared_ui_change.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_declared_ui_change.py
new file mode 100644
index 00000000000..935b42a9bc5
--- /dev/null
+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_declared_ui_change.py
@@ -0,0 +1,198 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""`--declared-ui-change` excuses a rendering difference and nothing else.
+
+A PR that says it changes the screen (#13402 renamed the composer's Code toggle to Tools) should not
+fail the parity gate for doing so, but the same exit code also carries "a control could not be
+operated on one build" and "an action's own assertion failed on one build". Those are the head build
+behaving differently, which a label about the screen does not excuse.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[4]))
+
+from tests.studio.studiobench.sweep import ui_parity as U # noqa: E402
+from tests.studio.studiobench.sweep.selftest import test_studiobench_swapped_parity as SW # noqa: E402
+from tests.studio.studiobench.sweep.selftest.test_studiobench_null_audit import ( # noqa: E402
+ action_row,
+ one_sided_payload,
+)
+
+
+def _main(tmp_path, capsys, result, null, declared) -> tuple[int, str]:
+ rdir = SW._write(result, tmp_path / "parity-result")
+ ndir = SW._write(null, tmp_path / "parity-null-control")
+ argv = ["--min-reps", "2", "--min-compared", "16", "--null", str(ndir), str(rdir)]
+ rc = U.main((["--declared-ui-change"] if declared else []) + argv)
+ return rc, capsys.readouterr().out
+
+
+def test_a_declared_change_reports_a_stable_rendering_difference(tmp_path, capsys):
+ result, null = SW._recorded()
+ SW._make_consistent(result, "thread_reopen")
+ rc, out = _main(tmp_path, capsys, result, null, declared = False)
+ assert rc == 1, out
+ rc, out = _main(tmp_path / "declared", capsys, result, null, declared = True)
+ assert rc == 0, out
+ assert "DECLARED UI CHANGE" in out
+ assert SW._stable(out).count("thread_reopen") == 2, SW._stable(out)
+
+
+def test_no_difference_prints_no_declaration(tmp_path, capsys):
+ result, null = SW._recorded()
+ rc, out = _main(tmp_path, capsys, result, null, declared = True)
+ assert rc == 0, out
+ assert "DECLARED UI CHANGE" not in out
+
+
+def test_a_declared_change_still_fails_a_control_that_runs_on_one_arm_only(tmp_path):
+ path = one_sided_payload(tmp_path, "scroll", "scroll_after", ("rep0", "rep1"))
+ args = ([path], "t", U.UNSTABLE_ACTIONS)
+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 1
+
+
+def _assertion_failed_on_treatment(tmp_path: Path, treatment_ok: bool = False) -> Path:
+ rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
+ for rep in ("rep0", "rep1"):
+ for arm in ("base", "treatment"):
+ cid = f"r100K.{arm}.{rep}"
+ for i in range(16):
+ rows.append(action_row(cid, f"filler{i}", "SAME"))
+ row = action_row(cid, "scroll_after", "SAME")
+ row["expect_ok"] = arm == "base" or treatment_ok
+ rows.append(row)
+ rows.append({"row_type": "cell", "cell_id": cid, "completed": True})
+ out = tmp_path / f"expect_{treatment_ok}"
+ out.mkdir()
+ path = out / "payload.jsonl"
+ path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding = "utf-8")
+ return path
+
+
+def test_a_declared_change_still_fails_an_assertion_that_failed_on_one_arm(tmp_path):
+ path = _assertion_failed_on_treatment(tmp_path)
+ args = ([path], "t", U.UNSTABLE_ACTIONS)
+ assert U.report(*args, min_reps = 2, min_compared = 16) == 1
+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 1
+
+
+def test_the_same_film_with_both_assertions_passing_is_clean(tmp_path):
+ path = _assertion_failed_on_treatment(tmp_path, treatment_ok = True)
+ args = ([path], "t", U.UNSTABLE_ACTIONS)
+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 0
+
+
+def _every_action_differs(tmp_path: Path) -> Path:
+ """A wrapper change: every action renders differently on head, in both repetitions."""
+ rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
+ for rep in ("rep0", "rep1"):
+ for arm in ("base", "treatment"):
+ cid = f"r100K.{arm}.{rep}"
+ for i in range(17):
+ rows.append(action_row(cid, f"filler{i}", "BASE" if arm == "base" else "HEAD"))
+ rows.append({"row_type": "cell", "cell_id": cid, "completed": True})
+ out = tmp_path / "wrapper"
+ out.mkdir()
+ path = out / "payload.jsonl"
+ path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding = "utf-8")
+ return path
+
+
+def test_a_declared_change_on_every_action_is_still_a_decided_run(tmp_path):
+ path = _every_action_differs(tmp_path)
+ args = ([path], "t", frozenset())
+ assert U.report(*args, min_reps = 2, min_compared = 16) == 1
+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 0
+
+
+def _windowed_film(
+ tmp_path: Path,
+ name: str,
+ head_visible: str,
+ head_root: str = "thread",
+) -> Path:
+ """`settings` on a windowed treatment arm (no structural verdict), its visible region as given."""
+ from tests.studio.studiobench.sweep.selftest.test_studiobench_windowed_parity import _capture
+
+ rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]
+ for rep in ("rep0", "rep1"):
+ for arm in ("base", "treatment"):
+ cid = f"r100K.{arm}.{rep}"
+ treat = arm == "treatment"
+ rows.append(
+ {
+ "row_type": "action",
+ "cell_id": cid,
+ "action": "settings",
+ "ran": True,
+ "parity": {
+ **_capture(6 if treat else 18, 18, "d"),
+ "root_kind": head_root if treat else "thread",
+ },
+ "visible": {
+ "visible_attempted": True,
+ "ever_visible": [1],
+ "ever_visible_count": 1,
+ "mounted_ever_visible": 1,
+ "unmounted_at_capture": 0,
+ "messages": {
+ "1": {
+ "role": "assistant",
+ "digest": head_visible if treat else "same",
+ "chars": 10,
+ }
+ },
+ },
+ }
+ )
+ rows.append({"row_type": "cell", "cell_id": cid, "completed": True})
+ out = tmp_path / name
+ out.mkdir()
+ (out / "payload.jsonl").write_text(
+ "".join(json.dumps(r) + "\n" for r in rows), encoding = "utf-8"
+ )
+ return out
+
+
+def test_a_windowed_visible_difference_gets_its_evidence(tmp_path):
+ from tests.studio.studiobench.sweep import parity_shots as S
+
+ result = U.shards_of(str(_windowed_film(tmp_path, "result", "HEAD")))
+ null = U.shards_of(str(_windowed_film(tmp_path, "null", "same")))
+ picked = S.differing_actions(result, null, min_reps = 2)
+ assert {(d["action"], d["cell"]) for d in picked} == {
+ ("settings", "r100K rep0"),
+ ("settings", "r100K rep1"),
+ }, picked
+
+
+def test_a_windowed_pair_that_matches_gets_no_picture(tmp_path):
+ from tests.studio.studiobench.sweep import parity_shots as S
+
+ result = U.shards_of(str(_windowed_film(tmp_path, "result", "same")))
+ null = U.shards_of(str(_windowed_film(tmp_path, "null", "same")))
+ assert S.differing_actions(result, null, min_reps = 2) == []
+
+
+def test_a_declared_visible_change_on_every_pair_is_still_a_decided_run(tmp_path):
+ paths = U.shards_of(str(_windowed_film(tmp_path, "result", "HEAD")))
+ assert U.visible_report(paths, "t", min_reps = 2) == 1
+ assert U.visible_report(paths, "t", min_reps = 2, declared_ui_change = True) == 0
+
+
+def test_a_visible_pair_the_digest_calls_not_comparable_still_gets_its_evidence(tmp_path):
+ """The verdict's own plan picks the pair, not the structural verdict (different root_kind here)."""
+ from tests.studio.studiobench.sweep import parity_shots as S
+
+ result = U.shards_of(str(_windowed_film(tmp_path, "result", "HEAD", head_root = "body")))
+ null = U.shards_of(str(_windowed_film(tmp_path, "null", "same")))
+ structural, _ = U.compare_all(result)
+ assert {r["verdict"] for *_k, r in structural} != {"not_applicable"}, structural
+ picked = S.differing_actions(result, null, min_reps = 2)
+ assert {d["action"] for d in picked} == {"settings"}, picked
diff --git a/tests/studio/studiobench/sweep/ui_parity.py b/tests/studio/studiobench/sweep/ui_parity.py
index 574571c39fd..cef259bb22c 100644
--- a/tests/studio/studiobench/sweep/ui_parity.py
+++ b/tests/studio/studiobench/sweep/ui_parity.py
@@ -880,6 +880,7 @@ def visible_report(
unstable: frozenset[tuple[str, str]] = frozenset(),
select: Optional[set] = None,
min_reps: int = 1,
+ declared_ui_change: bool = False,
) -> Outcome:
"""VISIBLE-REGION PARITY. The verdict the off-screen exemption asks for.
@@ -1008,9 +1009,14 @@ def visible_report(
)
# See the same three lines in `behaviour_report` and `report`: the diagnosis prints, then a
# failure outranks a refusal.
- if differing or build_failed:
+ if build_failed or (differing and not declared_ui_change):
return Outcome(1, frozenset(compared), _keys(results))
- if matched == 0:
+ if differing:
+ print(
+ "\n DECLARED UI CHANGE: the PR says it changes the screen (--declared-ui-change), so the "
+ "rendering differences above are reported, not failed. Behaviour differences still fail."
+ )
+ if matched == 0 and not differing:
return Outcome(2, frozenset(compared), _keys(results))
return Outcome(0, frozenset(compared), _keys(results))
@@ -1716,6 +1722,7 @@ def report(
select: Optional[set] = None,
min_reps: int = 1,
min_compared: int = 0,
+ declared_ui_change: bool = False,
) -> Outcome:
"""THREAD-STRUCTURE PARITY. `select` scores only those pair keys; see `decide_modes`.
@@ -2004,12 +2011,19 @@ def report(
# A one-sided action is a failure on the same footing as a differing digest: the two builds
# behaved differently and the difference leaves no digest to compare.
# `expect_bad` is collected ahead of the verdict branches.
- if stable_bad or one_sided or expect_bad:
+ if one_sided or expect_bad or (stable_bad and not declared_ui_change):
return Outcome(1, frozenset(compared), _keys(results))
+ if stable_bad:
+ print(
+ "\n DECLARED UI CHANGE: the PR says it changes the screen (--declared-ui-change), so the "
+ "rendering differences above are reported, not failed. Behaviour differences still fail."
+ )
+
# 2, the same code the empty-payload path uses: the tool was asked a question it could not
# answer, and exiting 0 would let CI go green on a run structurally incapable of firing.
# `decided`, not `matched`. AFTER the failure return, since "it failed" is more specific.
- if decided == 0:
+ # A declared difference was decided too: a wrapper change can differ on every pair.
+ if decided == 0 and not stable_bad:
return Outcome(2, frozenset(compared), _keys(results))
return Outcome(0, frozenset(compared), _keys(results))
@@ -2214,6 +2228,12 @@ def main(argv: list[str] | None = None) -> int:
"differences and the corroborated one-arm-only actions. An action nothing was going to "
"excuse needs no excuse. Uses --min-reps, so pass the verdict's own value",
)
+ ap.add_argument(
+ "--declared-ui-change",
+ action = "store_true",
+ help = "the PR declares that it changes the screen: rendering differences are reported, not "
+ "failed. One-arm-only actions, failed action assertions and behaviour differences still fail",
+ )
ap.add_argument(
"--allow-undecided",
metavar = "ACTIONS",
@@ -2343,7 +2363,12 @@ def main(argv: list[str] | None = None) -> int:
)
floor = frozenset()
vis = visible_report(
- paths, f"UI PARITY: {pattern}", floor, select = win, min_reps = args.min_reps
+ paths,
+ f"UI PARITY: {pattern}",
+ floor,
+ select = win,
+ min_reps = args.min_reps,
+ declared_ui_change = args.declared_ui_change,
)
worst = max(worst, int(vis))
windowed_compared = frozenset(vis.compared)
@@ -2495,7 +2520,14 @@ def main(argv: list[str] | None = None) -> int:
# comparing nine windowed and nine structural pairs would fail a floor of sixteen twice over.
# `args.min_compared` was the argument that went missing.
# `report` (`elif decided == 0`) and `visible_report` (`and not not_run`) each check their own.
- struct = report(paths, label, effective, select, args.min_reps)
+ struct = report(
+ paths,
+ label,
+ effective,
+ select,
+ args.min_reps,
+ declared_ui_change = args.declared_ui_change,
+ )
worst = max(worst, int(struct))
note(pattern, struct)
# COMBINED, and the worst outcome wins: a behavioural failure on one payload is not cancelled by