Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studiobench UI parity: report a UI change the PR declares instead of failing (#13583)
Daniel Han481c979eb09c
diff --git a/.github/workflows/studiobench-ui-parity.yml b/.github/workflows/studiobench-ui-parity.ymlindex 50cf0ed1c8b..18e76800461 100644--- a/.github/workflows/studiobench-ui-parity.yml+++ b/.github/workflows/studiobench-ui-parity.yml@@ -623,6 +623,9 @@ jobs:name: Parity verdict, scored against the nullneeds: armsruns-on: ubuntu-latest+ permissions:+ contents: read+ pull-requests: read # the PR's labels as they are now; see "Read the PR's labels"timeout-minutes: 10steps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -781,7 +784,32 @@ jobs:# The cost is stated in `corroborated()`: a genuine change that can only# render in one repetition is demoted to UNCORROBORATED and printed rather# than counted.+ #+ # `affects UI#` declares a UI change: rendering differences are reported, behaviour differences still fail.+ # Live labels, not `github.event.pull_request.labels` (stale on a re-run); a failed read is undeclared.+ - name: Read the PR's labels+ id: labels+ env:+ GH_TOKEN: ${{ github.token }}+ PR: ${{ github.event.pull_request.number }}+ REPO: ${{ github.repository }}+ run: |+ set -uo pipefail+ declared=false+ if names="$(gh api "repos/$REPO/pulls/$PR" --jq '.labels[].name')"; then+ if printf '%s\n' "$names" | grep -qxF 'affects UI#'; then+ declared=true+ fi+ else+ echo "::warning::could not read this PR's labels; scoring it as an undeclared UI change."+ fi+ echo "declared UI change: $declared"+ echo "declared=$declared" >> "$GITHUB_OUTPUT"+- name: The verdict+ id: verdict+ env:+ UI_CHANGE_DECLARED: ${{ steps.labels.outputs.declared }}run: |# `set +e` is not redundant. GitHub runs every `run:` block under# `bash -e {0}`, so a non-zero exit aborts the step before the line@@ -791,7 +819,12 @@ jobs:# measurement was fine.set +eset -uo pipefail+ declare_flag=()+ if [ "$UI_CHANGE_DECLARED" = true ]; then+ declare_flag=(--declared-ui-change)+ fipython -m tests.studio.studiobench.sweep.ui_parity \+ "${declare_flag[@]}" \--min-reps 2 \--min-compared 16 \--null outputs/parity-null-control \@@ -805,6 +838,11 @@ jobs:grep -E 'action pairs|matched:|stable actions differing|unstable actions|NOT COMPARABLE|NOT EXERCISED|No stable action' verdict.txt || trueecho '```'} >> "$GITHUB_STEP_SUMMARY"+ if [ "$rc" = 0 ] && grep -q 'DECLARED UI CHANGE' verdict.txt; then+ echo "::notice::The UI changed, as this PR's 'affects UI#' label declares. The before/after pairs are in the studiobench-parity-evidence artifact."+ echo "UI change declared by the 'affects UI#' label: reported, not failed." >> "$GITHUB_STEP_SUMMARY"+ echo "declared_diff=true" >> "$GITHUB_OUTPUT"+ fiexit "$rc"# ONLY ON FAILURE, which is what keeps a green run cheap: a passing run publishes no@@ -819,7 +857,9 @@ jobs:# the more comfortable design and a dishonest one: stream progress differs run to run, so# the second run would very often show two identical pages for a difference that was real.- name: Draw the before/after pairs for what turned it red- if: failure()+ if: failure() || steps.verdict.outputs.declared_diff == 'true'+ env:+ DECLARED_DIFF: ${{ steps.verdict.outputs.declared_diff }}run: |set -uo pipefailpip install --quiet 'pillow<12'@@ -837,9 +877,14 @@ jobs:echo "Composites are BEFORE (merge base) | AFTER (head), padded rather than scaled,"echo "each half labelled with its arm, action, cell and scrollTop. A pair whose two"echo "scroll offsets disagree is marked as not a like-for-like comparison."+ # A declared change is green only because these pictures stand in for the failure.+ if [ "$DECLARED_DIFF" = true ] && ! find outputs/parity-evidence -name '*__composite.png' -print -quit | grep -q .; then+ echo "::error::the declared UI change produced no before/after composite to review."+ exit 1+ fi- name: Upload the before/after evidence- if: failure()+ if: failure() || steps.verdict.outputs.declared_diff == 'true'uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1with:name: studiobench-parity-evidencediff --git a/tests/studio/studiobench/sweep/parity_shots.py b/tests/studio/studiobench/sweep/parity_shots.pyindex d082f7b741f..5cc753a0b59 100644--- a/tests/studio/studiobench/sweep/parity_shots.py+++ b/tests/studio/studiobench/sweep/parity_shots.py@@ -44,13 +44,18 @@ from tests.studio.studiobench.analysis import parity as P # noqa: E402from tests.studio.studiobench.scoring.from_payload import latest_attempt_rows # noqa: E402from tests.studio.studiobench.sweep.ui_parity import ( # noqa: E402compare_all,+ WINDOWED,+ compare_all_with,confine_to_runner,corroborated,+ decide_modes,in_arm_repeatability,is_unstable,+ rung_of_cell,shards_of,swapped_between_arms,unstable_set,+ visible_unstable_set,)BANNER = 46 # px of label strip above each half@@ -175,6 +180,20 @@ def differing_actions(firm_expect, _weak_expect = corroborated(expect_bad, min_reps)for action, shard, cell, moved, *_dir in firm + firm_one_sided + firm_expect:out.append({"action": action, "shard": shard, "cell": cell, "moved": moved})+ # The verdict's own visible-mode pairs and floor, so a declared change found only there gets a picture.+ windowed = {key for key, (mode, _why) in decide_modes(result_paths).items() if mode == WINDOWED}+ if windowed:+ floor = visible_unstable_set(null_paths or None)+ visible, _tally = compare_all_with(result_paths, P.compare_visible, "visible", windowed)+ for action, shard, cell, r in visible:+ if (+ r.get("_ran")+ and r["verdict"] == P.DIFFER+ and (r.get("severe") or (rung_of_cell(cell), action) not in floor)+ ):+ out.append(+ {"action": action, "shard": shard, "cell": cell, "moved": r.get("moved", [])}+ )return outdiff --git a/tests/studio/studiobench/sweep/selftest/test_studiobench_declared_ui_change.py b/tests/studio/studiobench/sweep/selftest/test_studiobench_declared_ui_change.pynew file mode 100644index 00000000000..935b42a9bc5--- /dev/null+++ b/tests/studio/studiobench/sweep/selftest/test_studiobench_declared_ui_change.py@@ -0,0 +1,198 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""`--declared-ui-change` excuses a rendering difference and nothing else.++A PR that says it changes the screen (#13402 renamed the composer's Code toggle to Tools) should not+fail the parity gate for doing so, but the same exit code also carries "a control could not be+operated on one build" and "an action's own assertion failed on one build". Those are the head build+behaving differently, which a label about the screen does not excuse.+"""++from __future__ import annotations++import json+import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parents[4]))++from tests.studio.studiobench.sweep import ui_parity as U # noqa: E402+from tests.studio.studiobench.sweep.selftest import test_studiobench_swapped_parity as SW # noqa: E402+from tests.studio.studiobench.sweep.selftest.test_studiobench_null_audit import ( # noqa: E402+ action_row,+ one_sided_payload,+)+++def _main(tmp_path, capsys, result, null, declared) -> tuple[int, str]:+ rdir = SW._write(result, tmp_path / "parity-result")+ ndir = SW._write(null, tmp_path / "parity-null-control")+ argv = ["--min-reps", "2", "--min-compared", "16", "--null", str(ndir), str(rdir)]+ rc = U.main((["--declared-ui-change"] if declared else []) + argv)+ return rc, capsys.readouterr().out+++def test_a_declared_change_reports_a_stable_rendering_difference(tmp_path, capsys):+ result, null = SW._recorded()+ SW._make_consistent(result, "thread_reopen")+ rc, out = _main(tmp_path, capsys, result, null, declared = False)+ assert rc == 1, out+ rc, out = _main(tmp_path / "declared", capsys, result, null, declared = True)+ assert rc == 0, out+ assert "DECLARED UI CHANGE" in out+ assert SW._stable(out).count("thread_reopen") == 2, SW._stable(out)+++def test_no_difference_prints_no_declaration(tmp_path, capsys):+ result, null = SW._recorded()+ rc, out = _main(tmp_path, capsys, result, null, declared = True)+ assert rc == 0, out+ assert "DECLARED UI CHANGE" not in out+++def test_a_declared_change_still_fails_a_control_that_runs_on_one_arm_only(tmp_path):+ path = one_sided_payload(tmp_path, "scroll", "scroll_after", ("rep0", "rep1"))+ args = ([path], "t", U.UNSTABLE_ACTIONS)+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 1+++def _assertion_failed_on_treatment(tmp_path: Path, treatment_ok: bool = False) -> Path:+ rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]+ for rep in ("rep0", "rep1"):+ for arm in ("base", "treatment"):+ cid = f"r100K.{arm}.{rep}"+ for i in range(16):+ rows.append(action_row(cid, f"filler{i}", "SAME"))+ row = action_row(cid, "scroll_after", "SAME")+ row["expect_ok"] = arm == "base" or treatment_ok+ rows.append(row)+ rows.append({"row_type": "cell", "cell_id": cid, "completed": True})+ out = tmp_path / f"expect_{treatment_ok}"+ out.mkdir()+ path = out / "payload.jsonl"+ path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding = "utf-8")+ return path+++def test_a_declared_change_still_fails_an_assertion_that_failed_on_one_arm(tmp_path):+ path = _assertion_failed_on_treatment(tmp_path)+ args = ([path], "t", U.UNSTABLE_ACTIONS)+ assert U.report(*args, min_reps = 2, min_compared = 16) == 1+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 1+++def test_the_same_film_with_both_assertions_passing_is_clean(tmp_path):+ path = _assertion_failed_on_treatment(tmp_path, treatment_ok = True)+ args = ([path], "t", U.UNSTABLE_ACTIONS)+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 0+++def _every_action_differs(tmp_path: Path) -> Path:+ """A wrapper change: every action renders differently on head, in both repetitions."""+ rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]+ for rep in ("rep0", "rep1"):+ for arm in ("base", "treatment"):+ cid = f"r100K.{arm}.{rep}"+ for i in range(17):+ rows.append(action_row(cid, f"filler{i}", "BASE" if arm == "base" else "HEAD"))+ rows.append({"row_type": "cell", "cell_id": cid, "completed": True})+ out = tmp_path / "wrapper"+ out.mkdir()+ path = out / "payload.jsonl"+ path.write_text("".join(json.dumps(r) + "\n" for r in rows), encoding = "utf-8")+ return path+++def test_a_declared_change_on_every_action_is_still_a_decided_run(tmp_path):+ path = _every_action_differs(tmp_path)+ args = ([path], "t", frozenset())+ assert U.report(*args, min_reps = 2, min_compared = 16) == 1+ assert U.report(*args, min_reps = 2, min_compared = 16, declared_ui_change = True) == 0+++def _windowed_film(+ tmp_path: Path,+ name: str,+ head_visible: str,+ head_root: str = "thread",+) -> Path:+ """`settings` on a windowed treatment arm (no structural verdict), its visible region as given."""+ from tests.studio.studiobench.sweep.selftest.test_studiobench_windowed_parity import _capture++ rows: list[dict] = [{"row_type": "run_meta", "tier": "fast"}]+ for rep in ("rep0", "rep1"):+ for arm in ("base", "treatment"):+ cid = f"r100K.{arm}.{rep}"+ treat = arm == "treatment"+ rows.append(+ {+ "row_type": "action",+ "cell_id": cid,+ "action": "settings",+ "ran": True,+ "parity": {+ **_capture(6 if treat else 18, 18, "d"),+ "root_kind": head_root if treat else "thread",+ },+ "visible": {+ "visible_attempted": True,+ "ever_visible": [1],+ "ever_visible_count": 1,+ "mounted_ever_visible": 1,+ "unmounted_at_capture": 0,+ "messages": {+ "1": {+ "role": "assistant",+ "digest": head_visible if treat else "same",+ "chars": 10,+ }+ },+ },+ }+ )+ rows.append({"row_type": "cell", "cell_id": cid, "completed": True})+ out = tmp_path / name+ out.mkdir()+ (out / "payload.jsonl").write_text(+ "".join(json.dumps(r) + "\n" for r in rows), encoding = "utf-8"+ )+ return out+++def test_a_windowed_visible_difference_gets_its_evidence(tmp_path):+ from tests.studio.studiobench.sweep import parity_shots as S++ result = U.shards_of(str(_windowed_film(tmp_path, "result", "HEAD")))+ null = U.shards_of(str(_windowed_film(tmp_path, "null", "same")))+ picked = S.differing_actions(result, null, min_reps = 2)+ assert {(d["action"], d["cell"]) for d in picked} == {+ ("settings", "r100K rep0"),+ ("settings", "r100K rep1"),+ }, picked+++def test_a_windowed_pair_that_matches_gets_no_picture(tmp_path):+ from tests.studio.studiobench.sweep import parity_shots as S++ result = U.shards_of(str(_windowed_film(tmp_path, "result", "same")))+ null = U.shards_of(str(_windowed_film(tmp_path, "null", "same")))+ assert S.differing_actions(result, null, min_reps = 2) == []+++def test_a_declared_visible_change_on_every_pair_is_still_a_decided_run(tmp_path):+ paths = U.shards_of(str(_windowed_film(tmp_path, "result", "HEAD")))+ assert U.visible_report(paths, "t", min_reps = 2) == 1+ assert U.visible_report(paths, "t", min_reps = 2, declared_ui_change = True) == 0+++def test_a_visible_pair_the_digest_calls_not_comparable_still_gets_its_evidence(tmp_path):+ """The verdict's own plan picks the pair, not the structural verdict (different root_kind here)."""+ from tests.studio.studiobench.sweep import parity_shots as S++ result = U.shards_of(str(_windowed_film(tmp_path, "result", "HEAD", head_root = "body")))+ null = U.shards_of(str(_windowed_film(tmp_path, "null", "same")))+ structural, _ = U.compare_all(result)+ assert {r["verdict"] for *_k, r in structural} != {"not_applicable"}, structural+ picked = S.differing_actions(result, null, min_reps = 2)+ assert {d["action"] for d in picked} == {"settings"}, pickeddiff --git a/tests/studio/studiobench/sweep/ui_parity.py b/tests/studio/studiobench/sweep/ui_parity.pyindex 574571c39fd..cef259bb22c 100644--- a/tests/studio/studiobench/sweep/ui_parity.py+++ b/tests/studio/studiobench/sweep/ui_parity.py@@ -880,6 +880,7 @@ def visible_report(unstable: frozenset[tuple[str, str]] = frozenset(),select: Optional[set] = None,min_reps: int = 1,+ declared_ui_change: bool = False,) -> Outcome:"""VISIBLE-REGION PARITY. The verdict the off-screen exemption asks for.@@ -1008,9 +1009,14 @@ def visible_report()# See the same three lines in `behaviour_report` and `report`: the diagnosis prints, then a# failure outranks a refusal.- if differing or build_failed:+ if build_failed or (differing and not declared_ui_change):return Outcome(1, frozenset(compared), _keys(results))- if matched == 0:+ if differing:+ print(+ "\n DECLARED UI CHANGE: the PR says it changes the screen (--declared-ui-change), so the "+ "rendering differences above are reported, not failed. Behaviour differences still fail."+ )+ if matched == 0 and not differing:return Outcome(2, frozenset(compared), _keys(results))return Outcome(0, frozenset(compared), _keys(results))@@ -1716,6 +1722,7 @@ def report(select: Optional[set] = None,min_reps: int = 1,min_compared: int = 0,+ declared_ui_change: bool = False,) -> Outcome:"""THREAD-STRUCTURE PARITY. `select` scores only those pair keys; see `decide_modes`.@@ -2004,12 +2011,19 @@ def report(# A one-sided action is a failure on the same footing as a differing digest: the two builds# behaved differently and the difference leaves no digest to compare.# `expect_bad` is collected ahead of the verdict branches.- if stable_bad or one_sided or expect_bad:+ if one_sided or expect_bad or (stable_bad and not declared_ui_change):return Outcome(1, frozenset(compared), _keys(results))+ if stable_bad:+ print(+ "\n DECLARED UI CHANGE: the PR says it changes the screen (--declared-ui-change), so the "+ "rendering differences above are reported, not failed. Behaviour differences still fail."+ )+# 2, the same code the empty-payload path uses: the tool was asked a question it could not# answer, and exiting 0 would let CI go green on a run structurally incapable of firing.# `decided`, not `matched`. AFTER the failure return, since "it failed" is more specific.- if decided == 0:+ # A declared difference was decided too: a wrapper change can differ on every pair.+ if decided == 0 and not stable_bad:return Outcome(2, frozenset(compared), _keys(results))return Outcome(0, frozenset(compared), _keys(results))@@ -2214,6 +2228,12 @@ def main(argv: list[str] | None = None) -> int:"differences and the corroborated one-arm-only actions. An action nothing was going to ""excuse needs no excuse. Uses --min-reps, so pass the verdict's own value",)+ ap.add_argument(+ "--declared-ui-change",+ action = "store_true",+ help = "the PR declares that it changes the screen: rendering differences are reported, not "+ "failed. One-arm-only actions, failed action assertions and behaviour differences still fail",+ )ap.add_argument("--allow-undecided",metavar = "ACTIONS",@@ -2343,7 +2363,12 @@ def main(argv: list[str] | None = None) -> int:)floor = frozenset()vis = visible_report(- paths, f"UI PARITY: {pattern}", floor, select = win, min_reps = args.min_reps+ paths,+ f"UI PARITY: {pattern}",+ floor,+ select = win,+ min_reps = args.min_reps,+ declared_ui_change = args.declared_ui_change,)worst = max(worst, int(vis))windowed_compared = frozenset(vis.compared)@@ -2495,7 +2520,14 @@ def main(argv: list[str] | None = None) -> int:# comparing nine windowed and nine structural pairs would fail a floor of sixteen twice over.# `args.min_compared` was the argument that went missing.# `report` (`elif decided == 0`) and `visible_report` (`and not not_run`) each check their own.- struct = report(paths, label, effective, select, args.min_reps)+ struct = report(+ paths,+ label,+ effective,+ select,+ args.min_reps,+ declared_ui_change = args.declared_ui_change,+ )worst = max(worst, int(struct))note(pattern, struct)# COMBINED, and the worst outcome wins: a behavioural failure on one payload is not cancelled by