Skip to content

Official Codebase of 3DCodeVerse

Python

judges/flywheel: three provenance and scoring-rule fixes (D36–D38)

ziyao039ec5075046
diff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.py
index 47f56ee8..1e80d850 100644
--- a/harness/codeverse/flywheel/record.py
+++ b/harness/codeverse/flywheel/record.py
@@ -47,17 +47,38 @@ def _cmd_first_line(cmd: list[str]) -> str:
return out[0].strip() if out else f"exit {proc.returncode}"
-def _harness_git_sha() -> str:
- repo = Path(__file__).resolve().parents[2]
- if not (repo / ".git").exists():
- return ""
+def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str] | None:
try:
- proc = subprocess.run(
- ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10, check=False
- )
+ return subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, timeout=10, check=False)
except (OSError, subprocess.TimeoutExpired):
+ return None
+
+
+def _harness_git_sha(pkg_file: Path | None = None) -> str:
+ """Commit of the harness checkout this package runs from (``""`` outside a checkout).
+
+ The repository root is wherever git finds it — ``harness/`` is a subdirectory of
+ the 3dcodeverse repo, so ``.git`` lives one level ABOVE the package (the old
+ ``parents[2] / ".git"`` probe never matched and every record shipped an empty sha).
+ Two guards keep a wheel or a venv copy from borrowing an unrelated repo's sha:
+ the file itself must be tracked by the repo git resolves to, and a modified
+ working tree is marked ``-dirty`` so a score is never attributed to a clean commit
+ it did not run on.
+ """
+ here = (pkg_file or Path(__file__)).resolve()
+ tracked = _git(["ls-files", "--error-unmatch", here.name], here.parent)
+ if tracked is None or tracked.returncode != 0:
+ return ""
+ head = _git(["rev-parse", "HEAD"], here.parent)
+ if head is None or head.returncode != 0 or not head.stdout.strip():
return ""
- return proc.stdout.strip() if proc.returncode == 0 else ""
+ sha = head.stdout.strip()
+ # dirty = any tracked file under the package tree (harness/) modified; untracked files ignored
+ pkg_root = here.parents[1] if here.parent.name == "flywheel" else here.parent
+ status = _git(["status", "--porcelain", "--untracked-files=no", "--", str(pkg_root)], here.parent)
+ if status is not None and status.returncode == 0 and status.stdout.strip():
+ sha += "-dirty"
+ return sha
def _chrome_version() -> str:
diff --git a/harness/codeverse/judges/prompt_builder.py b/harness/codeverse/judges/prompt_builder.py
index 7cea3910..43c1e8f6 100644
--- a/harness/codeverse/judges/prompt_builder.py
+++ b/harness/codeverse/judges/prompt_builder.py
@@ -176,25 +176,43 @@ def previous_section(prev: Judgment | None, round_index: int) -> str:
return "\n".join(lines)
+#: The fixed sentences of the view-rig paragraph (everything that is not a count or a
+#: per-run probe line). Kept as constants so ``judge_prompt_hash`` covers them.
+RIG_RULES: dict[str, str] = {
+ "grid": (
+ "Each montage is a ≤2×2 grid; every tile carries a label under it ('name · az/el[· mode][· t=]') and the "
+ "montage's own label strip lists which view sits top-left / top-right / bottom-left / bottom-right. "
+ "Cite evidence as 'MONTAGE k <position> (<view name>)'. Detail crops are zoomed regions of a view, labelled with what to look for."
+ ),
+ "azimuth": (
+ "Azimuth 0° = looking at the FRONT of the object, increasing counter-clockwise seen from above (90° = the object's right side, 180° = back); "
+ "elevation is degrees above the horizon (negative = looking up from below the ground plane, which reveals undersides and ground contact)."
+ ),
+ "geometry": "The GEOMETRY-ONLY montage shows the same object without materials/lighting: use it for holes, inverted (black) faces, intersections and floating parts; use the SHADED montage for materials and detail.",
+ "poses": "POSE tiles show the SAME object with joints moved by the harness (tile label = joint@value or rest). Judge articulation only from them and the joint table.",
+ "scene_cams": "Views named overview_* are harness cameras fitted to the scene bounds (layout X-ray); views named cam_* are the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.",
+ "scene_craft": "On the authored cameras also read the CRAFT of the picture, not only its contents: depth layering (is there anything within a few metres framing the shot, and anything on the horizon), ground variation (blended materials, paths, dressed edges vs one flat colour), variety among repeated natural elements, small-prop dressing, and aerial perspective (distant things hazier than near ones). The same camera at two times appears as separate tiles — compare them pixel-for-pixel before answering nothing_moves.",
+ "object": "All views show the same object. Use top + low views for symmetry, footprint and ground contact.",
+}
+
+
def view_rig_section(renders: RenderSet, montages: list[Montage], *, scene: bool) -> str:
n_grids = sum(1 for m in montages if not m.is_detail)
n_detail = len(montages) - n_grids
bits = [
f"VIEW RIG: {n_grids} MONTAGE image(s) follow" + (f" plus {n_detail} DETAIL CROP(s)" if n_detail else "") + ". "
- "Each montage is a ≤2×2 grid; every tile carries a label under it ('name · az/el[· mode][· t=]') and the "
- "montage's own label strip lists which view sits top-left / top-right / bottom-left / bottom-right. "
- "Cite evidence as 'MONTAGE k <position> (<view name>)'. Detail crops are zoomed regions of a view, labelled with what to look for.",
- "Azimuth 0° = looking at the FRONT of the object, increasing counter-clockwise seen from above (90° = the object's right side, 180° = back); elevation is degrees above the horizon (negative = looking up from below the ground plane, which reveals undersides and ground contact).",
+ + RIG_RULES["grid"],
+ RIG_RULES["azimuth"],
]
if any(m.kind == "geometry" for m in montages):
- bits.append("The GEOMETRY-ONLY montage shows the same object without materials/lighting: use it for holes, inverted (black) faces, intersections and floating parts; use the SHADED montage for materials and detail.")
+ bits.append(RIG_RULES["geometry"])
if any(m.kind in ("poses", "pose_sheet") for m in montages):
- bits.append("POSE tiles show the SAME object with joints moved by the harness (tile label = joint@value or rest). Judge articulation only from them and the joint table.")
+ bits.append(RIG_RULES["poses"])
if scene:
- bits.append("Views named overview_* are harness cameras fitted to the scene bounds (layout X-ray); views named cam_* are the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.")
- bits.append("On the authored cameras also read the CRAFT of the picture, not only its contents: depth layering (is there anything within a few metres framing the shot, and anything on the horizon), ground variation (blended materials, paths, dressed edges vs one flat colour), variety among repeated natural elements, small-prop dressing, and aerial perspective (distant things hazier than near ones). The same camera at two times appears as separate tiles — compare them pixel-for-pixel before answering nothing_moves.")
+ bits.append(RIG_RULES["scene_cams"])
+ bits.append(RIG_RULES["scene_craft"])
else:
- bits.append("All views show the same object. Use top + low views for symmetry, footprint and ground contact.")
+ bits.append(RIG_RULES["object"])
if renders.console_errors:
bits.append(f"PROBE: {len(renders.console_errors)} console error(s) during rendering, first: {_clip(renders.console_errors[0], 200)}")
if renders.fps is not None:
@@ -203,6 +221,31 @@ def view_rig_section(renders: RenderSet, montages: list[Montage], *, scene: bool
return "\n".join(bits)
+def judge_prompt_hash(rubric: Rubric) -> str:
+ """Hash of everything the judge is told that is constant for a rubric.
+
+ Covers the system prompt (``_ROLE`` + the rubric block as rendered), the fixed
+ view-rig rules and the wire schema's structure (field order is the
+ observe-then-score protocol; ``EVAL.md`` §6 records that changing it moved the
+ flash judge's σ from 0.01 to 0.08). Per-run content (brief, plan digest,
+ measurements, acceptance ids, images) is deliberately excluded, so two runs judged
+ under the same protocol share the hash and a prompt edit is visible in every
+ ``ScoreBreakdown.judge_prompt_hash`` / ``record.prompt_hashes["judge"]`` it touched.
+ ``rubric_hash`` (the YAML alone) stays alongside for the narrower question.
+ """
+ import json
+
+ from codeverse.judges.output_schema import wire_schema
+ from codeverse.prompts import prompt_hash
+
+ payload = "\n".join([
+ build_system_prompt(rubric),
+ *(RIG_RULES[k] for k in sorted(RIG_RULES)),
+ json.dumps(wire_schema(rubric, []), sort_keys=True),
+ ])
+ return prompt_hash(payload)
+
+
# --------------------------------------------------------------------------- images
def montage_image_parts(
renders: RenderSet,
diff --git a/harness/codeverse/judges/scoring.py b/harness/codeverse/judges/scoring.py
index ab572ed6..bb3a29bf 100644
--- a/harness/codeverse/judges/scoring.py
+++ b/harness/codeverse/judges/scoring.py
@@ -3,12 +3,20 @@
* per-criterion score = mean over samples; overall = rubric-weighted mean;
``score_std`` = std of the per-sample overalls;
* floors: any criterion mean below its floor → verdict fails;
-* defects: the rubric's binary checklist, majority vote over samples (ties →
- present); each present item subtracts its ``penalty`` from the overall and
- its ``cap`` (if any) bounds it — arithmetic the VLM never does;
+* defects: the rubric's binary checklist, majority vote over samples; each
+ present item subtracts its ``penalty`` from the overall and its ``cap`` (if
+ any) bounds it — arithmetic the VLM never does;
* caps: deterministic bounds from gate findings / acceptance / missing views
(``caps.py``) plus the defect caps;
-* acceptance: majority vote over samples (ties → False);
+* acceptance: majority vote over samples;
+* an EXACT tie on a binary item (only possible with an even ``n_samples``) follows
+ the *representative* sample (below) instead of a fixed direction. The old rule
+ (defect ties → present, acceptance ties → False) made ``n_samples=2`` strictly
+ harsher than both n=1 and n=3 — one dissenting sample applied every penalty and
+ failed every must item — so scores from the ``economy`` profile (n=2) were not
+ comparable with the others. With the representative rule the probability that
+ an item is flagged at n=2 equals n=1's, while the continuous scores still average
+ over both samples; ids decided this way are listed in ``tie_broken``;
* narrative (summary / issues / plan / strengths) is taken from the sample whose
overall is closest to the mean (the *representative* sample) so it is coherent.
"""
@@ -50,7 +58,11 @@ class ScoreBreakdown(BaseModel):
n_requested: int
n_used: int
sample_errors: list[str] = Field(default_factory=list)
+ tie_broken: list[str] = Field(default_factory=list,
+ description="defect / acceptance ids an exact vote tie handed to the representative sample")
rubric_hash: str = ""
+ judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "
+ "role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")
samples: list[dict[str, Any]] = Field(default_factory=list)
@@ -63,6 +75,16 @@ def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeO
return min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]
+def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:
+ """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""
+ if not votes:
+ return False, False
+ yes = sum(1 for v in votes if v)
+ if yes * 2 == len(votes):
+ return bool(tie_break), True
+ return yes * 2 > len(votes), False
+
+
def aggregate_samples(
rubric: Rubric,
samples: list[JudgeOutput],
@@ -75,6 +97,7 @@ def aggregate_samples(
judge_backend: str = "",
n_requested: int | None = None,
sample_errors: list[str] | None = None,
+ judge_prompt_hash: str = "",
) -> Judgment:
"""Combine ≥ 1 parsed samples into a Judgment (raises ValueError on zero samples).
@@ -97,20 +120,30 @@ def aggregate_samples(
floors = [
{"criterion": cid, "score": round(sc, 4), "floor": fl} for cid, sc, fl in rubric.floors_hit(mean_scores)
]
+ rep = _representative(samples, overalls)
+ tie_broken: list[str] = []
votes: dict[str, list[bool]] = {}
for s in samples:
for aid, ok in s.acceptance_bools.items():
votes.setdefault(aid, []).append(bool(ok))
for a in acceptance_items:
votes.setdefault(a.id, [False])
- acceptance = {aid: (sum(v) * 2 > len(v)) for aid, v in votes.items()}
+ acceptance: dict[str, bool] = {}
+ for aid, v in votes.items():
+ acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))
+ if tied:
+ tie_broken.append(aid)
must_missing = missing_must_items(acceptance_items, acceptance)
d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}
for s in samples:
for did, flag in s.defects.items():
d_votes.setdefault(did, []).append(bool(flag))
- defects = {did: (sum(v) * 2 >= len(v) and bool(v)) for did, v in d_votes.items()}
+ defects: dict[str, bool] = {}
+ for did, v in d_votes.items():
+ defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))
+ if tied:
+ tie_broken.append(did)
penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)
after_defects = max(0.0, overall_uncapped - penalty)
@@ -119,7 +152,6 @@ def aggregate_samples(
overall = caps.overall
passed = overall >= rubric.pass_threshold and not floors and not must_missing
- rep = _representative(samples, overalls)
summary = rep.summary.strip()
tail = _verdict_tail(rubric, overall, passed, floors, caps, must_missing, defects=defects, penalty=penalty)
if tail:
@@ -142,7 +174,9 @@ def aggregate_samples(
n_requested=n_requested or len(samples),
n_used=len(samples),
sample_errors=sample_errors or [],
+ tie_broken=tie_broken,
rubric_hash=rubric.content_hash(),
+ judge_prompt_hash=judge_prompt_hash,
samples=[s.model_dump(mode="json") for s in samples],
)
return Judgment(
@@ -180,7 +214,8 @@ def _verdict_tail(
return " | ".join(bits) + "]"
-def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int) -> Judgment:
+def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int,
+ judge_prompt_hash: str = "") -> Judgment:
"""A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""
breakdown = {
"status": "degraded",
@@ -188,6 +223,7 @@ def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend
"n_requested": n_requested,
"n_used": 0,
"rubric_hash": rubric.content_hash(),
+ "judge_prompt_hash": judge_prompt_hash,
}
return Judgment(
rubric=rubric.name,
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index b2ec16a9..16ccf0d8 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -34,7 +34,7 @@ from codeverse.judges.output_schema import (
parse_judge_output,
wire_schema,
)
-from codeverse.judges.prompt_builder import build_judge_messages
+from codeverse.judges.prompt_builder import build_judge_messages, judge_prompt_hash
from codeverse.judges.rubrics import Rubric, load_rubric
from codeverse.judges.scoring import aggregate_samples, degraded_judgment
from codeverse.models.base import ChatModel, ModelError
@@ -75,6 +75,9 @@ class VlmJudge:
self.rubric: Rubric = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)
self.model_id = model_id or get_settings().default_judge
self.n_samples = max(1, int(n_samples))
+ if self.n_samples % 2 == 0:
+ log.warning("judge n_samples=%d is even: exact vote ties on defects / acceptance items are decided by the "
+ "representative sample (scoring.py); an odd n gives a true majority", self.n_samples)
self.temperature = temperature
self.thinking = thinking
self.max_attempts = max(1, int(max_attempts))
@@ -87,6 +90,12 @@ class VlmJudge:
self.cache_dir = cache_dir
self.label = label
+ # ------------------------------------------------------------------ provenance
+ @property
+ def prompt_hash(self) -> str:
+ """``prompt_builder.judge_prompt_hash`` for this judge's rubric (recorded per verdict)."""
+ return judge_prompt_hash(self.rubric)
+
# ------------------------------------------------------------------ model
@property
def model(self) -> ChatModel:
@@ -150,12 +159,14 @@ class VlmJudge:
errors.append(err)
if not samples:
return degraded_judgment(
- self.rubric, " || ".join(errors)[:2000], usage=usage, judge_backend=self.model_id, n_requested=self.n_samples
+ self.rubric, " || ".join(errors)[:2000], usage=usage, judge_backend=self.model_id, n_requested=self.n_samples,
+ judge_prompt_hash=self.prompt_hash,
)
return aggregate_samples(
self.rubric, samples, gates=inp.gates, acceptance_items=inp.acceptance,
console_errors=inp.renders.console_errors, views=list(inp.renders.views), usage=usage,
judge_backend=self.model_id, n_requested=self.n_samples, sample_errors=errors,
+ judge_prompt_hash=self.prompt_hash,
)
# ------------------------------------------------------------------ hook (ReferenceJudge overrides)
diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.py
index 65c2b24a..314608e1 100644
--- a/harness/codeverse/tracks/lifecycle.py
+++ b/harness/codeverse/tracks/lifecycle.py
@@ -396,6 +396,11 @@ class BaseTrack:
"""Bind judge / generator backends once the plan exists (rubric threshold → policy target)."""
if ctx.judge is None:
ctx.judge = self.make_judge(ctx)
+ # provenance: the judge protocol (role prompt + rig rules + wire schema) is hashed
+ # like the generator prompts, so a judge-prompt edit is visible in record.json
+ judge_hash = getattr(ctx.judge, "prompt_hash", "")
+ if isinstance(judge_hash, str) and judge_hash:
+ ctx.prompt_hashes["judge"] = judge_hash
if self._policy is None:
thr = self.services.rubric_threshold(ctx.rubric)
if thr is not None:
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index 91ed26c3..347b0a70 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -254,7 +254,8 @@ labelled 2×2 montages (shaded / geometry-only clay-or-normals / poses) + ≤ 2
crops at ≤ 1024 px, shuffled per sample. The wire schema is **observe-then-score**
(summary, strengths, issues, **defect checklist**, acceptance *before* criteria) —
criteria-first measurably compressed flash to 0.6–0.7. Every rubric carries binary
-`defects` (id/text/penalty/cap); defect votes are majority (ties → present) and
+`defects` (id/text/penalty/cap); defect and acceptance votes are majority (an exact
+tie — even `n_samples` only — follows the representative sample, D36) and
`overall = caps(weighted_mean − Σ penalties)`. Floors, deterministic caps from gate
findings (`data["kind"]`), console errors, missing must-acceptance and
`missing_views` rules apply on top; degraded verdicts are glitches, not scores.
diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.md
index 994a995a..2284ae6c 100644
--- a/harness/docs/DECISIONS.md
+++ b/harness/docs/DECISIONS.md
@@ -169,6 +169,35 @@ written) that were accepted because the code works that way and the tests pin it
become description hints); `pricing.lookup_price` prefix fallback matches only
version/date/channel suffixes, so sibling models never inherit a parent's price.
+* **D36 Vote ties follow the representative sample (2026-08-25).** Context: defect
+ votes were majority with ties → present and acceptance votes majority with ties →
+ False, so at `n_samples=2` ONE dissenting sample applied every penalty/cap and
+ failed every must item — n=2 was strictly harsher than n=1 and n=3, and the
+ `economy` profile (n=2) was not comparable with the others. Decision: an exact
+ tie (even n only) takes the *representative* sample's answer (the sample whose
+ overall is closest to the mean — already the narrative source); ids decided this
+ way are listed in `ScoreBreakdown.tie_broken`; `VlmJudge` warns on an even n.
+ Consequence: P(item flagged) at n=2 equals n=1's while the continuous scores still
+ average over both samples; odd n is unchanged.
+* **D37 The judge protocol is hashed (2026-08-25).** Context: only the rubric YAML was
+ hashed; the role prompt, the view-rig rules and the wire schema (whose field order
+ IS the observe-then-score protocol — EVAL.md §6 measured σ 0.01 → 0.08 when it
+ changed) were not, so a prompt edit left no trace in any record. Decision:
+ `prompt_builder.judge_prompt_hash(rubric)` = hash(system prompt + rig rules +
+ wire-schema structure); stored in every `ScoreBreakdown.judge_prompt_hash` and,
+ for runs, `record.prompt_hashes["judge"]` (`BaseTrack.after_plan`). Per-run content
+ is excluded so runs under one protocol share the hash.
+* **D38 `harness_git_sha` resolves through git, not a `.git` probe (2026-08-25).**
+ Context: `flywheel/record._harness_git_sha` looked for `harness/.git`; the repo's
+ `.git` is one level up, so every record shipped an empty sha and EVAL.md §1.7's
+ provenance was never met. Decision: `git ls-files --error-unmatch` on the module
+ itself (a wheel / venv copy must not borrow an unrelated repo's sha), then
+ `rev-parse HEAD`, `-dirty` appended when tracked files under `harness/` are modified.
+* **D39 CI runs every offline test directory (2026-08-25).** Context: `ci.yml`
+ listed 7 of 24 directories, so 828 pure-python tests — every docs-vs-code drift
+ guard among them — never ran on a PR. Decision: `pytest tests -m "not live and
+ not blender and not node"` on both python floors; `timeout-minutes` on both jobs.
+
## Rejected / deferred
* Registering `single-shot` as a CodingAgent kind (rejected: it has no tools/session).
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 3e05f805..1e84341f 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -262,7 +262,8 @@ from codeverse.judges.base import JudgeInput # (spec, renders, measurement=Non
from codeverse.judges.montage import plan_montages, render_montage, Montage # ≤3 2×2 montages (shaded/geometry/poses) + ≤2 detail
# crops @≤1024px replace sheet+9 views; clay/normals views (RenderView.mode) auto-route to the GEOMETRY montage
from codeverse.judges.scoring import is_degraded, aggregate_samples # ScoreBreakdown adds defects, defect_votes (majority,
- # ties→present), defect_penalty, overall_after_defects; overall = caps(weighted_mean − Σpenalty)
+ # ties→representative sample, D36), tie_broken, defect_penalty, overall_after_defects,
+ # judge_prompt_hash (D37); overall = caps(weighted_mean − Σpenalty)
from codeverse.judges.caps import apply_caps # (rubric, overall, gates, acceptance_results, acceptance_items=None, *,
# console_errors=None, views=None, defects_present=None) -> CapResult;
# cap rules add when="missing_views" and ledger lines "defect:<id>"
diff --git a/harness/docs/RELEASE_NOTES.md b/harness/docs/RELEASE_NOTES.md
index a70fd9bf..c87a34f3 100644
--- a/harness/docs/RELEASE_NOTES.md
+++ b/harness/docs/RELEASE_NOTES.md
@@ -1,3 +1,20 @@
+# Addendum — evaluation integrity, 2026-08-25
+
+Four changes that affect what a recorded score MEANS, none of which changes a
+score computed under an odd `n_samples` (branch `ziyao/eval-integrity`, D36–D39):
+
+* `record.environment.harness_git_sha` is populated (it was always `""`: the probe
+ looked for `harness/.git`, which does not exist). `-dirty` marks a modified tree.
+* `judge_prompt_hash` (role prompt + rig rules + wire schema) is stored in every
+ `ScoreBreakdown` and in `record.prompt_hashes["judge"]`.
+* Exact vote ties on defects / acceptance items follow the representative sample
+ instead of a fixed direction; `ScoreBreakdown.tie_broken` names the ids. Under
+ the old rule `economy` (n=2) was strictly harsher than `balanced` (n=1) and
+ `quality` (n=3).
+* CI runs the whole offline suite (was 7 of 24 test directories).
+
+---
+
# Release notes — stability pass, 2026-08-24
Sign-off for the first release intended to be called **stable**. Five parallel waves hunted
diff --git a/harness/tests/flywheel_cli/test_record_env.py b/harness/tests/flywheel_cli/test_record_env.py
new file mode 100644
index 00000000..691d1fc8
--- /dev/null
+++ b/harness/tests/flywheel_cli/test_record_env.py
@@ -0,0 +1,71 @@
+"""D38: ``record.environment.harness_git_sha`` names the checkout the harness ran from."""
+
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from codeverse.flywheel import record as record_mod
+from codeverse.flywheel.record import _harness_git_sha, environment_versions
+
+SHA_RE = re.compile(r"^[0-9a-f]{40}(-dirty)?$")
+
+
+def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)
+
+
+def _in_a_checkout() -> bool:
+ here = Path(record_mod.__file__).resolve()
+ return shutil.which("git") is not None and _git("ls-files", "--error-unmatch", here.name, cwd=here.parent).returncode == 0
+
+
[email protected](not _in_a_checkout(), reason="harness is not running from a git checkout (wheel / sdist install)")
+def test_harness_git_sha_is_populated_inside_the_repo():
+ # the repo's .git lives ABOVE harness/ — the old probe (harness/.git) never matched
+ sha = _harness_git_sha()
+ assert SHA_RE.match(sha), sha
+ assert environment_versions()["harness_git_sha"] == sha
+
+
+def test_harness_git_sha_from_a_tracked_file_and_dirty_marker(tmp_path):
+ if shutil.which("git") is None:
+ pytest.skip("git not installed")
+ repo = tmp_path / "repo"
+ pkg = repo / "harness" / "codeverse" / "flywheel"
+ pkg.mkdir(parents=True)
+ mod = pkg / "record.py"
+ mod.write_text("# tracked\n")
+ _git("init", "-q", cwd=repo)
+ _git("-c", "user.email=t@t", "-c", "user.name=t", "add", "-A", cwd=repo)
+ _git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "init", cwd=repo)
+ head = _git("rev-parse", "HEAD", cwd=repo).stdout.strip()
+ assert _harness_git_sha(mod) == head
+ # a modified tracked file under harness/ marks the sha dirty; an untracked file does not
+ (repo / "harness" / "notes.txt").write_text("scratch")
+ assert _harness_git_sha(mod) == head
+ mod.write_text("# edited\n")
+ assert _harness_git_sha(mod) == head + "-dirty"
+
+
+def test_harness_git_sha_is_empty_for_an_untracked_copy(tmp_path):
+ if shutil.which("git") is None:
+ pytest.skip("git not installed")
+ # a venv / wheel copy inside SOME repo must not borrow that repo's sha
+ repo = tmp_path / "other"
+ repo.mkdir()
+ _git("init", "-q", cwd=repo)
+ (repo / "tracked.txt").write_text("x")
+ _git("-c", "user.email=t@t", "-c", "user.name=t", "add", "-A", cwd=repo)
+ _git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "init", cwd=repo)
+ copy = repo / ".venv" / "lib" / "codeverse" / "flywheel" / "record.py"
+ copy.parent.mkdir(parents=True)
+ copy.write_text("# untracked copy\n")
+ assert _harness_git_sha(copy) == ""
+ # and outside any repository
+ loose = tmp_path / "loose" / "record.py"
+ loose.parent.mkdir()
+ loose.write_text("# no git\n")
+ assert _harness_git_sha(loose) == ""
diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.py
index 7f7c4aa3..a1e13ca1 100644
--- a/harness/tests/judges/test_defects.py
+++ b/harness/tests/judges/test_defects.py
@@ -84,18 +84,54 @@ def test_penalties_and_caps_computed_in_code(judge_input, cache_dir):
assert raw2["samples"][0]["defects"]["floating_part"] is True
-def test_defect_majority_vote_ties_count_as_present(judge_input, cache_dir):
+def test_defect_majority_vote_with_odd_n(judge_input, cache_dir):
model = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)],
":s2": [_reply(0.9)]})
j = VlmJudge("static_object_v1", chat_model=model, n_samples=3, cache_dir=cache_dir).judge(judge_input)
raw = json.loads(j.raw)
assert raw["defects"]["render_artifacts"] is False and raw["defect_votes"]["render_artifacts"] == [True, False, False]
- assert j.overall == pytest.approx(0.9)
- model2 = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)]})
- j2 = VlmJudge("static_object_v1", chat_model=model2, n_samples=2, cache_dir=cache_dir).judge(judge_input)
- assert json.loads(j2.raw)["defects"]["render_artifacts"] is True and j2.overall == pytest.approx(0.85)
+ assert raw["tie_broken"] == [] and j.overall == pytest.approx(0.9)
# per-criterion std is reported for n-sample runs
- assert set(json.loads(j2.raw)["per_criterion_std"]) == set(R.weights)
+ assert set(raw["per_criterion_std"]) == set(R.weights)
+
+
+def test_defect_vote_ties_follow_the_representative_sample(judge_input, cache_dir, caplog):
+ """D36: an exact tie (even n) is decided by the representative sample, not a fixed direction.
+
+ Under the old rule (ties → present) ONE dissenting sample at n=2 applied the penalty,
+ so n=2 was strictly harsher than n=1 and n=3. With two samples both are equally close
+ to the mean, so the representative is the first one — the decision follows s0 in BOTH
+ directions, and the id is reported in ``tie_broken``.
+ """
+ flagged_first = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)]})
+ with caplog.at_level("WARNING", logger="codeverse.judges.vlm_judge"):
+ j = VlmJudge("static_object_v1", chat_model=flagged_first, n_samples=2, cache_dir=cache_dir).judge(judge_input)
+ assert "n_samples=2 is even" in caplog.text
+ raw = json.loads(j.raw)
+ assert raw["defect_votes"]["render_artifacts"] == [True, False]
+ assert raw["defects"]["render_artifacts"] is True and j.overall == pytest.approx(0.85)
+ assert raw["tie_broken"] == ["render_artifacts"]
+
+ clean_first = FakeChatModel(by_label={":s0": [_reply(0.9)], ":s1": [_reply(0.9, ["render_artifacts"])]})
+ j2 = VlmJudge("static_object_v1", chat_model=clean_first, n_samples=2, cache_dir=cache_dir).judge(judge_input)
+ raw2 = json.loads(j2.raw)
+ assert raw2["defect_votes"]["render_artifacts"] == [False, True]
+ assert raw2["defects"]["render_artifacts"] is False and j2.overall == pytest.approx(0.9)
+ assert raw2["tie_broken"] == ["render_artifacts"]
+
+
+def test_acceptance_vote_ties_follow_the_representative_sample(judge_input, cache_dir):
+ """Same rule for acceptance items: the old ties → False failed a must item on one dissent."""
+ a_ok, a_no = _reply(0.9), _reply(0.9)
+ a_no["acceptance"]["A1"]["verified"] = False
+ j = VlmJudge("static_object_v1", chat_model=FakeChatModel(by_label={":s0": [a_ok], ":s1": [a_no]}),
+ n_samples=2, cache_dir=cache_dir).judge(judge_input)
+ assert j.acceptance_results["A1"] is True and json.loads(j.raw)["tie_broken"] == ["A1"]
+ b_ok, b_no = _reply(0.9), _reply(0.9)
+ b_no["acceptance"]["A1"]["verified"] = False
+ j2 = VlmJudge("static_object_v1", chat_model=FakeChatModel(by_label={":s0": [b_no], ":s1": [b_ok]}),
+ n_samples=2, cache_dir=cache_dir).judge(judge_input)
+ assert j2.acceptance_results["A1"] is False and json.loads(j2.raw)["tie_broken"] == ["A1"]
def test_missing_views_cap_for_articulated():
diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.py
index f5dc6926..0059bd81 100644
--- a/harness/tests/judges/test_vlm_judge.py
+++ b/harness/tests/judges/test_vlm_judge.py
@@ -187,3 +187,38 @@ def test_judge_payload_size_comes_from_the_settings_dial(monkeypatch):
s2.apply_profile("economy") # a stated payload survives a profile
assert VlmJudge(rubric="static_object_v1", model_id="fake:fake-1").detail_crops == 0
get_settings.cache_clear()
+
+
+# ----------------------------------------------------------------------------- D37 judge protocol hash
+def test_judge_prompt_hash_is_recorded_in_every_verdict(judge_input, cache_dir):
+ from codeverse.judges.prompt_builder import judge_prompt_hash
+
+ j = _judge(FakeChatModel([good_reply(R, IDS, 0.8)]), cache_dir=cache_dir).judge(judge_input)
+ raw = json.loads(j.raw)
+ assert raw["judge_prompt_hash"] == judge_prompt_hash(R) and len(raw["judge_prompt_hash"]) == 12
+ assert raw["rubric_hash"] == R.content_hash() and raw["judge_prompt_hash"] != raw["rubric_hash"]
+ # a degraded verdict carries it too, so a glitch is still attributable to a protocol
+ d = _judge(FakeChatModel([ModelError("down", retryable=False)]), cache_dir=cache_dir).judge(judge_input)
+ assert is_degraded(d) and json.loads(d.raw)["judge_prompt_hash"] == judge_prompt_hash(R)
+ assert _judge(FakeChatModel([]), cache_dir=cache_dir).prompt_hash == judge_prompt_hash(R)
+
+
+def test_judge_prompt_hash_tracks_the_protocol_not_the_run(monkeypatch):
+ from codeverse.judges import prompt_builder as pb
+ from codeverse.judges.output_schema import wire_schema
+ from codeverse.judges.prompt_builder import judge_prompt_hash
+
+ base = judge_prompt_hash(R)
+ assert judge_prompt_hash(R) == base # deterministic
+ # per-run content (acceptance ids) is NOT part of it: the schema is hashed with no ids
+ assert wire_schema(R, ["A1"]) != wire_schema(R, []) and judge_prompt_hash(R) == base
+ # the rubric is part of it (through the rendered rubric block)
+ other = load_rubric("shader_v1")
+ assert judge_prompt_hash(other) != base
+ # an edit to the role prompt or the rig rules changes it — that is the whole point
+ monkeypatch.setattr(pb, "_ROLE", pb._ROLE + "\nScore generously.")
+ edited_role = judge_prompt_hash(R)
+ assert edited_role != base
+ monkeypatch.setattr(pb, "RIG_RULES", {**pb.RIG_RULES, "object": "All views show the same object."})
+ assert judge_prompt_hash(R) not in (base, edited_role)
+
diff --git a/harness/tests/orchestrator_tracks/fakes.py b/harness/tests/orchestrator_tracks/fakes.py
index 729b9914..a4faf216 100644
--- a/harness/tests/orchestrator_tracks/fakes.py
+++ b/harness/tests/orchestrator_tracks/fakes.py
@@ -197,6 +197,7 @@ class FakeChatModel:
# ----------------------------------------------------------------------------- judge
class FakeJudge:
name = "fake"
+ prompt_hash = "fakejudge001" # D37: BaseTrack.after_plan records it as prompt_hashes["judge"]
def __init__(self, scores: Sequence[float] = (0.55, 0.7, 0.85), *, targets: Sequence[str] = ("Seat", "FrontLeg", "Backrest", "Armrest"),
acceptance_fail: Sequence[str] = (), cost: float = 0.003):
diff --git a/harness/tests/orchestrator_tracks/test_tracks.py b/harness/tests/orchestrator_tracks/test_tracks.py
index 3b8c6db3..9f628c5f 100644
--- a/harness/tests/orchestrator_tracks/test_tracks.py
+++ b/harness/tests/orchestrator_tracks/test_tracks.py
@@ -82,6 +82,8 @@ def test_static_track_end_to_end_agent_path(tmp_path, chair_plan, settings):
assert all("EDIT ONLY THESE FILES" in j.prompt for j in agent.jobs if j.label.startswith("refine_"))
# git history: one commit per round
assert len({r.commit for r in rec.rounds}) == 3 and ws.head() == rec.rounds[2].commit
+ # provenance (D37): the judge protocol hash sits next to the generator prompt hashes
+ assert rec.prompt_hashes["judge"] == FakeJudge.prompt_hash and "generate" in rec.prompt_hashes
def test_static_track_single_shot_with_repair_and_budget_stop(tmp_path, chair_plan, settings):