Official Codebase of 3DCodeVerse
Python
judges/flywheel: three provenance and scoring-rule fixes (D36–D38)
ziyao039ec5075046
diff --git a/harness/codeverse/flywheel/record.py b/harness/codeverse/flywheel/record.pyindex 47f56ee8..1e80d850 100644--- a/harness/codeverse/flywheel/record.py+++ b/harness/codeverse/flywheel/record.py@@ -47,17 +47,38 @@ def _cmd_first_line(cmd: list[str]) -> str:return out[0].strip() if out else f"exit {proc.returncode}"-def _harness_git_sha() -> str:- repo = Path(__file__).resolve().parents[2]- if not (repo / ".git").exists():- return ""+def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str] | None:try:- proc = subprocess.run(- ["git", "-C", str(repo), "rev-parse", "HEAD"], capture_output=True, text=True, timeout=10, check=False- )+ return subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, timeout=10, check=False)except (OSError, subprocess.TimeoutExpired):+ return None+++def _harness_git_sha(pkg_file: Path | None = None) -> str:+ """Commit of the harness checkout this package runs from (``""`` outside a checkout).++ The repository root is wherever git finds it — ``harness/`` is a subdirectory of+ the 3dcodeverse repo, so ``.git`` lives one level ABOVE the package (the old+ ``parents[2] / ".git"`` probe never matched and every record shipped an empty sha).+ Two guards keep a wheel or a venv copy from borrowing an unrelated repo's sha:+ the file itself must be tracked by the repo git resolves to, and a modified+ working tree is marked ``-dirty`` so a score is never attributed to a clean commit+ it did not run on.+ """+ here = (pkg_file or Path(__file__)).resolve()+ tracked = _git(["ls-files", "--error-unmatch", here.name], here.parent)+ if tracked is None or tracked.returncode != 0:+ return ""+ head = _git(["rev-parse", "HEAD"], here.parent)+ if head is None or head.returncode != 0 or not head.stdout.strip():return ""- return proc.stdout.strip() if proc.returncode == 0 else ""+ sha = head.stdout.strip()+ # dirty = any tracked file under the package tree (harness/) modified; untracked files ignored+ pkg_root = here.parents[1] if here.parent.name == "flywheel" else here.parent+ status = _git(["status", "--porcelain", "--untracked-files=no", "--", str(pkg_root)], here.parent)+ if status is not None and status.returncode == 0 and status.stdout.strip():+ sha += "-dirty"+ return shadef _chrome_version() -> str:diff --git a/harness/codeverse/judges/prompt_builder.py b/harness/codeverse/judges/prompt_builder.pyindex 7cea3910..43c1e8f6 100644--- a/harness/codeverse/judges/prompt_builder.py+++ b/harness/codeverse/judges/prompt_builder.py@@ -176,25 +176,43 @@ def previous_section(prev: Judgment | None, round_index: int) -> str:return "\n".join(lines)+#: The fixed sentences of the view-rig paragraph (everything that is not a count or a+#: per-run probe line). Kept as constants so ``judge_prompt_hash`` covers them.+RIG_RULES: dict[str, str] = {+ "grid": (+ "Each montage is a ≤2×2 grid; every tile carries a label under it ('name · az/el[· mode][· t=]') and the "+ "montage's own label strip lists which view sits top-left / top-right / bottom-left / bottom-right. "+ "Cite evidence as 'MONTAGE k <position> (<view name>)'. Detail crops are zoomed regions of a view, labelled with what to look for."+ ),+ "azimuth": (+ "Azimuth 0° = looking at the FRONT of the object, increasing counter-clockwise seen from above (90° = the object's right side, 180° = back); "+ "elevation is degrees above the horizon (negative = looking up from below the ground plane, which reveals undersides and ground contact)."+ ),+ "geometry": "The GEOMETRY-ONLY montage shows the same object without materials/lighting: use it for holes, inverted (black) faces, intersections and floating parts; use the SHADED montage for materials and detail.",+ "poses": "POSE tiles show the SAME object with joints moved by the harness (tile label = joint@value or rest). Judge articulation only from them and the joint table.",+ "scene_cams": "Views named overview_* are harness cameras fitted to the scene bounds (layout X-ray); views named cam_* are the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.",+ "scene_craft": "On the authored cameras also read the CRAFT of the picture, not only its contents: depth layering (is there anything within a few metres framing the shot, and anything on the horizon), ground variation (blended materials, paths, dressed edges vs one flat colour), variety among repeated natural elements, small-prop dressing, and aerial perspective (distant things hazier than near ones). The same camera at two times appears as separate tiles — compare them pixel-for-pixel before answering nothing_moves.",+ "object": "All views show the same object. Use top + low views for symmetry, footprint and ground contact.",+}++def view_rig_section(renders: RenderSet, montages: list[Montage], *, scene: bool) -> str:n_grids = sum(1 for m in montages if not m.is_detail)n_detail = len(montages) - n_gridsbits = [f"VIEW RIG: {n_grids} MONTAGE image(s) follow" + (f" plus {n_detail} DETAIL CROP(s)" if n_detail else "") + ". "- "Each montage is a ≤2×2 grid; every tile carries a label under it ('name · az/el[· mode][· t=]') and the "- "montage's own label strip lists which view sits top-left / top-right / bottom-left / bottom-right. "- "Cite evidence as 'MONTAGE k <position> (<view name>)'. Detail crops are zoomed regions of a view, labelled with what to look for.",- "Azimuth 0° = looking at the FRONT of the object, increasing counter-clockwise seen from above (90° = the object's right side, 180° = back); elevation is degrees above the horizon (negative = looking up from below the ground plane, which reveals undersides and ground contact).",+ + RIG_RULES["grid"],+ RIG_RULES["azimuth"],]if any(m.kind == "geometry" for m in montages):- bits.append("The GEOMETRY-ONLY montage shows the same object without materials/lighting: use it for holes, inverted (black) faces, intersections and floating parts; use the SHADED montage for materials and detail.")+ bits.append(RIG_RULES["geometry"])if any(m.kind in ("poses", "pose_sheet") for m in montages):- bits.append("POSE tiles show the SAME object with joints moved by the harness (tile label = joint@value or rest). Judge articulation only from them and the joint table.")+ bits.append(RIG_RULES["poses"])if scene:- bits.append("Views named overview_* are harness cameras fitted to the scene bounds (layout X-ray); views named cam_* are the scene's own authored cameras (grade composition/lighting on those); 't=' is the animation time.")- bits.append("On the authored cameras also read the CRAFT of the picture, not only its contents: depth layering (is there anything within a few metres framing the shot, and anything on the horizon), ground variation (blended materials, paths, dressed edges vs one flat colour), variety among repeated natural elements, small-prop dressing, and aerial perspective (distant things hazier than near ones). The same camera at two times appears as separate tiles — compare them pixel-for-pixel before answering nothing_moves.")+ bits.append(RIG_RULES["scene_cams"])+ bits.append(RIG_RULES["scene_craft"])else:- bits.append("All views show the same object. Use top + low views for symmetry, footprint and ground contact.")+ bits.append(RIG_RULES["object"])if renders.console_errors:bits.append(f"PROBE: {len(renders.console_errors)} console error(s) during rendering, first: {_clip(renders.console_errors[0], 200)}")if renders.fps is not None:@@ -203,6 +221,31 @@ def view_rig_section(renders: RenderSet, montages: list[Montage], *, scene: boolreturn "\n".join(bits)+def judge_prompt_hash(rubric: Rubric) -> str:+ """Hash of everything the judge is told that is constant for a rubric.++ Covers the system prompt (``_ROLE`` + the rubric block as rendered), the fixed+ view-rig rules and the wire schema's structure (field order is the+ observe-then-score protocol; ``EVAL.md`` §6 records that changing it moved the+ flash judge's σ from 0.01 to 0.08). Per-run content (brief, plan digest,+ measurements, acceptance ids, images) is deliberately excluded, so two runs judged+ under the same protocol share the hash and a prompt edit is visible in every+ ``ScoreBreakdown.judge_prompt_hash`` / ``record.prompt_hashes["judge"]`` it touched.+ ``rubric_hash`` (the YAML alone) stays alongside for the narrower question.+ """+ import json++ from codeverse.judges.output_schema import wire_schema+ from codeverse.prompts import prompt_hash++ payload = "\n".join([+ build_system_prompt(rubric),+ *(RIG_RULES[k] for k in sorted(RIG_RULES)),+ json.dumps(wire_schema(rubric, []), sort_keys=True),+ ])+ return prompt_hash(payload)++# --------------------------------------------------------------------------- imagesdef montage_image_parts(renders: RenderSet,diff --git a/harness/codeverse/judges/scoring.py b/harness/codeverse/judges/scoring.pyindex ab572ed6..bb3a29bf 100644--- a/harness/codeverse/judges/scoring.py+++ b/harness/codeverse/judges/scoring.py@@ -3,12 +3,20 @@* per-criterion score = mean over samples; overall = rubric-weighted mean;``score_std`` = std of the per-sample overalls;* floors: any criterion mean below its floor → verdict fails;-* defects: the rubric's binary checklist, majority vote over samples (ties →- present); each present item subtracts its ``penalty`` from the overall and- its ``cap`` (if any) bounds it — arithmetic the VLM never does;+* defects: the rubric's binary checklist, majority vote over samples; each+ present item subtracts its ``penalty`` from the overall and its ``cap`` (if+ any) bounds it — arithmetic the VLM never does;* caps: deterministic bounds from gate findings / acceptance / missing views(``caps.py``) plus the defect caps;-* acceptance: majority vote over samples (ties → False);+* acceptance: majority vote over samples;+* an EXACT tie on a binary item (only possible with an even ``n_samples``) follows+ the *representative* sample (below) instead of a fixed direction. The old rule+ (defect ties → present, acceptance ties → False) made ``n_samples=2`` strictly+ harsher than both n=1 and n=3 — one dissenting sample applied every penalty and+ failed every must item — so scores from the ``economy`` profile (n=2) were not+ comparable with the others. With the representative rule the probability that+ an item is flagged at n=2 equals n=1's, while the continuous scores still average+ over both samples; ids decided this way are listed in ``tie_broken``;* narrative (summary / issues / plan / strengths) is taken from the sample whoseoverall is closest to the mean (the *representative* sample) so it is coherent."""@@ -50,7 +58,11 @@ class ScoreBreakdown(BaseModel):n_requested: intn_used: intsample_errors: list[str] = Field(default_factory=list)+ tie_broken: list[str] = Field(default_factory=list,+ description="defect / acceptance ids an exact vote tie handed to the representative sample")rubric_hash: str = ""+ judge_prompt_hash: str = Field(default="", description="hash of everything the judge is told that is constant per rubric: "+ "role prompt + view-rig rules + wire schema (prompt_builder.judge_prompt_hash)")samples: list[dict[str, Any]] = Field(default_factory=list)@@ -63,6 +75,16 @@ def _representative(samples: list[JudgeOutput], overalls: list[float]) -> JudgeOreturn min(zip(samples, overalls, strict=True), key=lambda so: abs(so[1] - mean))[0]+def _vote(votes: list[bool], tie_break: bool) -> tuple[bool, bool]:+ """Majority of ``votes``; an exact tie takes ``tie_break``. Returns (decision, was_tie)."""+ if not votes:+ return False, False+ yes = sum(1 for v in votes if v)+ if yes * 2 == len(votes):+ return bool(tie_break), True+ return yes * 2 > len(votes), False++def aggregate_samples(rubric: Rubric,samples: list[JudgeOutput],@@ -75,6 +97,7 @@ def aggregate_samples(judge_backend: str = "",n_requested: int | None = None,sample_errors: list[str] | None = None,+ judge_prompt_hash: str = "",) -> Judgment:"""Combine ≥ 1 parsed samples into a Judgment (raises ValueError on zero samples).@@ -97,20 +120,30 @@ def aggregate_samples(floors = [{"criterion": cid, "score": round(sc, 4), "floor": fl} for cid, sc, fl in rubric.floors_hit(mean_scores)]+ rep = _representative(samples, overalls)+ tie_broken: list[str] = []votes: dict[str, list[bool]] = {}for s in samples:for aid, ok in s.acceptance_bools.items():votes.setdefault(aid, []).append(bool(ok))for a in acceptance_items:votes.setdefault(a.id, [False])- acceptance = {aid: (sum(v) * 2 > len(v)) for aid, v in votes.items()}+ acceptance: dict[str, bool] = {}+ for aid, v in votes.items():+ acceptance[aid], tied = _vote(v, bool(rep.acceptance_bools.get(aid, False)))+ if tied:+ tie_broken.append(aid)must_missing = missing_must_items(acceptance_items, acceptance)d_votes: dict[str, list[bool]] = {d.id: [] for d in rubric.defects}for s in samples:for did, flag in s.defects.items():d_votes.setdefault(did, []).append(bool(flag))- defects = {did: (sum(v) * 2 >= len(v) and bool(v)) for did, v in d_votes.items()}+ defects: dict[str, bool] = {}+ for did, v in d_votes.items():+ defects[did], tied = _vote(v, bool(rep.defects.get(did, False)))+ if tied:+ tie_broken.append(did)penalty = round(sum(rubric.defect(did).penalty for did, on in defects.items() if on), 4)after_defects = max(0.0, overall_uncapped - penalty)@@ -119,7 +152,6 @@ def aggregate_samples(overall = caps.overallpassed = overall >= rubric.pass_threshold and not floors and not must_missing- rep = _representative(samples, overalls)summary = rep.summary.strip()tail = _verdict_tail(rubric, overall, passed, floors, caps, must_missing, defects=defects, penalty=penalty)if tail:@@ -142,7 +174,9 @@ def aggregate_samples(n_requested=n_requested or len(samples),n_used=len(samples),sample_errors=sample_errors or [],+ tie_broken=tie_broken,rubric_hash=rubric.content_hash(),+ judge_prompt_hash=judge_prompt_hash,samples=[s.model_dump(mode="json") for s in samples],)return Judgment(@@ -180,7 +214,8 @@ def _verdict_tail(return " | ".join(bits) + "]"-def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int) -> Judgment:+def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend: str, n_requested: int,+ judge_prompt_hash: str = "") -> Judgment:"""A verdict the orchestrator must treat as a GLITCH (not a score): ``raw.status == 'degraded'``."""breakdown = {"status": "degraded",@@ -188,6 +223,7 @@ def degraded_judgment(rubric: Rubric, error: str, *, usage: Usage, judge_backend"n_requested": n_requested,"n_used": 0,"rubric_hash": rubric.content_hash(),+ "judge_prompt_hash": judge_prompt_hash,}return Judgment(rubric=rubric.name,diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.pyindex b2ec16a9..16ccf0d8 100644--- a/harness/codeverse/judges/vlm_judge.py+++ b/harness/codeverse/judges/vlm_judge.py@@ -34,7 +34,7 @@ from codeverse.judges.output_schema import (parse_judge_output,wire_schema,)-from codeverse.judges.prompt_builder import build_judge_messages+from codeverse.judges.prompt_builder import build_judge_messages, judge_prompt_hashfrom codeverse.judges.rubrics import Rubric, load_rubricfrom codeverse.judges.scoring import aggregate_samples, degraded_judgmentfrom codeverse.models.base import ChatModel, ModelError@@ -75,6 +75,9 @@ class VlmJudge:self.rubric: Rubric = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)self.model_id = model_id or get_settings().default_judgeself.n_samples = max(1, int(n_samples))+ if self.n_samples % 2 == 0:+ log.warning("judge n_samples=%d is even: exact vote ties on defects / acceptance items are decided by the "+ "representative sample (scoring.py); an odd n gives a true majority", self.n_samples)self.temperature = temperatureself.thinking = thinkingself.max_attempts = max(1, int(max_attempts))@@ -87,6 +90,12 @@ class VlmJudge:self.cache_dir = cache_dirself.label = label+ # ------------------------------------------------------------------ provenance+ @property+ def prompt_hash(self) -> str:+ """``prompt_builder.judge_prompt_hash`` for this judge's rubric (recorded per verdict)."""+ return judge_prompt_hash(self.rubric)+# ------------------------------------------------------------------ model@propertydef model(self) -> ChatModel:@@ -150,12 +159,14 @@ class VlmJudge:errors.append(err)if not samples:return degraded_judgment(- self.rubric, " || ".join(errors)[:2000], usage=usage, judge_backend=self.model_id, n_requested=self.n_samples+ self.rubric, " || ".join(errors)[:2000], usage=usage, judge_backend=self.model_id, n_requested=self.n_samples,+ judge_prompt_hash=self.prompt_hash,)return aggregate_samples(self.rubric, samples, gates=inp.gates, acceptance_items=inp.acceptance,console_errors=inp.renders.console_errors, views=list(inp.renders.views), usage=usage,judge_backend=self.model_id, n_requested=self.n_samples, sample_errors=errors,+ judge_prompt_hash=self.prompt_hash,)# ------------------------------------------------------------------ hook (ReferenceJudge overrides)diff --git a/harness/codeverse/tracks/lifecycle.py b/harness/codeverse/tracks/lifecycle.pyindex 65c2b24a..314608e1 100644--- a/harness/codeverse/tracks/lifecycle.py+++ b/harness/codeverse/tracks/lifecycle.py@@ -396,6 +396,11 @@ class BaseTrack:"""Bind judge / generator backends once the plan exists (rubric threshold → policy target)."""if ctx.judge is None:ctx.judge = self.make_judge(ctx)+ # provenance: the judge protocol (role prompt + rig rules + wire schema) is hashed+ # like the generator prompts, so a judge-prompt edit is visible in record.json+ judge_hash = getattr(ctx.judge, "prompt_hash", "")+ if isinstance(judge_hash, str) and judge_hash:+ ctx.prompt_hashes["judge"] = judge_hashif self._policy is None:thr = self.services.rubric_threshold(ctx.rubric)if thr is not None:diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex 91ed26c3..347b0a70 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -254,7 +254,8 @@ labelled 2×2 montages (shaded / geometry-only clay-or-normals / poses) + ≤ 2crops at ≤ 1024 px, shuffled per sample. The wire schema is **observe-then-score**(summary, strengths, issues, **defect checklist**, acceptance *before* criteria) —criteria-first measurably compressed flash to 0.6–0.7. Every rubric carries binary-`defects` (id/text/penalty/cap); defect votes are majority (ties → present) and+`defects` (id/text/penalty/cap); defect and acceptance votes are majority (an exact+tie — even `n_samples` only — follows the representative sample, D36) and`overall = caps(weighted_mean − Σ penalties)`. Floors, deterministic caps from gatefindings (`data["kind"]`), console errors, missing must-acceptance and`missing_views` rules apply on top; degraded verdicts are glitches, not scores.diff --git a/harness/docs/DECISIONS.md b/harness/docs/DECISIONS.mdindex 994a995a..2284ae6c 100644--- a/harness/docs/DECISIONS.md+++ b/harness/docs/DECISIONS.md@@ -169,6 +169,35 @@ written) that were accepted because the code works that way and the tests pin itbecome description hints); `pricing.lookup_price` prefix fallback matches onlyversion/date/channel suffixes, so sibling models never inherit a parent's price.+* **D36 Vote ties follow the representative sample (2026-08-25).** Context: defect+ votes were majority with ties → present and acceptance votes majority with ties →+ False, so at `n_samples=2` ONE dissenting sample applied every penalty/cap and+ failed every must item — n=2 was strictly harsher than n=1 and n=3, and the+ `economy` profile (n=2) was not comparable with the others. Decision: an exact+ tie (even n only) takes the *representative* sample's answer (the sample whose+ overall is closest to the mean — already the narrative source); ids decided this+ way are listed in `ScoreBreakdown.tie_broken`; `VlmJudge` warns on an even n.+ Consequence: P(item flagged) at n=2 equals n=1's while the continuous scores still+ average over both samples; odd n is unchanged.+* **D37 The judge protocol is hashed (2026-08-25).** Context: only the rubric YAML was+ hashed; the role prompt, the view-rig rules and the wire schema (whose field order+ IS the observe-then-score protocol — EVAL.md §6 measured σ 0.01 → 0.08 when it+ changed) were not, so a prompt edit left no trace in any record. Decision:+ `prompt_builder.judge_prompt_hash(rubric)` = hash(system prompt + rig rules ++ wire-schema structure); stored in every `ScoreBreakdown.judge_prompt_hash` and,+ for runs, `record.prompt_hashes["judge"]` (`BaseTrack.after_plan`). Per-run content+ is excluded so runs under one protocol share the hash.+* **D38 `harness_git_sha` resolves through git, not a `.git` probe (2026-08-25).**+ Context: `flywheel/record._harness_git_sha` looked for `harness/.git`; the repo's+ `.git` is one level up, so every record shipped an empty sha and EVAL.md §1.7's+ provenance was never met. Decision: `git ls-files --error-unmatch` on the module+ itself (a wheel / venv copy must not borrow an unrelated repo's sha), then+ `rev-parse HEAD`, `-dirty` appended when tracked files under `harness/` are modified.+* **D39 CI runs every offline test directory (2026-08-25).** Context: `ci.yml`+ listed 7 of 24 directories, so 828 pure-python tests — every docs-vs-code drift+ guard among them — never ran on a PR. Decision: `pytest tests -m "not live and+ not blender and not node"` on both python floors; `timeout-minutes` on both jobs.+## Rejected / deferred* Registering `single-shot` as a CodingAgent kind (rejected: it has no tools/session).diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex 3e05f805..1e84341f 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -262,7 +262,8 @@ from codeverse.judges.base import JudgeInput # (spec, renders, measurement=Nonfrom codeverse.judges.montage import plan_montages, render_montage, Montage # ≤3 2×2 montages (shaded/geometry/poses) + ≤2 detail# crops @≤1024px replace sheet+9 views; clay/normals views (RenderView.mode) auto-route to the GEOMETRY montagefrom codeverse.judges.scoring import is_degraded, aggregate_samples # ScoreBreakdown adds defects, defect_votes (majority,- # ties→present), defect_penalty, overall_after_defects; overall = caps(weighted_mean − Σpenalty)+ # ties→representative sample, D36), tie_broken, defect_penalty, overall_after_defects,+ # judge_prompt_hash (D37); overall = caps(weighted_mean − Σpenalty)from codeverse.judges.caps import apply_caps # (rubric, overall, gates, acceptance_results, acceptance_items=None, *,# console_errors=None, views=None, defects_present=None) -> CapResult;# cap rules add when="missing_views" and ledger lines "defect:<id>"diff --git a/harness/docs/RELEASE_NOTES.md b/harness/docs/RELEASE_NOTES.mdindex a70fd9bf..c87a34f3 100644--- a/harness/docs/RELEASE_NOTES.md+++ b/harness/docs/RELEASE_NOTES.md@@ -1,3 +1,20 @@+# Addendum — evaluation integrity, 2026-08-25++Four changes that affect what a recorded score MEANS, none of which changes a+score computed under an odd `n_samples` (branch `ziyao/eval-integrity`, D36–D39):++* `record.environment.harness_git_sha` is populated (it was always `""`: the probe+ looked for `harness/.git`, which does not exist). `-dirty` marks a modified tree.+* `judge_prompt_hash` (role prompt + rig rules + wire schema) is stored in every+ `ScoreBreakdown` and in `record.prompt_hashes["judge"]`.+* Exact vote ties on defects / acceptance items follow the representative sample+ instead of a fixed direction; `ScoreBreakdown.tie_broken` names the ids. Under+ the old rule `economy` (n=2) was strictly harsher than `balanced` (n=1) and+ `quality` (n=3).+* CI runs the whole offline suite (was 7 of 24 test directories).++---+# Release notes — stability pass, 2026-08-24Sign-off for the first release intended to be called **stable**. Five parallel waves hunteddiff --git a/harness/tests/flywheel_cli/test_record_env.py b/harness/tests/flywheel_cli/test_record_env.pynew file mode 100644index 00000000..691d1fc8--- /dev/null+++ b/harness/tests/flywheel_cli/test_record_env.py@@ -0,0 +1,71 @@+"""D38: ``record.environment.harness_git_sha`` names the checkout the harness ran from."""++import re+import shutil+import subprocess+from pathlib import Path++import pytest++from codeverse.flywheel import record as record_mod+from codeverse.flywheel.record import _harness_git_sha, environment_versions++SHA_RE = re.compile(r"^[0-9a-f]{40}(-dirty)?$")+++def _git(*args: str, cwd: Path) -> subprocess.CompletedProcess[str]:+ return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=False)+++def _in_a_checkout() -> bool:+ here = Path(record_mod.__file__).resolve()+ return shutil.which("git") is not None and _git("ls-files", "--error-unmatch", here.name, cwd=here.parent).returncode == 0++[email protected](not _in_a_checkout(), reason="harness is not running from a git checkout (wheel / sdist install)")+def test_harness_git_sha_is_populated_inside_the_repo():+ # the repo's .git lives ABOVE harness/ — the old probe (harness/.git) never matched+ sha = _harness_git_sha()+ assert SHA_RE.match(sha), sha+ assert environment_versions()["harness_git_sha"] == sha+++def test_harness_git_sha_from_a_tracked_file_and_dirty_marker(tmp_path):+ if shutil.which("git") is None:+ pytest.skip("git not installed")+ repo = tmp_path / "repo"+ pkg = repo / "harness" / "codeverse" / "flywheel"+ pkg.mkdir(parents=True)+ mod = pkg / "record.py"+ mod.write_text("# tracked\n")+ _git("init", "-q", cwd=repo)+ _git("-c", "user.email=t@t", "-c", "user.name=t", "add", "-A", cwd=repo)+ _git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "init", cwd=repo)+ head = _git("rev-parse", "HEAD", cwd=repo).stdout.strip()+ assert _harness_git_sha(mod) == head+ # a modified tracked file under harness/ marks the sha dirty; an untracked file does not+ (repo / "harness" / "notes.txt").write_text("scratch")+ assert _harness_git_sha(mod) == head+ mod.write_text("# edited\n")+ assert _harness_git_sha(mod) == head + "-dirty"+++def test_harness_git_sha_is_empty_for_an_untracked_copy(tmp_path):+ if shutil.which("git") is None:+ pytest.skip("git not installed")+ # a venv / wheel copy inside SOME repo must not borrow that repo's sha+ repo = tmp_path / "other"+ repo.mkdir()+ _git("init", "-q", cwd=repo)+ (repo / "tracked.txt").write_text("x")+ _git("-c", "user.email=t@t", "-c", "user.name=t", "add", "-A", cwd=repo)+ _git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "init", cwd=repo)+ copy = repo / ".venv" / "lib" / "codeverse" / "flywheel" / "record.py"+ copy.parent.mkdir(parents=True)+ copy.write_text("# untracked copy\n")+ assert _harness_git_sha(copy) == ""+ # and outside any repository+ loose = tmp_path / "loose" / "record.py"+ loose.parent.mkdir()+ loose.write_text("# no git\n")+ assert _harness_git_sha(loose) == ""diff --git a/harness/tests/judges/test_defects.py b/harness/tests/judges/test_defects.pyindex 7f7c4aa3..a1e13ca1 100644--- a/harness/tests/judges/test_defects.py+++ b/harness/tests/judges/test_defects.py@@ -84,18 +84,54 @@ def test_penalties_and_caps_computed_in_code(judge_input, cache_dir):assert raw2["samples"][0]["defects"]["floating_part"] is True-def test_defect_majority_vote_ties_count_as_present(judge_input, cache_dir):+def test_defect_majority_vote_with_odd_n(judge_input, cache_dir):model = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)],":s2": [_reply(0.9)]})j = VlmJudge("static_object_v1", chat_model=model, n_samples=3, cache_dir=cache_dir).judge(judge_input)raw = json.loads(j.raw)assert raw["defects"]["render_artifacts"] is False and raw["defect_votes"]["render_artifacts"] == [True, False, False]- assert j.overall == pytest.approx(0.9)- model2 = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)]})- j2 = VlmJudge("static_object_v1", chat_model=model2, n_samples=2, cache_dir=cache_dir).judge(judge_input)- assert json.loads(j2.raw)["defects"]["render_artifacts"] is True and j2.overall == pytest.approx(0.85)+ assert raw["tie_broken"] == [] and j.overall == pytest.approx(0.9)# per-criterion std is reported for n-sample runs- assert set(json.loads(j2.raw)["per_criterion_std"]) == set(R.weights)+ assert set(raw["per_criterion_std"]) == set(R.weights)+++def test_defect_vote_ties_follow_the_representative_sample(judge_input, cache_dir, caplog):+ """D36: an exact tie (even n) is decided by the representative sample, not a fixed direction.++ Under the old rule (ties → present) ONE dissenting sample at n=2 applied the penalty,+ so n=2 was strictly harsher than n=1 and n=3. With two samples both are equally close+ to the mean, so the representative is the first one — the decision follows s0 in BOTH+ directions, and the id is reported in ``tie_broken``.+ """+ flagged_first = FakeChatModel(by_label={":s0": [_reply(0.9, ["render_artifacts"])], ":s1": [_reply(0.9)]})+ with caplog.at_level("WARNING", logger="codeverse.judges.vlm_judge"):+ j = VlmJudge("static_object_v1", chat_model=flagged_first, n_samples=2, cache_dir=cache_dir).judge(judge_input)+ assert "n_samples=2 is even" in caplog.text+ raw = json.loads(j.raw)+ assert raw["defect_votes"]["render_artifacts"] == [True, False]+ assert raw["defects"]["render_artifacts"] is True and j.overall == pytest.approx(0.85)+ assert raw["tie_broken"] == ["render_artifacts"]++ clean_first = FakeChatModel(by_label={":s0": [_reply(0.9)], ":s1": [_reply(0.9, ["render_artifacts"])]})+ j2 = VlmJudge("static_object_v1", chat_model=clean_first, n_samples=2, cache_dir=cache_dir).judge(judge_input)+ raw2 = json.loads(j2.raw)+ assert raw2["defect_votes"]["render_artifacts"] == [False, True]+ assert raw2["defects"]["render_artifacts"] is False and j2.overall == pytest.approx(0.9)+ assert raw2["tie_broken"] == ["render_artifacts"]+++def test_acceptance_vote_ties_follow_the_representative_sample(judge_input, cache_dir):+ """Same rule for acceptance items: the old ties → False failed a must item on one dissent."""+ a_ok, a_no = _reply(0.9), _reply(0.9)+ a_no["acceptance"]["A1"]["verified"] = False+ j = VlmJudge("static_object_v1", chat_model=FakeChatModel(by_label={":s0": [a_ok], ":s1": [a_no]}),+ n_samples=2, cache_dir=cache_dir).judge(judge_input)+ assert j.acceptance_results["A1"] is True and json.loads(j.raw)["tie_broken"] == ["A1"]+ b_ok, b_no = _reply(0.9), _reply(0.9)+ b_no["acceptance"]["A1"]["verified"] = False+ j2 = VlmJudge("static_object_v1", chat_model=FakeChatModel(by_label={":s0": [b_no], ":s1": [b_ok]}),+ n_samples=2, cache_dir=cache_dir).judge(judge_input)+ assert j2.acceptance_results["A1"] is False and json.loads(j2.raw)["tie_broken"] == ["A1"]def test_missing_views_cap_for_articulated():diff --git a/harness/tests/judges/test_vlm_judge.py b/harness/tests/judges/test_vlm_judge.pyindex f5dc6926..0059bd81 100644--- a/harness/tests/judges/test_vlm_judge.py+++ b/harness/tests/judges/test_vlm_judge.py@@ -187,3 +187,38 @@ def test_judge_payload_size_comes_from_the_settings_dial(monkeypatch):s2.apply_profile("economy") # a stated payload survives a profileassert VlmJudge(rubric="static_object_v1", model_id="fake:fake-1").detail_crops == 0get_settings.cache_clear()+++# ----------------------------------------------------------------------------- D37 judge protocol hash+def test_judge_prompt_hash_is_recorded_in_every_verdict(judge_input, cache_dir):+ from codeverse.judges.prompt_builder import judge_prompt_hash++ j = _judge(FakeChatModel([good_reply(R, IDS, 0.8)]), cache_dir=cache_dir).judge(judge_input)+ raw = json.loads(j.raw)+ assert raw["judge_prompt_hash"] == judge_prompt_hash(R) and len(raw["judge_prompt_hash"]) == 12+ assert raw["rubric_hash"] == R.content_hash() and raw["judge_prompt_hash"] != raw["rubric_hash"]+ # a degraded verdict carries it too, so a glitch is still attributable to a protocol+ d = _judge(FakeChatModel([ModelError("down", retryable=False)]), cache_dir=cache_dir).judge(judge_input)+ assert is_degraded(d) and json.loads(d.raw)["judge_prompt_hash"] == judge_prompt_hash(R)+ assert _judge(FakeChatModel([]), cache_dir=cache_dir).prompt_hash == judge_prompt_hash(R)+++def test_judge_prompt_hash_tracks_the_protocol_not_the_run(monkeypatch):+ from codeverse.judges import prompt_builder as pb+ from codeverse.judges.output_schema import wire_schema+ from codeverse.judges.prompt_builder import judge_prompt_hash++ base = judge_prompt_hash(R)+ assert judge_prompt_hash(R) == base # deterministic+ # per-run content (acceptance ids) is NOT part of it: the schema is hashed with no ids+ assert wire_schema(R, ["A1"]) != wire_schema(R, []) and judge_prompt_hash(R) == base+ # the rubric is part of it (through the rendered rubric block)+ other = load_rubric("shader_v1")+ assert judge_prompt_hash(other) != base+ # an edit to the role prompt or the rig rules changes it — that is the whole point+ monkeypatch.setattr(pb, "_ROLE", pb._ROLE + "\nScore generously.")+ edited_role = judge_prompt_hash(R)+ assert edited_role != base+ monkeypatch.setattr(pb, "RIG_RULES", {**pb.RIG_RULES, "object": "All views show the same object."})+ assert judge_prompt_hash(R) not in (base, edited_role)+diff --git a/harness/tests/orchestrator_tracks/fakes.py b/harness/tests/orchestrator_tracks/fakes.pyindex 729b9914..a4faf216 100644--- a/harness/tests/orchestrator_tracks/fakes.py+++ b/harness/tests/orchestrator_tracks/fakes.py@@ -197,6 +197,7 @@ class FakeChatModel:# ----------------------------------------------------------------------------- judgeclass FakeJudge:name = "fake"+ prompt_hash = "fakejudge001" # D37: BaseTrack.after_plan records it as prompt_hashes["judge"]def __init__(self, scores: Sequence[float] = (0.55, 0.7, 0.85), *, targets: Sequence[str] = ("Seat", "FrontLeg", "Backrest", "Armrest"),acceptance_fail: Sequence[str] = (), cost: float = 0.003):diff --git a/harness/tests/orchestrator_tracks/test_tracks.py b/harness/tests/orchestrator_tracks/test_tracks.pyindex 3b8c6db3..9f628c5f 100644--- a/harness/tests/orchestrator_tracks/test_tracks.py+++ b/harness/tests/orchestrator_tracks/test_tracks.py@@ -82,6 +82,8 @@ def test_static_track_end_to_end_agent_path(tmp_path, chair_plan, settings):assert all("EDIT ONLY THESE FILES" in j.prompt for j in agent.jobs if j.label.startswith("refine_"))# git history: one commit per roundassert len({r.commit for r in rec.rounds}) == 3 and ws.head() == rec.rounds[2].commit+ # provenance (D37): the judge protocol hash sits next to the generator prompt hashes+ assert rec.prompt_hashes["judge"] == FakeJudge.prompt_hash and "generate" in rec.prompt_hashesdef test_static_track_single_shot_with_repair_and_budget_stop(tmp_path, chair_plan, settings):