Skip to content

Official Codebase of 3DCodeVerse

Python

readers: gallery, dataset, cost report, calibration and the CLI read select, not the record's best

gaoypengb98be358a3f2
diff --git a/harness/codeverse3d/addons/calibration.py b/harness/codeverse3d/addons/calibration.py
index b71002cb..8abb1ec1 100644
--- a/harness/codeverse3d/addons/calibration.py
+++ b/harness/codeverse3d/addons/calibration.py
@@ -9,8 +9,8 @@ tabulates, per round: gate error count, the stored overall, the new mean/std,
per-criterion std, defects and caps. Across rounds it reports the Pearson and
Spearman correlation between gate error counts and the new scores (expected
negative), and between stored and new overalls. Optionally renders a clay /
-normals geometry set for a best round that stored none (``artifacts/object.glb``) so
-the geometry montage is exercised. Output goes to ``out_dir`` (never into the run).
+normals geometry set for the picked round (``addons/select``) when it stored none — from
+its own ``artifacts/rNN/object.glb`` — so the geometry montage is exercised. Output goes to ``out_dir`` (never into the run).
CLI: ``python -m codeverse3d.addons.calibration runs/a runs/b --model gemini:gemini-3.7-flash --n 3 --out scratch/``
"""
@@ -28,6 +28,7 @@ from typing import Any
from pydantic import BaseModel, Field
+from codeverse3d.addons import select
from codeverse3d.cli._judge import build_judge_input, make_judge, rubric_for
from codeverse3d.contracts.artifacts import Judgment, RenderSet
from codeverse3d.contracts.run import RoundRecord, RunRecord
@@ -35,7 +36,6 @@ from codeverse3d.contracts.spec import Spec
from codeverse3d.judges.base import JudgeInput
from codeverse3d.judges.rubrics import is_degraded
from codeverse3d.judges.vlm_judge import VlmJudge
-from codeverse3d.proc import read_json_or_none
from codeverse3d.record.record import RecordError, load_record
from codeverse3d.workspace import Workspace
@@ -65,7 +65,7 @@ class RoundCase(BaseModel):
gate_errors: int
gate_warnings: int
stored: Judgment | None = None
- is_best: bool = False
+ is_picked: bool = False
glb: str | None = None
@@ -162,11 +162,11 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Ro
ws = Workspace(run_dir)
try:
run = load_record(ws)
- except RecordError: # an interrupted run: rounds/ + spec.json (+ whatever best_round it left)
- run = RunRecord(spec=Spec.model_validate_json(ws.spec_path.read_text()), workspace=str(run_dir),
- best_round=(read_json_or_none(ws.record_path) or {}).get("best_round"))
- best = run.best_round
- glb = run_dir / "artifacts" / "object.glb"
+ picked = select.summarise(run_dir, record=run).picked_round
+ except RecordError: # an interrupted run: rounds/ + spec.json
+ run = RunRecord(spec=Spec.model_validate_json(ws.spec_path.read_text()), workspace=str(run_dir))
+ picked = None
+ canonical = run_dir / "artifacts" / "object.glb" # where a round's GLB was before rounds kept their own
cases: list[RoundCase] = []
for path in sorted((run_dir / "rounds").glob("r*.json")):
rec = RoundRecord.model_validate_json(path.read_text())
@@ -180,16 +180,16 @@ def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[Ro
inp=build_judge_input(ws, run, rec),
gate_errors=sum(len(g.errors) for g in rec.gates),
gate_warnings=sum(1 for g in rec.gates for f in g.findings if f.severity.value == "warn"),
- stored=rec.judgment, is_best=(best == rec.index) if best is not None else False,
- glb=str(glb) if glb.is_file() else None,
+ stored=rec.judgment, is_picked=picked == rec.index,
+ glb=next((str(p) for p in (ws.round_artifacts(rec.index) / "object.glb", canonical) if p.is_file()), None),
))
- if best is None and cases: # no record: treat the last round as the one the GLB belongs to
- cases[-1].is_best = True
+ if picked is None and cases: # no record / nothing judged: the tree's GLB is its last round's
+ cases[-1].is_picked = True
return cases
def render_geometry_views(case: RoundCase, out_dir: Path, mode: str) -> RenderSet | None:
- """Clay/normals renders of the run's final GLB for the best round (object tracks only)."""
+ """Clay/normals renders of the picked round's GLB (object tracks only)."""
if not case.glb or case.inp.spec.track.value == "scene":
return None
from codeverse3d.conventions import OBJECT_CLAY_VIEWS
@@ -288,7 +288,7 @@ def calibrate(
raise ValueError(f"no judgeable rounds found under {list(map(str, run_dirs))}")
geometry: dict[int, RenderSet | None] = {}
for i, c in enumerate(cases): # a round's stored clay views win; render only where there are none
- want = geometry_mode and c.is_best and c.inp.geometry_views is None
+ want = geometry_mode and c.is_picked and c.inp.geometry_views is None
geometry[i] = render_geometry_views(c, out, geometry_mode) if want else None
def kind(c: RoundCase) -> tuple[str, bool]: # what make_judge's choice of class depends on
@@ -322,7 +322,7 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument("--model", default=None, help="chat model id, e.g. gemini:gemini-3.7-flash")
ap.add_argument("--n", type=int, default=3, help="samples per round")
ap.add_argument("--out", default=".", help="output directory (json + md)")
- ap.add_argument("--geometry", default="clay", help="clay | normals | none (geometry montage for the best round)")
+ ap.add_argument("--geometry", default="clay", help="clay | normals | none (geometry montage for the picked round)")
ap.add_argument("--rounds", default="", help="comma-separated round indices (default all)")
ap.add_argument("--thinking", default="low")
ap.add_argument("--workers", type=int, default=4)
diff --git a/harness/codeverse3d/addons/costreport/audit.py b/harness/codeverse3d/addons/costreport/audit.py
index f49f22f8..15ac72f3 100644
--- a/harness/codeverse3d/addons/costreport/audit.py
+++ b/harness/codeverse3d/addons/costreport/audit.py
@@ -7,10 +7,11 @@ stamps every row with its run's track / language / status, and aggregates:
* token composition (input / cached / output / thoughts) and the effective
$/1k-token rate each bucket actually paid;
* latency next to the money — wall clock, model time, harness (non-model) time;
-* **waste**: rounds that scored worse than the round before, judge verdicts paid
- for on rounds that were never promoted, repair loops that never converged,
- degraded verdicts, best-of-N candidates that lost, and spend after the budget
- was already blown.
+* **waste**: repair loops that never got their round to build, degraded verdicts,
+ best-of-N candidates that lost, and spend after the budget was already blown.
+ Nothing is waste for scoring below another round: since 2026-09-22 every round is
+ kept and any of them can be the one handed over (the "regression", "zero_delta_round"
+ and "unpromoted_judge" kinds measured against an in-run best that no longer exists).
Everything is measured from the recorded telemetry; nothing is estimated except
where a row says so (``price_source == "event-cost"`` — the event recorded a
@@ -19,7 +20,7 @@ cost but no tokens).
from __future__ import annotations
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable
from dataclasses import dataclass, field
from pathlib import Path
@@ -36,30 +37,13 @@ AUDIT_DIMENSIONS = ("run", "track", "language", "stage", "role", "backend", "mod
class WasteItem:
"""One dollar that bought nothing (or bought a worse artifact)."""
- kind: str # regression | unpromoted_judge | repair_no_converge | degraded_judge |
- # # lost_candidate | post_budget | zero_delta_round
+ kind: str # repair_no_converge | degraded_judge | lost_candidate | post_budget
run: str
usd: float
detail: str = ""
round: int | None = None
-@dataclass
-class RoundCost:
- run: str
- index: int
- usd: float
- score: float | None
- prev_score: float | None
- promoted: bool = False
-
- @property
- def delta(self) -> float | None:
- if self.score is None or self.prev_score is None:
- return None
- return self.score - self.prev_score
-
-
@dataclass
class Audit:
"""Everything the report needs; ``summary`` holds the bucket maps."""
@@ -82,20 +66,10 @@ class Audit:
def n_runs(self) -> int:
return len(self.runs)
- @property
- def n_passed(self) -> int:
- return sum(1 for r in self.runs if r.passed)
-
@property
def usd_per_run(self) -> float:
return self.total_usd / self.n_runs if self.n_runs else 0.0
- @property
- def usd_per_passing_artifact(self) -> float:
- """Total spend (including the runs that failed) divided by the artifacts
- that actually passed — the only cost-per-unit that matters for a dataset."""
- return self.total_usd / self.n_passed if self.n_passed else float("inf")
-
@property
def wall_s(self) -> float:
return sum(r.wall_s for r in self.runs)
@@ -120,47 +94,20 @@ class Audit:
return calls / len(rounds) if rounds else 0.0
-def _round_costs(led: RunLedger) -> list[RoundCost]:
- out: list[RoundCost] = []
- best = None
- for idx, score in enumerate(led.round_scores):
- usd = sum(r.cost_usd for r in led.rows if r.round == idx)
- promoted = score is not None and (best is None or score > best)
- rc = RoundCost(run=led.run, index=idx, usd=usd, score=score, prev_score=best, promoted=promoted)
- if promoted:
- best = score
- out.append(rc)
- return out
-
-
-def _waste(led: RunLedger, rounds: Sequence[RoundCost]) -> list[WasteItem]:
+def _waste(led: RunLedger) -> list[WasteItem]:
items: list[WasteItem] = []
- for rc in rounds:
- if rc.index == 0:
- continue
- if rc.delta is not None and rc.delta < 0:
- items.append(WasteItem("regression", led.run, rc.usd, round=rc.index,
- detail=f"r{rc.index:02d} scored {rc.score:.3f} vs {rc.prev_score:.3f} "
- f"({rc.delta:+.3f}) — the round was paid for and thrown away"))
- elif rc.delta is not None and abs(rc.delta) < 0.005:
- items.append(WasteItem("zero_delta_round", led.run, rc.usd, round=rc.index,
- detail=f"r{rc.index:02d} moved the score by {rc.delta:+.3f}"))
- # judge verdicts on rounds that never became best
for row in led.rows:
- if row.stage is not Stage.JUDGE and row.role is not Role.JUDGE:
- continue
- rc = next((r for r in rounds if r.index == row.round), None)
- if rc is not None and not rc.promoted and rc.index > 0:
- items.append(WasteItem("unpromoted_judge", led.run, row.cost_usd, round=row.round,
- detail=f"judge verdict for r{row.round:02d}, which never became best"))
- if row.outcome == "degraded":
+ if row.outcome == "degraded" and (row.stage is Stage.JUDGE or row.role is Role.JUDGE):
items.append(WasteItem("degraded_judge", led.run, row.cost_usd, round=row.round,
detail="degraded verdict (glitch, not a score) — paid for, not usable"))
- # repair loops that never produced a passing run
- repair = [r for r in led.rows if r.stage is Stage.REPAIR]
- if repair and not led.passed:
+ # repair loops that never got their round to build
+ unbuilt = {i for i, ok in enumerate(led.round_built) if ok is False}
+ repair = [r for r in led.rows if r.stage is Stage.REPAIR and r.round in unbuilt]
+ if repair:
items.append(WasteItem("repair_no_converge", led.run, sum(r.cost_usd for r in repair),
- detail=f"{len(repair)} repair call(s) in a run that ended '{led.status}'"))
+ detail=f"{len(repair)} repair call(s) in round(s) "
+ f"{', '.join(f'r{i:02d}' for i in sorted({r.round for r in repair}))} "
+ f"that still did not build"))
# best-of-N candidates that lost (their whole sub-workspace is thrown away). Two label
# forms: a live ledger row is ``baseline_c<k>`` (the candidate clone's label), a
# reconstructed one ``c<k>:baseline``. Generator sessions only — the quick-judge rows
@@ -174,8 +121,7 @@ def _waste(led: RunLedger, rounds: Sequence[RoundCost]) -> list[WasteItem]:
detail=f"best-of-N: {', '.join(names)} lost to {won or '(unknown)'}"))
# anything spent on a round that finished after the budget was blown
if led.stop_reason in ("budget",) or led.status == "budget":
- last = rounds[-1] if rounds else None
- extra = [r for r in led.rows if r.round is not None and last is not None and r.round > last.index]
+ extra = [r for r in led.rows if r.round is not None and led.n_rounds and r.round > led.n_rounds - 1]
if extra:
items.append(WasteItem("post_budget", led.run, sum(r.cost_usd for r in extra),
detail=f"round r{extra[0].round:02d} completed after the budget was already gone"))
@@ -202,10 +148,9 @@ def audit_runs(paths: Iterable[str | Path], *, recheck: bool = False) -> Audit:
row.track = led.track or "?"
row.language = led.language or "?"
row.status = led.status or "?"
- rounds = _round_costs(led)
audit.runs.append(led)
audit.rows += led.rows
- audit.waste += _waste(led, rounds)
+ audit.waste += _waste(led)
audit.summary = summarise(audit.rows, dimensions=AUDIT_DIMENSIONS)
audit.runs.sort(key=lambda r: -r.ledger_usd)
audit.waste.sort(key=lambda w: -w.usd)
diff --git a/harness/codeverse3d/addons/costreport/report.py b/harness/codeverse3d/addons/costreport/report.py
index ae318bfe..3e9c9460 100644
--- a/harness/codeverse3d/addons/costreport/report.py
+++ b/harness/codeverse3d/addons/costreport/report.py
@@ -1,12 +1,15 @@
"""Render an :class:`~codeverse3d.addons.costreport.audit.Audit` as markdown or a console table.
-Pure formatting: every number comes from the audit, nothing is recomputed here.
+Pure formatting: every number comes from the audit, nothing is recomputed here — the
+per-run baseline / picked scores come from ``addons/select`` (a one-shot compare cell,
+which has no record.json, reports its one score).
"""
from __future__ import annotations
from collections.abc import Sequence
+from codeverse3d.addons import select
from codeverse3d.addons.costreport.audit import (
Audit,
cached_input_share,
@@ -14,6 +17,7 @@ from codeverse3d.addons.costreport.audit import (
stage_latency,
uncached_if_no_cache,
)
+from codeverse3d.cost.reconstruct import RunLedger
from codeverse3d.cost.routing import pro_break_even, routing_table
from codeverse3d.cost.types import CostBucket
@@ -78,14 +82,27 @@ def dimension_table(audit: Audit, dim: str, *, limit: int | None = None, order:
return table(BUCKET_HEADERS, bucket_rows(buckets, audit.total_usd))
+def _scores(r: RunLedger) -> tuple[float | None, float | None]:
+ """``(baseline, picked)``: round 0's score and the round ``addons/select`` picks."""
+ if (r.path / "record.json").is_file():
+ try:
+ s = select.summarise(r.path)
+ return s.baseline_score, s.picked_score
+ except Exception: # noqa: BLE001 - an unreadable record still gets its cost row
+ return None, None
+ score = r.round_scores[0] if r.round_scores else None # a one-shot cell: one round
+ return score, score
+
+
def runs_table(audit: Audit, *, limit: int = 20) -> str:
rows = []
for r in audit.runs[:limit]:
+ baseline, picked = _scores(r)
rows.append([r.run, r.track or "-", _usd(r.ledger_usd), r.n_rounds,
- f"{r.baseline_score:.3f}" if r.baseline_score is not None else "-",
- f"{r.final_score:.3f}" if r.final_score is not None else "-",
- r.status, f"{r.wall_s / 60:.1f}", f"{r.model_s / 60:.1f}"])
- return table(("run", "track", "USD", "rounds", "baseline", "final", "status", "wall min", "model min"), rows)
+ f"{baseline:.3f}" if baseline is not None else "-",
+ f"{picked:.3f}" if picked is not None else "-",
+ r.stop_reason or r.status, f"{r.wall_s / 60:.1f}", f"{r.model_s / 60:.1f}"])
+ return table(("run", "track", "USD", "rounds", "baseline", "picked", "stop", "wall min", "model min"), rows)
def waste_table(audit: Audit) -> str:
@@ -100,8 +117,7 @@ def summary_lines(audit: Audit) -> list[str]:
cached, total_in, cache_usd = cached_input_share(audit)
no_cache = uncached_if_no_cache(audit)
return [
- f"- runs: **{audit.n_runs}** ({audit.n_passed} passed) — total **{_usd(audit.total_usd)}**, "
- f"{_usd(audit.usd_per_run)} per run, **{_usd(audit.usd_per_passing_artifact)} per passing artifact**",
+ f"- runs: **{audit.n_runs}** — total **{_usd(audit.total_usd)}**, {_usd(audit.usd_per_run)} per run",
f"- tokens: {_tok(total_in)} input of which **{_share(cached, total_in):.0f}% cached** "
f"({_tok(audit.summary.total.output_tokens)} output, {_tok(audit.summary.total.thoughts_tokens)} thoughts)",
f"- prompt caching already saves **{_usd(no_cache - audit.total_usd)}** "
diff --git a/harness/codeverse3d/addons/dataset/captions.py b/harness/codeverse3d/addons/dataset/captions.py
index 5ab43276..d771fb3b 100644
--- a/harness/codeverse3d/addons/dataset/captions.py
+++ b/harness/codeverse3d/addons/dataset/captions.py
@@ -24,13 +24,12 @@ from pathlib import Path
from pydantic import BaseModel, Field
-from codeverse3d.addons.dataset.sample import code_files_for_round
+from codeverse3d.addons.dataset.sample import code_files_for_round, exported_round
from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart
from codeverse3d.contracts.common import ENTRY_FILE, Language, Track
from codeverse3d.contracts.run import RunRecord
from codeverse3d.prompts import prompt_hash
from codeverse3d.record import _git
-from codeverse3d.record.record import best_round_record
from codeverse3d.workspace import Workspace
MAX_CODE_CHARS = 12_000
@@ -99,7 +98,7 @@ def _system_prompt() -> str:
def _round_images(ws: Workspace, record: RunRecord) -> tuple[list[ImagePart], list[str]]:
- rnd = best_round_record(record)
+ rnd = exported_round(ws, record)
images: list[ImagePart] = []
used: list[str] = []
if rnd is None or rnd.renders is None:
@@ -122,7 +121,7 @@ def _round_images(ws: Workspace, record: RunRecord) -> tuple[list[ImagePart], li
def _code_excerpt(ws: Workspace, record: RunRecord) -> str:
- rnd = best_round_record(record)
+ rnd = exported_round(ws, record)
files, _src = code_files_for_round(ws, rnd)
entry = ENTRY_FILE[record.spec.language]
ordered = {k: files[k] for k in sorted(files, key=lambda k: (k != entry, k))}
@@ -158,7 +157,7 @@ def caption_sample(
ws: Workspace, record: RunRecord, model_id: str, *, model: object | None = None,
out_dir: Path | str | None = None, slug: str | None = None
) -> Captions:
- """Caption the best round of ``record``.
+ """Caption the round of ``record`` a sample is made of (``sample.exported_round``).
Default: store into ``record.extra["captions"]`` + ``<ws>/captions.json`` and
refresh what a caption changes: telemetry/ (``package_run``), ``record.json``, and
@@ -199,7 +198,7 @@ def caption_sample(
ChatMessage.user("Fix these problems and return the JSON again:\n- " + "\n- ".join(problems))]
if caps is None or problems:
raise CaptionError(f"{ws.root}: captions rejected after retry: {problems}")
- rnd = best_round_record(record)
+ rnd = exported_round(ws, record)
prov = CaptionProvenance(captioner=model_id, round_index=rnd.index if rnd else None,
images_used=used, code_chars=len(code),
prompt_hash=prompt_hash(system), cost_usd=cost)
diff --git a/harness/codeverse3d/addons/dataset/export.py b/harness/codeverse3d/addons/dataset/export.py
index 6530569b..c59d9fd1 100644
--- a/harness/codeverse3d/addons/dataset/export.py
+++ b/harness/codeverse3d/addons/dataset/export.py
@@ -41,7 +41,7 @@ from codeverse3d.contracts.common import ENTRY_FILE, Language
from codeverse3d.contracts.run import RunId, RunRecord
from codeverse3d.proc import sha256_file
from codeverse3d.record._git import CODE_ROOTS
-from codeverse3d.record.record import FoundRun, best_round_record, iter_runs
+from codeverse3d.record.record import FoundRun, iter_runs
from codeverse3d.workspace import Workspace
log = logging.getLogger(__name__)
@@ -118,7 +118,7 @@ def export_one(
one the key falls back to the directory basename (flat layouts only)."""
key = run_id.slug if run_id is not None else S.sample_key(ws)
dest = out_dir / S.sample_rel_dir(record, key)
- rnd = best_round_record(record)
+ rnd = S.exported_round(ws, record)
files, code_source = S.code_files_for_round(ws, rnd)
if not files:
raise S.SampleError("no code files in src/ (nothing to export)")
@@ -158,8 +158,8 @@ def export_samples(
) -> ExportReport:
"""Export every eligible run under ``runs_dir`` into ``out_dir`` and rebuild the index.
- The best round is chosen by ``record.best_round`` or the
- highest judged score. Runs whose best round never built are skipped unless
+ The exported round is the one ``addons/select`` picks (the highest judged score, or
+ what the run's ``selection.json`` handed over). Runs whose round never built are skipped unless
``include_unbuilt`` (failed runs are still useful for repair pairs, not as
dataset samples). Byte-identical duplicates (same raw ``code_sha256`` + prompt)
are marked ``duplicate_of`` and, with ``drop_duplicates``, left out of the index
@@ -198,17 +198,17 @@ def export_samples(
for ws, rec, rid in found:
rep.n_runs += 1
- rnd = best_round_record(rec)
+ rnd = S.exported_round(ws, rec)
j = S.effective_judgment(rnd) if rnd is not None else None
score = j.overall if j is not None else None
if rnd is None:
_skip(ws.root, rid, "no rounds")
continue
if not include_unbuilt and not (rnd.build is not None and rnd.build.ok):
- _skip(ws.root, rid, "best round did not build")
+ _skip(ws.root, rid, "the exported round did not build")
continue
- if only_passed and not (j is not None and j.passed):
- _skip(ws.root, rid, "not passed")
+ if only_passed and not (j is not None and j.passed): # the judge's verdict on THAT round
+ _skip(ws.root, rid, "the judge did not pass the exported round")
continue
if min_score is not None and (score is None or score < min_score):
_skip(ws.root, rid, f"score {score} < min_score {min_score}")
@@ -405,7 +405,7 @@ class ManifestFilters(BaseModel):
only_passed: bool = False
include_unbuilt: bool = False
drop_duplicates: bool = False
- best_round: bool = True
+ picked_round: bool = Field(default=True, description="each sample is the round addons/select picks")
captions_dir: str = ""
diff --git a/harness/codeverse3d/addons/dataset/index.py b/harness/codeverse3d/addons/dataset/index.py
index 70fde686..a9df97b0 100644
--- a/harness/codeverse3d/addons/dataset/index.py
+++ b/harness/codeverse3d/addons/dataset/index.py
@@ -1,7 +1,9 @@
"""Small SQLite index over run records (runs / rounds / usage) for dashboards.
``build_index(runs_dir, out_sqlite)`` rebuilds the database from scratch;
-query helpers return plain dicts so a dashboard / notebook needs no ORM.
+query helpers return plain dicts so a dashboard / notebook needs no ORM. A run's
+row describes the round ``addons/select`` picks (``picked_round``); a run has no
+pass/fail — the ``rounds`` table carries the judge's own verdict per round.
"""
from __future__ import annotations
@@ -10,25 +12,21 @@ import sqlite3
from pathlib import Path
from typing import Any
+from codeverse3d.addons import select
from codeverse3d.addons.dataset.quality import prompt_hash, quality_tier
from codeverse3d.addons.dataset.sample import gate_error_summary
from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord
-from codeverse3d.record.record import (
- best_round_index,
- best_round_record,
- effective_judgment,
- iter_runs,
-)
+from codeverse3d.record.record import effective_judgment, iter_runs
from codeverse3d.workspace import Workspace
_SCHEMA = """
CREATE TABLE runs (
slug TEXT PRIMARY KEY, workspace TEXT, track TEXT, language TEXT, prompt TEXT, prompt_hash TEXT,
- generator TEXT, planner TEXT, judge TEXT, status TEXT,
- baseline_score REAL, final_score REAL, best_round INTEGER, n_rounds INTEGER, passed INTEGER,
+ generator TEXT, planner TEXT, judge TEXT, status TEXT, -- status: why the run stopped
+ baseline_score REAL, picked_score REAL, picked_round INTEGER, n_rounds INTEGER,
cost_usd REAL, input_tokens INTEGER, output_tokens INTEGER,
started_at TEXT, finished_at TEXT, duration_s REAL, error TEXT, has_captions INTEGER,
- gate_errors INTEGER, quality_tier TEXT, best_commit TEXT,
+ gate_errors INTEGER, quality_tier TEXT, picked_commit TEXT,
rel TEXT, arm TEXT, cell TEXT -- run dir relative to the scan root + battery layout segments
);
CREATE TABLE rounds (
@@ -49,22 +47,21 @@ CREATE INDEX idx_rounds_slug ON rounds(slug);
def _run_row(ws: Workspace, rec: RunRecord, rid: RunId) -> tuple:
- best = best_round_record(rec)
- best_j = effective_judgment(best) if best is not None else None # degraded → unjudged
- passed = None if best_j is None else int(best_j.passed)
+ s = select.summarise(ws.root, record=rec)
+ picked = next((r for r in rec.rounds if r.index == s.picked_round), None)
+ j = effective_judgment(picked) if picked is not None else None # degraded → unjudged
dur = (rec.finished_at - rec.started_at).total_seconds() if rec.finished_at else None
- n_err = sum(gate_error_summary(best).values())
- score = best_j.overall if best_j is not None else None
- tier = quality_tier(passed=None if passed is None else bool(passed), gate_errors=n_err, score=score)
+ n_err = sum(gate_error_summary(picked).values())
+ tier = quality_tier(passed=j.passed if j else None, gate_errors=n_err, score=j.overall if j else None)
return (
rid.slug, str(ws.root), rec.spec.track.value, rec.spec.language.value, rec.spec.prompt,
prompt_hash(rec.spec.prompt), rec.spec.backends.generator, rec.spec.backends.planner,
- rec.spec.backends.judge, rec.status.value, rec.baseline_score, rec.final_score, best_round_index(rec),
- len(rec.rounds), passed, rec.total_usage.cost_usd, rec.total_usage.input_tokens,
+ rec.spec.backends.judge, s.stop_reason, s.baseline_score, s.picked_score, s.picked_round,
+ len(rec.rounds), rec.total_usage.cost_usd, rec.total_usage.input_tokens,
rec.total_usage.output_tokens, rec.started_at.isoformat(),
rec.finished_at.isoformat() if rec.finished_at else None, dur, rec.error,
int(bool((rec.extra.get("captions") or {}).get("detailed"))),
- n_err, tier, best.commit if best is not None else "",
+ n_err, tier, picked.commit if picked is not None else "",
rid.rel, rid.arm, rid.cell,
)
@@ -106,7 +103,7 @@ def build_index(runs_dir: Path | str, out_sqlite: Path | str) -> int:
for ws, rec, rid in iter_runs(runs_dir):
slug = rid.slug
try:
- con.execute(f"INSERT INTO runs VALUES ({','.join('?' * 29)})", _run_row(ws, rec, rid))
+ con.execute(f"INSERT INTO runs VALUES ({','.join('?' * 28)})", _run_row(ws, rec, rid))
except sqlite3.IntegrityError as e:
# never INSERT OR REPLACE: a silent overwrite is the bug this guards
raise sqlite3.IntegrityError(
@@ -140,12 +137,12 @@ def query(db: Path | str, sql: str, params: tuple = ()) -> list[dict[str, Any]]:
def summary(db: Path | str) -> list[dict[str, Any]]:
- """Per (track, language, generator): n, pass rate, mean baseline/final, cost."""
+ """Per (track, language, generator): n, mean baseline / picked score and their delta, cost."""
return query(
db,
- "SELECT track, language, generator, COUNT(*) AS n, AVG(passed) AS pass_rate, "
- "AVG(baseline_score) AS baseline_mean, AVG(final_score) AS final_mean, "
- "AVG(final_score - baseline_score) AS delta_mean, SUM(cost_usd) AS cost_usd, "
+ "SELECT track, language, generator, COUNT(*) AS n, "
+ "AVG(baseline_score) AS baseline_mean, AVG(picked_score) AS picked_mean, "
+ "AVG(picked_score - baseline_score) AS delta_mean, SUM(cost_usd) AS cost_usd, "
"AVG(n_rounds) AS rounds_mean, SUM(quality_tier = 'A') AS n_tier_a, SUM(quality_tier = 'B') AS n_tier_b "
"FROM runs GROUP BY track, language, generator ORDER BY track, language",
)
diff --git a/harness/codeverse3d/addons/dataset/pairs.py b/harness/codeverse3d/addons/dataset/pairs.py
index 39d3a9cc..296a22a6 100644
--- a/harness/codeverse3d/addons/dataset/pairs.py
+++ b/harness/codeverse3d/addons/dataset/pairs.py
@@ -18,8 +18,9 @@ Output is JSONL, one object per pair. All kinds share::
(``notes`` say "repair attempts: N (fixed)"): the ``rNN <kind>: generated``
commit is the rejected state, the round's final commit the chosen one; the
error message comes from the round's failing ``build.done`` events.
-* ``cross_backend`` the same (prompt, track, language) run under ≥ 2 generators;
- best vs each other candidate with Δ ≥ ``min_delta``.
+* ``cross_backend`` the same (prompt, track, language) run under ≥ 2 generators; each
+ run is its picked round (``addons/select``), the top one against each other candidate
+ with Δ ≥ ``min_delta``.
"""
from __future__ import annotations
@@ -29,11 +30,12 @@ from collections import defaultdict
from pathlib import Path
from typing import Any
+from codeverse3d.addons import select
from codeverse3d.addons.dataset.quality import prompt_hash
from codeverse3d.contracts.run import RoundRecord, RunRecord
from codeverse3d.proc import read_jsonl_lenient
from codeverse3d.record import _git
-from codeverse3d.record.record import best_round_record, effective_judgment, iter_runs
+from codeverse3d.record.record import effective_judgment, iter_runs
from codeverse3d.workspace import Workspace
__all__ = ["build_pairs", "preference_pairs", "repair_pairs", "in_round_repair_pairs",
@@ -215,10 +217,10 @@ def cross_backend_pairs(
continue
cands = []
for ws, rec, slug in runs:
- best = best_round_record(rec)
- if best is None or effective_judgment(best) is None: # unjudged or degraded verdict
- continue
- cands.append((ws, rec, slug, best))
+ picked = select.summarise(ws.root, record=rec).picked_round # None = no judged round
+ rnd = next((r for r in rec.rounds if r.index == picked), None)
+ if rnd is not None:
+ cands.append((ws, rec, slug, rnd))
if len(cands) < 2:
continue
cands.sort(key=lambda c: effective_judgment(c[3]).overall, reverse=True) # type: ignore[union-attr]
diff --git a/harness/codeverse3d/addons/dataset/sample.py b/harness/codeverse3d/addons/dataset/sample.py
index 8ed01088..06591d32 100644
--- a/harness/codeverse3d/addons/dataset/sample.py
+++ b/harness/codeverse3d/addons/dataset/sample.py
@@ -22,6 +22,7 @@ from typing import Any
from pydantic import BaseModel, Field
from codeverse3d import __version__
+from codeverse3d.addons import select
from codeverse3d.addons.dataset.quality import (
QualityTier,
code_fingerprint,
@@ -91,7 +92,7 @@ class SampleMeta(BaseModel):
judge_backend: str = ""
judge_rubric: str = ""
score: float | None = None
- passed: bool | None = None
+ passed: bool | None = Field(default=None, description="the judge's own verdict for the exported round")
baseline_score: float | None = None
acceptance_results: dict[str, bool] = Field(default_factory=dict)
acceptance: list[AcceptanceEntry] = Field(default_factory=list, description="plan checklist + verdicts")
@@ -102,7 +103,7 @@ class SampleMeta(BaseModel):
default=None, description="objective complexity index of the exported artifact (spatial/complexity.py)")
complexity_band: str = Field(default="", description="trivial | simple | moderate | complex | intricate")
rounds: int = 0
- best_round: int | None = None
+ round: int | None = Field(default=None, description="the exported round (addons/select's pick)")
rounds_summary: list[dict[str, Any]] = Field(default_factory=list, description="compact per-round digest")
stop_reason: str = ""
code_commit: str = ""
@@ -138,9 +139,20 @@ def sample_rel_dir(record: RunRecord, key: str) -> Path:
return Path(record.spec.track.value) / record.spec.language.value / key
+def exported_round(ws: Workspace, record: RunRecord) -> RoundRecord | None:
+ """The round a sample is made of: the one ``addons/select`` picks, else — a run with no
+ judged round — its last round that built (a score-less, tier-D sample), else its last."""
+ picked = select.summarise(ws.root, record=record).picked_round
+ rnd = next((r for r in record.rounds if r.index == picked), None)
+ if rnd is None:
+ built = [r for r in record.rounds if r.build is not None and r.build.ok]
+ rnd = (built or record.rounds or [None])[-1]
+ return rnd
+
+
def code_files_for_round(ws: Workspace, rnd: RoundRecord | None) -> tuple[dict[str, bytes], str]:
"""Raw code tree for a round → ``(files, source)`` where source is ``commit``,
- ``deliverable`` (the packaged snapshot of the best round) or ``working_tree``."""
+ ``deliverable`` (the packaged snapshot of the handed-over round) or ``working_tree``."""
if rnd is not None and rnd.commit and _git.commit_exists(ws, rnd.commit):
return _git.read_tree_at(ws, rnd.commit), "commit"
packaged = _deliverable_code(ws)
@@ -211,7 +223,7 @@ def copy_renders(ws: Workspace, rnd: RoundRecord | None, dest: Path) -> list[str
if sheet.is_file():
_cp(sheet, f"sheet{sheet.suffix or '.png'}")
if not any(o.startswith("renders/sheet") for o in out):
- packaged = ws.deliverable / "sheet.png" # new layout keeps the best sheet here
+ packaged = ws.deliverable / "sheet.png" # the handed-over round's sheet
_cp(packaged if packaged.is_file() else None, "sheet.png")
glb = deliverable_path(ws, "object.glb")
if glb is not None and glb.stat().st_size < MAX_GLB_BYTES:
@@ -320,7 +332,7 @@ def build_meta(
judge_rubric=j.rubric if j else "",
score=j.overall if j else None,
passed=j.passed if j else None,
- baseline_score=record.baseline_score,
+ baseline_score=select.summarise(ws.root, record=record).baseline_score,
acceptance_results=dict(j.acceptance_results) if j else {},
acceptance=acceptance_entries(record, rnd),
gate_errors=n_gate_errors,
@@ -329,7 +341,7 @@ def build_meta(
complexity=cx.get("index") if cx else None,
complexity_band=str(cx.get("band") or "") if cx else "",
rounds=len(record.rounds),
- best_round=rnd.index if rnd is not None else None,
+ round=rnd.index if rnd is not None else None,
rounds_summary=[round_summary(r) for r in record.rounds],
stop_reason=str(record.extra.get("stop_reason", "") or ""),
code_commit=rnd.commit if rnd is not None else "",
diff --git a/harness/codeverse3d/addons/gallery/cards.py b/harness/codeverse3d/addons/gallery/cards.py
index f3834a73..d64ab194 100644
--- a/harness/codeverse3d/addons/gallery/cards.py
+++ b/harness/codeverse3d/addons/gallery/cards.py
@@ -113,7 +113,7 @@ def render_card(entry: RunEntry, urls: UrlMaker, *, hidden: bool = False) -> str
name = entry.title or entry.slug
head = (f"<div class='titlerow'>{tier_tag(entry)}"
f"<a class='name' href='{esc(urls.detail(entry))}' title='{esc(entry.slug)}'>{esc(name)}</a>"
- f"<span class='score num' title='best score'>{fmt(entry.score)}</span></div>")
+ f"<span class='score num' title='the picked round\'s score'>{fmt(entry.score)}</span></div>")
prompt = (f"<p class='prompt' title='{esc(entry.prompt)}'>{esc(entry.prompt)}</p>"
if entry.prompt else "")
if entry.state != "ok":
@@ -127,19 +127,17 @@ def render_card(entry: RunEntry, urls: UrlMaker, *, hidden: bool = False) -> str
if entry.baseline_score is not None and entry.score is not None:
cls = "up" if entry.score >= entry.baseline_score else "down"
delta = f" <span class='{cls}'>{entry.score - entry.baseline_score:+.3f}</span>"
- best = f"r{entry.best_round}" if entry.best_round is not None else "r–"
+ picked = f"r{entry.picked_round}" if entry.picked_round is not None else "r–"
cx = complexity_chip(entry)
facts2 = (f"<span>{fmt(entry.baseline_score)} → <b>{fmt(entry.score)}</b>{delta}</span>"
- f"<span>{best}/{entry.rounds}</span>"
+ f"<span title='picked round / rounds run'>{picked}/{entry.rounds}</span>"
f"<span>${entry.cost_usd:.2f}</span>"
+ (f"<span>{entry.minutes:.0f} min</span>" if entry.minutes is not None else "")
+ cx)
gates = (f"<span class='tag pill-warn'>{entry.gate_errors} gate err</span>" if entry.gate_errors
else "<span class='tag pill-ok'>gates clean</span>")
- # the run status only earns a chip when it says something the verdict does not
- # ("passed" next to a green `passed` pill is noise; "plateau"/"budget"/"failed" is not)
- redundant = not entry.status or (entry.passed and entry.status == "passed")
- status = "" if redundant else f"<span class='tag'>{esc(entry.status)}</span>"
+ # why the run stopped (max_rounds / budget / agent_quota / …) — never a pass or a fail
+ status = f"<span class='tag' title='why the run stopped'>{esc(entry.status)}</span>" if entry.status else ""
caption = f"<p class='xs faint clamp1'>{esc(entry.caption)}</p>" if entry.caption else ""
err = f"<div class='err clamp2' title='{esc(entry.error)}'>{esc(entry.error[:300])}</div>" \
if entry.error else ""
@@ -157,7 +155,7 @@ def render_card(entry: RunEntry, urls: UrlMaker, *, hidden: bool = False) -> str
TABLE_HEAD = ("<tr><th class='pickcol'><span class='sr'>select</span></th><th class='thumbcol'></th>"
"<th class='runcol'>run</th><th>verdict</th><th class='track'>track</th>"
"<th class='lang'>lang</th><th class='backend'>backend</th><th class='tier'>tier</th>"
- "<th class='n base'>base</th><th class='n'>best</th><th class='n delta'>Δ</th>"
+ "<th class='n base'>base</th><th class='n'>picked</th><th class='n delta'>Δ</th>"
"<th class='n'>gates</th><th class='n'>$</th><th class='n min'>min</th>"
"<th class='rowlinks'>links</th></tr>")
diff --git a/harness/codeverse3d/addons/gallery/compare.py b/harness/codeverse3d/addons/gallery/compare.py
index 3234dbb9..8dd4f39c 100644
--- a/harness/codeverse3d/addons/gallery/compare.py
+++ b/harness/codeverse3d/addons/gallery/compare.py
@@ -68,7 +68,7 @@ def _spark(entry: RunEntry) -> str:
top = max(r.score or 0.0 for r in rows) or 1.0
bars = "".join(
f"<i style='height:{max(3, round(26 * (r.score or 0.0) / top))}px'"
- f"{' class=best' if r.index == entry.best_round else ''}"
+ f"{' class=best' if r.index == entry.picked_round else ''}"
f" title='r{r.index} {r.score:.3f}'></i>" for r in rows)
return f"<div class='spark'>{bars}</div>"
@@ -122,7 +122,7 @@ def render_compare(entries: list[RunEntry], urls: UrlMaker, *, note: str = "") -
_row("gate errors", entries, lambda e: str(e.gate_errors) + (
" <span class='xs faint'>" + esc(", ".join(f"{k}:{v}" for k, v in e.gate_summary.items()))
+ "</span>" if e.gate_summary else "")),
- _row("rounds", entries, lambda e: f"{e.rounds} · best r{e.best_round if e.best_round is not None else '–'}"),
+ _row("rounds", entries, lambda e: f"{e.rounds} · picked r{e.picked_round if e.picked_round is not None else '–'}"),
_row("cost", entries, lambda e: f"${e.cost_usd:.3f}"),
_row("wall clock", entries, lambda e: "—" if e.minutes is None else f"{e.minutes:.1f} min"),
_row("track", entries, lambda e: esc(e.track)),
@@ -165,7 +165,7 @@ def csv_safe(value: object) -> object:
CSV_COLUMNS = ("battery", "slug", "title", "verdict", "tier", "track", "language", "generator",
"judge", "score", "baseline_score", "delta", "gate_errors", "cost_usd", "minutes",
- "rounds", "best_round", "status", "state", "path", "prompt")
+ "rounds", "picked_round", "status", "state", "path", "prompt")
def export_csv(entries: list[RunEntry]) -> str:
@@ -182,6 +182,6 @@ def export_csv(entries: list[RunEntry]) -> str:
"" if e.baseline_score is None else f"{e.baseline_score:.4f}",
"" if delta is None else f"{delta:+.4f}", e.gate_errors, f"{e.cost_usd:.4f}",
"" if e.minutes is None else f"{e.minutes:.2f}", e.rounds,
- "" if e.best_round is None else e.best_round, e.status, e.state, e.path, e.prompt,
+ "" if e.picked_round is None else e.picked_round, e.status, e.state, e.path, e.prompt,
]])
return buf.getvalue()
diff --git a/harness/codeverse3d/addons/gallery/detail.py b/harness/codeverse3d/addons/gallery/detail.py
index 2c7fd93b..402ba90a 100644
--- a/harness/codeverse3d/addons/gallery/detail.py
+++ b/harness/codeverse3d/addons/gallery/detail.py
@@ -2,8 +2,8 @@
Everything the harness recorded about one run, in the order you actually ask
for it: what it produced, how each round went, what the judge said about the
-best round, what the deterministic measurement says, every render, where the
-money went, and the code itself. The record is re-read per request, so a run a
+picked round (``addons/select``), what the deterministic measurement says, every
+render, where the money went, and the code itself. The record is re-read per request, so a run a
bench is still writing shows its latest rounds without a restart.
"""
@@ -19,8 +19,8 @@ from codeverse3d.addons.gallery.model import RunEntry
from codeverse3d.addons.gallery.theme import esc, footer, page_shell, top_bar
from codeverse3d.addons.gallery.urls import UrlMaker
from codeverse3d.contracts.common import ENTRY_FILE
-from codeverse3d.contracts.run import RunRecord
-from codeverse3d.record.record import best_round_record, effective_judgment
+from codeverse3d.contracts.run import RoundRecord, RunRecord
+from codeverse3d.record.record import effective_judgment
from codeverse3d.workspace import Workspace
DETAIL_CSS = CODE_CSS + """
@@ -38,8 +38,8 @@ DETAIL_CSS = CODE_CSS + """
.linkrow{display:flex;flex-wrap:wrap;gap:6px 10px;margin-top:var(--s-3);font-size:var(--fs-sm);
overflow-wrap:anywhere}
.badges{display:flex;flex-wrap:wrap;gap:6px;margin-bottom:var(--s-3);align-items:center}
-.best{background:var(--accent-soft)}
-.best td:first-child{box-shadow:inset 3px 0 0 var(--accent)}
+.picked{background:var(--accent-soft)}
+.picked td:first-child{box-shadow:inset 3px 0 0 var(--accent)}
details.round{border-top:1px solid var(--line);padding:var(--s-2) 0}
details.round summary{cursor:pointer;font-size:var(--fs-sm)}
.jump{display:flex;flex-wrap:wrap;gap:6px;margin:var(--s-4) 0 0;position:sticky;top:52px;z-index:10;
@@ -73,6 +73,10 @@ def _panel(title: str, inner: str, *, anchor: str = "", extra_head: str = "") ->
# --------------------------------------------------------------------------- sections
+def _picked(entry: RunEntry, rec: RunRecord) -> RoundRecord | None:
+ return next((r for r in rec.rounds if r.index == entry.picked_round), None)
+
+
def _hero(entry: RunEntry, urls: UrlMaker, rec: RunRecord) -> str:
"""One big legible view + the facts. Every other angle is in ``renders`` below,
which is where a grid belongs — the top of the page answers "what is this?"."""
@@ -83,15 +87,14 @@ def _hero(entry: RunEntry, urls: UrlMaker, rec: RunRecord) -> str:
f"alt='{esc(label)} — {esc(entry.slug)}' loading='eager'>"
f"<figcaption>{esc(label)}</figcaption></a>") if image else \
"<div class='empty'>no render</div>"
- redundant = not entry.status or (entry.passed and entry.status == "passed")
badges = (f"{tier_tag(entry)}{verdict_tag(entry)}"
- + ("" if redundant else f"<span class='tag'>{esc(entry.status)}</span>")
+ + (f"<span class='tag' title='why the run stopped'>{esc(entry.status)}</span>" if entry.status else "")
+ f"<span class='tag'>{esc(entry.track)}</span><span class='tag'>{esc(entry.language)}</span>")
delta = ("—" if entry.score is None or entry.baseline_score is None
else format(entry.score - entry.baseline_score, "+.3f"))
facts = "".join([
_kv("score", f"<b>{fmt(entry.score)}</b> (baseline {fmt(entry.baseline_score)}, Δ {delta})"),
- _kv("rounds", f"{entry.rounds} · best r{entry.best_round if entry.best_round is not None else '–'}"),
+ _kv("rounds", f"{entry.rounds} · picked r{entry.picked_round if entry.picked_round is not None else '–'}"),
_kv("gate errors", str(entry.gate_errors) + (
" — " + ", ".join(f"{k}:{v}" for k, v in entry.gate_summary.items()) if entry.gate_summary else "")),
_kv("cost", f"${entry.cost_usd:.3f}" + (f" · {entry.minutes:.1f} min" if entry.minutes is not None else "")),
@@ -119,8 +122,8 @@ def _rounds_table(entry: RunEntry, urls: UrlMaker) -> str:
build = "—" if r.build_ok is None else ("ok" if r.build_ok else "failed")
gates = str(r.gate_errors) + (" (" + ", ".join(f"{k}:{v}" for k, v in r.gates.items()) + ")" if r.gates else "")
sheet = f"<a href='{esc(urls.file(entry, r.sheet))}'>all views</a>" if r.sheet else "—"
- best = " class='best'" if r.index == entry.best_round else ""
- rows.append(f"<tr{best}><td class='n'>{r.index}</td><td>{esc(r.kind)}</td>"
+ picked = " class='picked'" if r.index == entry.picked_round else ""
+ rows.append(f"<tr{picked}><td class='n'>{r.index}</td><td>{esc(r.kind)}</td>"
f"<td class='n'>{fmt(r.score)}</td><td>{verdict}</td><td class='n'>{esc(gates)}</td>"
f"<td>{build}</td><td class='n'>{r.cost_usd:.3f}</td><td class='n'>{r.duration_s:.0f}</td>"
f"<td><code class='xs'>{esc(r.commit)}</code></td><td>{sheet}</td></tr>")
@@ -130,11 +133,11 @@ def _rounds_table(entry: RunEntry, urls: UrlMaker) -> str:
f"<tbody>{body}</tbody></table></div>", anchor="rounds")
-def _judgment_panel(rec: RunRecord, best: int | None) -> str:
- rnd = best_round_record(rec)
+def _judgment_panel(entry: RunEntry, rec: RunRecord) -> str:
+ rnd = _picked(entry, rec)
j = effective_judgment(rnd) if rnd is not None else None
if j is None:
- return _panel("judge", "<p class='faint small'>the best round has no (non-degraded) verdict.</p>",
+ return _panel("judge", "<p class='faint small'>no round has a (non-degraded) verdict.</p>",
anchor="judge")
scores = "".join(_kv(k, f"{v:.3f}") for k, v in sorted(j.scores.items()))
issues = "".join(
@@ -151,7 +154,7 @@ def _judgment_panel(rec: RunRecord, best: int | None) -> str:
inner = (
f"<p class='small muted'>rubric <code>{esc(j.rubric)}</code> · {esc(j.judge_backend)} · "
f"n={j.n_samples} · std {j.score_std:.3f} · overall <b>{j.overall:.3f}</b> "
- f"({'passed' if j.passed else 'failed'})</p>"
+ f"(the judge: {'pass' if j.passed else 'fail'})</p>"
f"<p style='margin-top:var(--s-2)'>{esc(j.summary)}</p>"
f"<div class='kvs' style='margin-top:var(--s-3)'>{scores}</div>"
+ (f"<h3 style='margin-top:var(--s-4)'>strengths</h3><ul class='plain'>{strengths}</ul>" if strengths else "")
@@ -159,11 +162,11 @@ def _judgment_panel(rec: RunRecord, best: int | None) -> str:
+ f"<h3 style='margin-top:var(--s-4)'>improvement plan</h3><ul class='plain'>{plan}</ul>"
+ (f"<h3 style='margin-top:var(--s-4)'>acceptance</h3><div class='facts'>{accept}</div>" if accept else "")
)
- return _panel(f"judge — best round r{best}", inner, anchor="judge")
+ return _panel(f"judge — picked round r{rnd.index}", inner, anchor="judge")
-def _measurement_panel(rec: RunRecord) -> str:
- rnd = best_round_record(rec)
+def _measurement_panel(entry: RunEntry, rec: RunRecord) -> str:
+ rnd = _picked(entry, rec)
m = rnd.measurement if rnd is not None else None
if m is None:
m = next((r.measurement for r in reversed(rec.rounds) if r.measurement is not None), None)
@@ -225,14 +228,14 @@ def _complexity_panel(entry: RunEntry, rec: RunRecord) -> str:
trail = block.get("by_round")
if isinstance(trail, list) and len(trail) > 1:
rows.append(_kv("by round", " → ".join(f"{float(x):.2f}" for x in trail)))
- note = ("<p class='xs faint'>objective complexity of the delivered artifact "
+ note = ("<p class='xs faint'>objective complexity of the picked round's artifact "
"(codeverse3d/spatial/complexity.py) — difficulty, not quality; see eval/docs/COMPLEXITY.md</p>")
return _panel("complexity", f"<div class='kvs'>{''.join(rows)}</div>{note}", anchor="complexity")
def _renders_panel(entry: RunEntry, urls: UrlMaker, rec: RunRecord) -> str:
ws = Workspace(entry.path)
- rnd = best_round_record(rec)
+ rnd = _picked(entry, rec)
figs: list[tuple[str, str]] = []
if rnd is not None and rnd.renders is not None:
for v in rnd.renders.views:
@@ -351,7 +354,6 @@ document.addEventListener('keydown',function(e){
def render_detail(entry: RunEntry, urls: UrlMaker, ws: Workspace, rec: RunRecord, *,
prev: RunEntry | None = None, nxt: RunEntry | None = None) -> str:
"""Full detail page for a run whose record parsed."""
- best = entry.best_round
nav = "".join(f"<a href='#{a}'>{a}</a>" for a in
("rounds", "judge", "complexity", "measurement", "renders", "cost", "code"))
body = (
@@ -364,9 +366,9 @@ def render_detail(entry: RunEntry, urls: UrlMaker, ws: Workspace, rec: RunRecord
+ _hero(entry, urls, rec)
+ f"<nav class='jump' aria-label='sections'>{nav}</nav>"
+ _rounds_table(entry, urls)
- + _judgment_panel(rec, best)
+ + _judgment_panel(entry, rec)
+ _complexity_panel(entry, rec)
- + _measurement_panel(rec)
+ + _measurement_panel(entry, rec)
+ _renders_panel(entry, urls, rec)
+ _cost_panel(entry, ws, rec)
+ _code_panel(entry, urls, rec)
diff --git a/harness/codeverse3d/addons/gallery/index.py b/harness/codeverse3d/addons/gallery/index.py
index 302a7243..ddcda0e2 100644
--- a/harness/codeverse3d/addons/gallery/index.py
+++ b/harness/codeverse3d/addons/gallery/index.py
@@ -15,6 +15,7 @@ from __future__ import annotations
import time
from pathlib import Path
+from codeverse3d.addons import select
from codeverse3d.addons.dataset.quality import quality_tier
from codeverse3d.addons.dataset.sample import gate_error_summary, telemetry_digest
from codeverse3d.addons.gallery.model import (
@@ -30,8 +31,6 @@ from codeverse3d.proc import read_json_or_none
from codeverse3d.record.record import (
RecordError,
battery_label,
- best_round_index,
- best_round_record,
effective_judgment,
find_run_dirs,
is_run_dir,
@@ -81,11 +80,11 @@ def _round_sheet(ws: Workspace, rec: RunRecord, index: int) -> str:
return _first_file(ws.root, f"artifacts/renders/r{index:02d}/sheet.png")
-def best_sheet(ws: Workspace, rec: RunRecord) -> str:
- """Contact sheet of the best round, falling back to the packaged deliverable sheet."""
- best = best_round_index(rec)
- if best is not None:
- rel = _round_sheet(ws, rec, best)
+def picked_sheet(ws: Workspace, rec: RunRecord, picked: int | None) -> str:
+ """Contact sheet of the picked round, else the latest round that has one, else the
+ packaged deliverable sheet."""
+ if picked is not None:
+ rel = _round_sheet(ws, rec, picked)
if rel:
return rel
for rnd in reversed(rec.rounds):
@@ -102,13 +101,13 @@ HERO_PREFERENCE = ("front_right_high", "back_left_high", "front_right_low",
"front_right_34", "back_left_34", "low_front_left", "front")
-def hero_view(ws: Workspace, rec: RunRecord) -> tuple[str, str, int]:
- """``(rel, label, n_views)`` — the ONE image a card should show.
+def hero_view(ws: Workspace, rec: RunRecord, picked: int | None) -> tuple[str, str, int]:
+ """``(rel, label, n_views)`` — the ONE image a card should show (from the picked round).
A card that shows an 8-up contact sheet at 320 px shows eight unreadable
thumbnails; one 320 px hero view is legible. Costs a single ``is_file``
beyond what the record already told us, so the index stays cheap."""
- rnd = best_round_record(rec)
+ rnd = next((r for r in rec.rounds if r.index == picked), None)
if rnd is None or rnd.renders is None or not rnd.renders.views:
return "", "", 0
views = [v for v in rnd.renders.views if not v.name.startswith(("pose_", "articulation"))]
@@ -122,8 +121,8 @@ def hero_view(ws: Workspace, rec: RunRecord) -> tuple[str, str, int]:
return rel, humanize_view(chosen.name), len(views)
-def _articulation_sheet(ws: Workspace, best: int | None) -> str:
- order = [best] if best is not None else []
+def _articulation_sheet(ws: Workspace, picked: int | None) -> str:
+ order = [picked] if picked is not None else []
order += [int(p.name[1:]) for p in sorted((ws.artifacts / "renders").glob("r[0-9][0-9]"), reverse=True)
if p.is_dir()]
for idx in order:
@@ -133,7 +132,7 @@ def _articulation_sheet(ws: Workspace, best: int | None) -> str:
return ""
-def entry_links(ws: Workspace, rec: RunRecord | None, best: int | None) -> list[RunLink]:
+def entry_links(ws: Workspace, rec: RunRecord | None, picked: int | None) -> list[RunLink]:
"""The working links of a card: workspace, record, code, sheet, GLB, track extras."""
run = ws.root
links = [RunLink(label="workspace", rel="", kind="dir"),
@@ -143,7 +142,7 @@ def entry_links(ws: Workspace, rec: RunRecord | None, best: int | None) -> list[
links.append(RunLink(label=name, rel=name))
if (run / "src").is_dir():
links.append(RunLink(label="src/", rel="src", kind="code"))
- sheet = best_sheet(ws, rec) if rec is not None else ""
+ sheet = picked_sheet(ws, rec, picked) if rec is not None else ""
if sheet:
links.append(RunLink(label="sheet", rel=sheet))
glb = _first_file(run, "deliverable/object.glb", "artifacts/object.glb")
@@ -159,7 +158,7 @@ def entry_links(ws: Workspace, rec: RunRecord | None, best: int | None) -> list[
urdf = _first_file(run, "deliverable/robot.urdf", "artifacts/robot.urdf", "src/robot.urdf")
if urdf:
links.append(RunLink(label="robot.urdf", rel=urdf))
- art = _articulation_sheet(ws, best)
+ art = _articulation_sheet(ws, picked)
if art:
links.append(RunLink(label="articulation", rel=art))
gif = _first_file(run, "deliverable/preview.gif", "artifacts/preview.gif")
@@ -207,15 +206,17 @@ def _measurement_complexity(ws: Workspace) -> dict | None:
return block if isinstance(block, dict) else None
-def _complexity(ws: Workspace, rec: RunRecord) -> tuple[float | None, str, dict[str, float]]:
- """``(index, band, axes)`` of the delivered artifact: the record's own
- complexity block, else the best round's measurement, else the measurement
- file on disk. A run built before the complexity vector existed has none
- (``eval/bench/complexity_report.py`` recomputes those from the GLB)."""
- from codeverse3d.record.record import complexity_block
+def _complexity(ws: Workspace, rec: RunRecord, picked: int | None) -> tuple[float | None, str, dict[str, float]]:
+ """``(index, band, axes)`` of the picked round's artifact: its own measurement,
+ else the record's complexity block, else the measurement file on disk. A run built
+ before the complexity vector existed has none (``eval/bench/complexity_report.py``
+ recomputes those from the GLB)."""
+ from codeverse3d.record.record import round_complexity
+ rnd = next((r for r in rec.rounds if r.index == picked), None)
try:
- block = rec.extra.get("complexity") or complexity_block(rec) or _measurement_complexity(ws)
+ block = ((round_complexity(rnd) if rnd is not None else None) or rec.extra.get("complexity")
+ or _measurement_complexity(ws))
except Exception: # noqa: BLE001 - a card is never worth an exception
block = None
if not isinstance(block, dict) or block.get("index") is None:
@@ -228,33 +229,34 @@ def entry_from_record(battery: str, ws: Workspace, rec: RunRecord, *, slug: str
"""A complete card from a parsed record (never raises: every field degrades).
``slug`` is the RunId slug the scan minted; without one the directory basename is
- used (correct for flat layouts only — nested battery runs are all named ``run``)."""
- best = best_round_index(rec)
- rnd = best_round_record(rec)
+ used (correct for flat layouts only — nested battery runs are all named ``run``).
+ The card is the round ``addons/select`` picks: its score, sheet, hero and tier."""
+ summary = select.summarise(ws.root, record=rec)
+ picked = summary.picked_round
+ rnd = next((r for r in rec.rounds if r.index == picked), None)
j = effective_judgment(rnd) if rnd is not None else None
gates = gate_error_summary(rnd)
n_err = sum(gates.values())
minutes = ((rec.finished_at - rec.started_at).total_seconds() / 60.0) if rec.finished_at else None
digest = telemetry_digest(ws, rec)
caps = rec.extra.get("captions") or {}
- hero, hero_label, n_views = hero_view(ws, rec)
+ hero, hero_label, n_views = hero_view(ws, rec, picked)
plan = rec.plan
- cx_index, cx_band, cx_axes = _complexity(ws, rec)
+ cx_index, cx_band, cx_axes = _complexity(ws, rec, picked)
return RunEntry(
battery=battery, slug=slug or ws.root.name, path=str(ws.root), state="ok",
title=(getattr(plan, "object_name", "") or getattr(plan, "title", "") or "") if plan else "",
prompt=rec.spec.prompt, track=rec.spec.track.value, language=rec.spec.language.value,
generator=rec.spec.backends.generator, judge=rec.spec.backends.judge,
- status=rec.status.value, error=rec.error, caption=str(caps.get("instruction", "") or ""),
- score=j.overall if j else rec.final_score, baseline_score=rec.baseline_score,
- passed=j.passed if j else None,
+ status=summary.stop_reason, error=rec.error, caption=str(caps.get("instruction", "") or ""),
+ score=summary.picked_score, baseline_score=summary.baseline_score,
tier=quality_tier(passed=j.passed if j else None, gate_errors=n_err, score=j.overall if j else None),
gate_errors=n_err, gate_summary={k: v for k, v in gates.items() if v},
cost_usd=rec.total_usage.cost_usd, minutes=round(minutes, 1) if minutes is not None else None,
- rounds=len(rec.rounds), best_round=best,
+ rounds=len(rec.rounds), picked_round=picked,
complexity=cx_index, complexity_band=cx_band, complexity_axes=cx_axes,
- sheet=best_sheet(ws, rec), hero=hero, hero_label=hero_label, n_views=n_views,
- links=entry_links(ws, rec, best), round_rows=round_rows(ws, rec),
+ sheet=picked_sheet(ws, rec, picked), hero=hero, hero_label=hero_label, n_views=n_views,
+ links=entry_links(ws, rec, picked), round_rows=round_rows(ws, rec),
cost_by_stage={k: round(v, 4) for k, v in (digest.get("by_stage") or {}).items()},
models=dict(digest.get("models") or {}),
)
diff --git a/harness/codeverse3d/addons/gallery/model.py b/harness/codeverse3d/addons/gallery/model.py
index 6ad2ea89..5c968789 100644
--- a/harness/codeverse3d/addons/gallery/model.py
+++ b/harness/codeverse3d/addons/gallery/model.py
@@ -5,8 +5,10 @@ Two small vocabularies live here so the page renderers never invent wording:
* :func:`humanize_view` — the render pipeline's machine view names
(``front_right_34``, ``t=2.5s``, ``pose_door_hinge@upper``) become the words a
person would say ("Front Right ¾", "t = 2.5 s", "Pose · door_hinge@upper");
-* :data:`VERDICTS` — the four **disjoint** triage buckets every run falls into
- exactly once, so a status breakdown always sums to the number of runs shown.
+* :data:`VERDICTS` — the three **disjoint** triage buckets every run falls into
+ exactly once, so a status breakdown always sums to the number of runs shown. None of
+ them is a pass or a fail: a run has none since 2026-09-22 — its card shows the score of
+ the round ``addons/select`` picks.
"""
from __future__ import annotations
@@ -17,14 +19,13 @@ from typing import Literal
from pydantic import BaseModel, Field
-#: the four buckets, in the order they are always displayed
-VERDICTS = ("passed", "failed", "unjudged", "error")
+#: the three buckets, in the order they are always displayed
+VERDICTS = ("judged", "unjudged", "error")
#: bucket → (short label, css class, what it means)
VERDICT_META: dict[str, tuple[str, str, str]] = {
- "passed": ("passed", "v-pass", "judged and over the rubric threshold"),
- "failed": ("failed", "v-fail", "judged and under the rubric threshold"),
- "unjudged": ("unjudged", "v-none", "the run finished but has no verdict"),
+ "judged": ("judged", "v-judged", "a round was judged: the card shows the picked round's score"),
+ "unjudged": ("unjudged", "v-none", "the run finished but no round has a verdict"),
"error": ("error", "v-err", "no record.json, or a corrupt / half-written one"),
}
@@ -92,7 +93,7 @@ LinkKind = Literal["file", "dir", "code", "viewer"]
RunState = Literal["ok", "pending", "broken"]
#: filter names the index page and ``/api/runs`` both understand
-FILTER_KEYS = ("q", "track", "lang", "tier", "backend", "pass", "verdict", "battery")
+FILTER_KEYS = ("q", "track", "lang", "tier", "backend", "verdict", "battery")
class RunLink(BaseModel):
@@ -104,7 +105,8 @@ class RunLink(BaseModel):
class RoundRow(BaseModel):
- """One refine round, flattened for the detail page's rounds table."""
+ """One round, flattened for the detail page's rounds table (``passed`` is the judge's
+ own verdict for THAT round — a fact about the round, never about the run)."""
index: int
kind: str = ""
@@ -142,16 +144,15 @@ class RunEntry(BaseModel):
status: str = ""
caption: str = ""
- score: float | None = None
+ score: float | None = Field(default=None, description="the picked round's score (addons/select)")
baseline_score: float | None = None
- passed: bool | None = None
tier: str = "D"
gate_errors: int = 0
gate_summary: dict[str, int] = Field(default_factory=dict)
cost_usd: float = 0.0
minutes: float | None = None
rounds: int = 0
- best_round: int | None = None
+ picked_round: int | None = Field(default=None, description="the round addons/select hands over")
complexity: float | None = Field(
default=None, description="objective complexity index of the delivered artifact (spatial/complexity.py)")
@@ -159,10 +160,10 @@ class RunEntry(BaseModel):
complexity_axes: dict[str, float] = Field(
default_factory=dict, description="the measured axes behind the index, for the detail page")
- sheet: str = Field(default="", description="run-relative contact sheet (best round)")
+ sheet: str = Field(default="", description="run-relative contact sheet (picked round)")
hero: str = Field(default="", description="run-relative single hero view (the card's image)")
hero_label: str = Field(default="", description="what the hero view shows, humanised")
- n_views: int = Field(default=0, description="how many individual views the best round has")
+ n_views: int = Field(default=0, description="how many individual views the picked round has")
links: list[RunLink] = Field(default_factory=list)
round_rows: list[RoundRow] = Field(default_factory=list)
cost_by_stage: dict[str, float] = Field(default_factory=dict)
@@ -173,22 +174,16 @@ class RunEntry(BaseModel):
"""Stable id used in URLs and as the DOM key: ``<battery>/<slug>``."""
return f"{self.battery}/{self.slug}"
- @property
- def pass_state(self) -> str:
- return "pass" if self.passed else ("fail" if self.passed is False else "na")
-
@property
def verdict(self) -> str:
"""The triage bucket — exactly one of :data:`~codeverse3d.addons.gallery.model.VERDICTS`.
- The four are disjoint *and* exhaustive on purpose: a status breakdown built
+ The three are disjoint *and* exhaustive on purpose: a status breakdown built
from them always sums to the number of runs on screen. A run whose record
is missing or corrupt is an ``error``, never a silent "unjudged"."""
if self.state != "ok":
return "error"
- if self.passed is None:
- return "unjudged"
- return "passed" if self.passed else "failed"
+ return "unjudged" if self.score is None else "judged"
@property
def card_image(self) -> str:
@@ -204,7 +199,7 @@ class RunEntry(BaseModel):
return {
"key": self.key, "battery": self.battery, "slug": self.slug, "name": self.title or self.slug,
"track": self.track, "lang": self.language, "backend": self.generator, "tier": self.tier,
- "pass": self.pass_state, "verdict": self.verdict, "score": self.score, "path": self.path,
+ "verdict": self.verdict, "score": self.score, "path": self.path,
"cost": round(self.cost_usd, 6), "minutes": self.minutes, "state": self.state,
"complexity": self.complexity, "text": self.search_text(),
}
@@ -221,26 +216,23 @@ class RootSection(BaseModel):
class Summary(BaseModel):
"""The numbers in the summary strip — computed over whatever is *currently* selected.
- ``breakdown`` is the load-bearing one: four disjoint buckets that **sum to
- ``n``**, so the strip can state "89 runs = 40 passed · 40 failed · 4 unjudged
- · 5 error" instead of the old un-addable "89 runs / 50% (40/80) / not ok 5"."""
+ ``breakdown`` is the load-bearing one: disjoint buckets that **sum to ``n``**, so the
+ strip can state "89 runs = 80 judged · 4 unjudged · 5 error". There is no pass rate:
+ a run is not passed or failed (2026-09-22) — the scores say how the runs went."""
n: int = 0
n_judged: int = 0
- n_passed: int = 0
breakdown: dict[str, int] = Field(default_factory=dict,
description="verdict bucket → count; sums to n")
- pass_rate: float | None = None
mean_score: float | None = None
median_score: float | None = None
total_usd: float = 0.0
- usd_per_pass: float | None = None
minutes: float = 0.0
broken: int = 0
def verdict_breakdown(entries: list[RunEntry]) -> dict[str, int]:
- """``{bucket: count}`` over all four buckets (zeros included), summing to ``len(entries)``."""
+ """``{bucket: count}`` over every bucket (zeros included), summing to ``len(entries)``."""
counts = dict.fromkeys(VERDICTS, 0)
for e in entries:
counts[e.verdict] += 1
@@ -250,17 +242,13 @@ def verdict_breakdown(entries: list[RunEntry]) -> dict[str, int]:
def summarize(entries: list[RunEntry]) -> Summary:
"""Summary strip for ``entries`` (the same arithmetic the page's JS does)."""
scored = [e.score for e in entries if e.score is not None]
- judged = [e for e in entries if e.passed is not None]
- passed = [e for e in judged if e.passed]
total = sum(e.cost_usd for e in entries)
return Summary(
- n=len(entries), n_judged=len(judged), n_passed=len(passed),
+ n=len(entries), n_judged=len(scored),
breakdown=verdict_breakdown(entries),
- pass_rate=(len(passed) / len(judged)) if judged else None,
mean_score=round(statistics.fmean(scored), 4) if scored else None,
median_score=round(statistics.median(scored), 4) if scored else None,
total_usd=round(total, 4),
- usd_per_pass=round(total / len(passed), 4) if passed else None,
minutes=round(sum(e.minutes or 0.0 for e in entries), 1),
broken=sum(1 for e in entries if e.state != "ok"),
)
@@ -307,10 +295,7 @@ def match(entry: RunEntry, flt: dict[str, str]) -> bool:
if want and want != value:
return False
want_verdict = (flt.get("verdict") or "").strip()
- if want_verdict and want_verdict != entry.verdict:
- return False
- want_pass = (flt.get("pass") or "").strip()
- return not (want_pass and want_pass != entry.pass_state)
+ return not (want_verdict and want_verdict != entry.verdict)
def sort_entries(entries: list[RunEntry], key: str) -> list[RunEntry]:
diff --git a/harness/codeverse3d/addons/gallery/page.py b/harness/codeverse3d/addons/gallery/page.py
index 23b8215a..f29665c3 100644
--- a/harness/codeverse3d/addons/gallery/page.py
+++ b/harness/codeverse3d/addons/gallery/page.py
@@ -6,10 +6,10 @@ initial filter is applied **server-side** (so ``curl '/?track=graphics'`` and a
no-JS browser both see the right runs and the right summary) and then re-applied
client-side on every keystroke without a reload.
-The summary is a *breakdown*, not a scoreboard: four disjoint buckets that add up
-to the number of runs on screen (``89 shown = 40 passed + 40 failed + 4 unjudged
-+ 5 error``). "pass 50% (40/80), not ok 5" against "89 runs" is arithmetic the
-reader has to do — and cannot.
+The summary is a *breakdown*, not a scoreboard: disjoint buckets that add up to the
+number of runs on screen (``89 shown = 80 judged + 4 unjudged + 5 error``), next to the
+mean / median picked-round score. There is no pass rate: a run is not passed or failed
+(2026-09-22).
"""
from __future__ import annotations
@@ -54,13 +54,9 @@ def _summary_strip(entries: list[RunEntry], total: int) -> str:
f"<span class='seg {VERDICT_META[b][1]}' id='vs-{b}' style='flex:{s.breakdown[b]}'"
f" title='{s.breakdown[b]} {b}'></span>" for b in VERDICTS)
chips = "".join(_verdict_chip(b, s.breakdown[b], total=s.n) for b in VERDICTS)
- per_pass = "—" if s.usd_per_pass is None else f"${s.usd_per_pass:.2f}"
- nums = (f"<span title='mean / median best score'>score <b class='num' id='s-score'>"
+ nums = (f"<span title='mean / median score of the picked rounds'>score <b class='num' id='s-score'>"
f"{fmt(s.mean_score)} / {fmt(s.median_score)}</b></span>"
f"<span>spend <b class='num' id='s-cost'>${s.total_usd:.2f}</b></span>"
- f"<span title='total spend divided by the number of passing runs'>per pass "
- f"<b class='num' id='s-perpass'>"
- f"{per_pass}</b></span>"
f"<span>wall clock <b class='num' id='s-time'>"
f"{(f'{s.minutes / 60:.1f} h' if s.minutes >= 90 else f'{round(s.minutes)} min')}</b></span>")
shown = f"{s.n}" if s.n == total else f"{s.n} <span class='faint'>of {total}</span>"
diff --git a/harness/codeverse3d/addons/gallery/theme.py b/harness/codeverse3d/addons/gallery/theme.py
index 2436c4da..8c8490ce 100644
--- a/harness/codeverse3d/addons/gallery/theme.py
+++ b/harness/codeverse3d/addons/gallery/theme.py
@@ -120,8 +120,9 @@ section.battery > h2{display:flex;gap:var(--s-2) var(--s-3);align-items:baseline
.tier{font-weight:750;color:var(--accent-fg);min-width:20px;justify-content:center}
.tier.A{background:var(--tier-a)} .tier.B{background:var(--tier-b)}
.tier.C{background:var(--tier-c)} .tier.D{background:var(--tier-d)}
-.pill-pass,.v-pass.tag,.pill-ok{color:var(--ok);background:var(--ok-soft)}
-.pill-fail,.v-fail.tag{color:var(--bad);background:var(--bad-soft)}
+.pill-pass,.pill-ok{color:var(--ok);background:var(--ok-soft)}
+.pill-fail{color:var(--bad);background:var(--bad-soft)}
+.v-judged.tag{color:var(--accent);background:var(--accent-soft)}
.pill-warn,.v-err.tag{color:var(--warn);background:var(--warn-soft)}
.v-none.tag{color:var(--fg-3);background:var(--chip)}
.pill-accent{color:var(--accent);background:var(--accent-soft)}
@@ -171,7 +172,7 @@ footer.foot{max-width:var(--maxw);margin:0 auto;padding:var(--s-4);color:var(--f
"""
INDEX_CSS = """
-/* --- status rail: four disjoint buckets that add up to the runs on screen --- */
+/* --- status rail: disjoint buckets that add up to the runs on screen ------- */
.summary{display:flex;flex-wrap:wrap;align-items:center;gap:var(--s-2) var(--s-3);
margin:var(--s-3) 0 var(--s-2);padding:8px var(--s-3);background:var(--surface);
border:1px solid var(--line);border-radius:var(--r-2)}
@@ -186,14 +187,13 @@ INDEX_CSS = """
.vchip .pct{font-size:var(--fs-xs);opacity:.7}
.vchip:hover{filter:brightness(.97)}
:root[data-theme="dark"] .vchip:hover{filter:brightness(1.2)}
-.vchip.v-pass{color:var(--ok);background:var(--ok-soft)}
-.vchip.v-fail{color:var(--bad);background:var(--bad-soft)}
+.vchip.v-judged{color:var(--accent);background:var(--accent-soft)}
.vchip.v-err{color:var(--warn);background:var(--warn-soft)}
.vchip[aria-pressed="true"]{outline:2px solid currentColor;outline-offset:1px}
.vbar{display:flex;height:8px;flex:1 1 160px;min-width:120px;border-radius:999px;overflow:hidden;
background:var(--sunken)}
.vbar .seg{display:block;min-width:0;transition:flex-grow .2s}
-.vbar .seg.v-pass{background:var(--ok)} .vbar .seg.v-fail{background:var(--bad)}
+.vbar .seg.v-judged{background:var(--accent)}
.vbar .seg.v-none{background:var(--line-strong)} .vbar .seg.v-err{background:var(--warn)}
.sumnums{display:flex;flex-wrap:wrap;gap:2px var(--s-3);font-size:var(--fs-sm);color:var(--fg-3);
margin-left:auto}
@@ -222,11 +222,10 @@ details.filters[open] > summary{border-bottom:1px solid var(--line)}
.card{position:relative;background:var(--surface);border:1px solid var(--line);
border-left:6px solid var(--line-strong);border-radius:var(--r-2);overflow:hidden;
display:flex;flex-direction:column;box-shadow:var(--shadow)}
-.card.v-pass{border-left-color:var(--ok)}
-.card.v-fail{border-left-color:var(--bad)}
+.card.v-judged{border-left-color:var(--accent)}
.card.v-err{border-left-color:var(--warn)}
.card.v-none{border-left-color:var(--line-strong)}
-.card.v-fail,.card.v-err{background:color-mix(in srgb,var(--surface) 94%,var(--bad-soft))}
+.card.v-err{background:color-mix(in srgb,var(--surface) 94%,var(--bad-soft))}
.card .shot{position:relative;display:block;background-color:var(--shot-bg);
background-image:repeating-conic-gradient(var(--shot-check) 0% 25%,transparent 0% 50%);
background-size:22px 22px;aspect-ratio:4/3;overflow:hidden;
@@ -295,8 +294,7 @@ td.pickcol .pick{opacity:1;position:static;background:none;padding:0}
td.thumbcol,th.thumbcol{width:66px;padding:3px 6px}
img.rowthumb{width:52px;height:40px;object-fit:contain;background:var(--shot-bg);border-radius:var(--r-1);
display:block;box-shadow:inset 0 0 0 1px var(--shot-ring)}
-tr.v-fail td:first-child{box-shadow:inset 3px 0 0 var(--bad)}
-tr.v-pass td:first-child{box-shadow:inset 3px 0 0 var(--ok)}
+tr.v-judged td:first-child{box-shadow:inset 3px 0 0 var(--accent)}
tr.v-err td:first-child{box-shadow:inset 3px 0 0 var(--warn)}
tr.picked{background:var(--accent-soft)}
td.rowlinks a{margin-right:8px}
@@ -399,8 +397,8 @@ INDEX_JS = r"""
var node=document.getElementById('gallery-data');
if(!node) return;
var DATA=JSON.parse(node.textContent);
-var KEYS=['q','track','lang','tier','backend','pass','verdict','battery'];
-var BUCKETS=['passed','failed','unjudged','error'];
+var KEYS=['q','track','lang','tier','backend','verdict','battery'];
+var BUCKETS=['judged','unjudged','error'];
var els={};
KEYS.concat(['sort']).forEach(function(k){els[k]=document.getElementById('f-'+k);});
var F={};
@@ -416,7 +414,6 @@ function match(r){
if(F.lang && r.lang!==F.lang) return false;
if(F.tier && r.tier!==F.tier) return false;
if(F.backend && r.backend!==F.backend) return false;
- if(F.pass && r.pass!==F.pass) return false;
if(F.verdict && r.verdict!==F.verdict) return false;
if(F.battery && r.battery!==F.battery) return false;
return true;
@@ -433,16 +430,15 @@ function cmp(a,b){
function fmt(v,d){return (v===null||v===undefined)?'—':Number(v).toFixed(d);}
function set(id,txt){var e=document.getElementById(id); if(e) e.textContent=txt;}
-/* the strip's promise: the four buckets always add up to the runs on screen */
+/* the strip's promise: the buckets always add up to the runs on screen */
function summarize(rows){
- var scored=[],total=0,mins=0,passed=0;
- var b={passed:0,failed:0,unjudged:0,error:0};
+ var scored=[],total=0,mins=0;
+ var b={judged:0,unjudged:0,error:0};
rows.forEach(function(r){
if(typeof r.score==='number') scored.push(r.score);
total+=r.cost||0; mins+=r.minutes||0;
if(b[r.verdict]===undefined) b[r.verdict]=0;
b[r.verdict]++;
- if(r.verdict==='passed') passed++;
});
var mean=scored.length?scored.reduce(function(a,c){return a+c;},0)/scored.length:null;
var srt=scored.slice().sort(function(a,c){return a-c;});
@@ -459,7 +455,6 @@ function summarize(rows){
});
set('s-score', fmt(mean,3)+' / '+fmt(med,3));
set('s-cost', '$'+total.toFixed(2));
- set('s-perpass', passed?'$'+(total/passed).toFixed(2):'—');
set('s-time', mins>=90?(mins/60).toFixed(1)+' h':Math.round(mins)+' min');
}
diff --git a/harness/codeverse3d/cli/_common.py b/harness/codeverse3d/cli/_common.py
index 77f7fe46..c1ff5217 100644
--- a/harness/codeverse3d/cli/_common.py
+++ b/harness/codeverse3d/cli/_common.py
@@ -51,48 +51,53 @@ def fmt_usd(v: float) -> str:
return f"${v:.4f}"
-def rounds_table(record: RunRecord) -> Table:
- t = Table(title="rounds (* = best)", show_lines=False)
+def rounds_table(record: RunRecord, picked: int | None) -> Table:
+ """One row per round; "passed" is the judge's own verdict for THAT round."""
+ t = Table(title="rounds (* = picked)", show_lines=False)
for col in ("#", "kind", "build", "gate err", "score", "passed", "cost", "secs", "commit"):
t.add_column(col, justify="right" if col in ("#", "gate err", "score", "cost", "secs") else "left")
for r in record.rounds:
build = "-" if r.build is None else ("ok" if r.build.ok else "[red]FAIL[/red]")
j = r.judgment
passed = "-" if j is None else ("[green]yes[/green]" if j.passed else "no")
- best = "*" if record.best_round is not None and r.index == record.best_round else ""
- t.add_row(f"{r.index}{best}", r.kind, build, str(sum(len(g.errors) for g in r.gates)), fmt_score(r.score),
+ mark = "*" if r.index == picked else ""
+ t.add_row(f"{r.index}{mark}", r.kind, build, str(sum(len(g.errors) for g in r.gates)), fmt_score(r.score),
passed, fmt_usd(r.usage.cost_usd), f"{r.duration_s:.0f}", r.commit[:8])
return t
def print_record_summary(record: RunRecord, ws_root: Path | None = None) -> None:
+ """The run in one panel — its stop reason, the baseline and the round ``addons/select``
+ picks (never a pass or a fail: a run has none) — and a row per round."""
+ from codeverse3d.addons import select
+
spec = record.spec
- status_col = {"passed": "green", "failed": "red", "budget": "yellow", "plateau": "yellow"}.get(record.status.value, "cyan")
+ root = Path(ws_root or record.workspace)
+ s = select.summarise(root, record=record)
+ status_col = {"failed": "red", "budget": "yellow", "agent_quota": "yellow"}.get(record.status.value, "cyan")
lines = [
f"[bold]{spec.prompt}[/bold]",
f"track={spec.track.value} language={spec.language.value} generator={spec.backends.generator}",
- f"status=[{status_col}]{record.status.value}[/{status_col}] baseline={fmt_score(record.baseline_score)} "
- f"best={fmt_score(record.final_score)} (round {record.best_round}) rounds={len(record.rounds)} "
+ f"stop=[{status_col}]{s.stop_reason}[/{status_col}] baseline={fmt_score(s.baseline_score)} "
+ f"picked={fmt_score(s.picked_score)} (round {s.picked_round}, by {s.method}) rounds={s.rounds} "
f"cost={fmt_usd(record.total_usage.cost_usd)}",
]
if record.error:
lines.append(f"[red]error: {record.error}[/red]")
if ws_root is not None:
lines.append(f"workspace: {ws_root}")
- from codeverse3d.record.record import best_round_record
-
- best = best_round_record(record)
- if best is not None and best.renders is not None and best.renders.contact_sheet:
+ picked = next((r for r in record.rounds if r.index == s.picked_round), None)
+ if picked is not None and picked.renders is not None and picked.renders.contact_sheet:
# rebase, never print the stored string: record.json holds the ABSOLUTE path
# of the host that produced the run, so a moved/archived run printed a sheet
# that does not exist while the real one sat under this root.
- lines.append(f"sheet: {Workspace(ws_root).rebase(best.renders.contact_sheet)}")
- glb = ws_root / "artifacts" / "object.glb"
- if glb.is_file():
- lines.append(f"glb: {glb}")
+ lines.append(f"sheet: {Workspace(ws_root).rebase(picked.renders.contact_sheet)}")
+ handed = ws_root / "deliverable"
+ if (handed / "manifest.json").is_file():
+ lines.append(f"deliverable: {handed}")
console.print(Panel("\n".join(lines), title="run", expand=False))
if record.rounds:
- console.print(rounds_table(record))
+ console.print(rounds_table(record, s.picked_round))
def kv_table(title: str, rows: dict[str, Any]) -> Table:
diff --git a/harness/codeverse3d/cli/cost_cmd.py b/harness/codeverse3d/cli/cost_cmd.py
index 1aaee5d5..ffc9c039 100644
--- a/harness/codeverse3d/cli/cost_cmd.py
+++ b/harness/codeverse3d/cli/cost_cmd.py
@@ -52,7 +52,7 @@ def cost(
recheck: Annotated[bool, typer.Option("--recheck", help="re-price every call with today's table")] = False,
md: Annotated[Path | None, typer.Option("--md", help="also write the full markdown report here")] = None,
) -> None:
- """Cost breakdown: per stage, per role, per model, waste, $ per passing artifact.
+ """Cost breakdown: per stage, per role, per model, waste, $ per run.
``3dcode cost <slug>`` reports one run from its own ledger; ``--runs-dir <dir>``
aggregates every run under a root (a bench battery)."""
diff --git a/harness/codeverse3d/cli/flywheel_cmd.py b/harness/codeverse3d/cli/flywheel_cmd.py
index 8f68c3cf..3e1e8472 100644
--- a/harness/codeverse3d/cli/flywheel_cmd.py
+++ b/harness/codeverse3d/cli/flywheel_cmd.py
@@ -153,10 +153,10 @@ def index_cmd(
from rich.table import Table
t = Table(title="summary")
- for col in ("track", "language", "generator", "n", "pass", "baseline", "final", "Δ", "cost"):
+ for col in ("track", "language", "generator", "n", "baseline", "picked", "Δ", "cost"):
t.add_column(col)
for r in summary(out_sqlite):
t.add_row(r["track"], r["language"], r["generator"], str(r["n"]),
- f"{(r['pass_rate'] or 0):.2f}", f"{(r['baseline_mean'] or 0):.3f}", f"{(r['final_mean'] or 0):.3f}",
+ f"{(r['baseline_mean'] or 0):.3f}", f"{(r['picked_mean'] or 0):.3f}",
f"{(r['delta_mean'] or 0):+.3f}", f"${(r['cost_usd'] or 0):.2f}")
console.print(t)
diff --git a/harness/codeverse3d/cli/inspect_cmd.py b/harness/codeverse3d/cli/inspect_cmd.py
index 2ea9e31e..32ebf7f4 100644
--- a/harness/codeverse3d/cli/inspect_cmd.py
+++ b/harness/codeverse3d/cli/inspect_cmd.py
@@ -2,9 +2,9 @@
Owns the three read-mostly commands on a finished (or in-flight) workspace: the
``status`` summary (spec / run_state / record / candidates / recent events), the
-``render`` command with its round-label refusal (`_render_round_or_refuse`) and the
-graphics-track frame copy, and the ``judge`` re-judge that writes
-``artifacts/judge/rNN_cli.json``. Its sibling ``cli/main.py`` owns the typer app,
+``render`` command (the working tree, which ends at the run's last round) with its
+round-label refusal and the graphics-track frame copy, and the ``judge`` re-judge that
+writes ``artifacts/judge/rNN_cli.json`` (default: the round ``addons/select`` picks). Its sibling ``cli/main.py`` owns the typer app,
``make`` / ``resume`` (spec building + track dispatch), ``mcp`` and the registration
of every command — including these three, which it registers with ``app.command``
like ``layout_cmd.py``'s ``show``.
@@ -192,16 +192,14 @@ def judge(
"""Re-judge a round with the SAME inputs as the in-run judge (renders + measurement +
gates + acceptance + plan digest + previous verdict + stored clay views); writes
artifacts/judge/rNN_cli.json."""
+ from codeverse3d.addons import select
from codeverse3d.cli import _judge as J
from codeverse3d.record.record import load_record
ws = C.open_workspace(slug, runs_dir)
rec = load_record(ws)
- idx = (
- round_index
- if round_index is not None
- else (rec.best_round if rec.best_round is not None else _latest_round(ws))
- )
+ picked = select.pick(ws.root, record=rec) if round_index is None else None
+ idx = round_index if round_index is not None else (picked if picked is not None else _latest_round(ws))
rnd = J.load_round(ws, rec, idx)
if rnd is None or rnd.renders is None or not rnd.renders.views:
raise C.CliError(f"round {idx} has no renders (rounds/r{idx:02d}.json / record.json)")
@@ -284,52 +282,24 @@ def _render_graphics(ws, spec: Spec, out_dir: Path) -> None:
def _render_round_or_refuse(ws, round_index: int | None) -> int:
"""Which round this render is labelled as — refusing when the label would lie.
- ``render`` renders the WORKING TREE, which sits at the last round the run wrote.
- ``--round`` only chose the output folder, so `3dcode render X --round 3` wrote
- r03-labelled images of round 4's code, and with no flag at all a run whose best round
- was not its last silently published its worst one.
-
- Measured 2026-08-25 on tsr_scn_neon_alley: judge by round 0.338 / 0.375 / 0.529 /
- 0.632 / 0.000 — round 4 rendered completely blank, all eight tiles empty. The harness
- correctly kept r3 and the deliverable is correct, but the tree was left at r4, so a
- plain `3dcode render` re-rendered eight blank frames and was very nearly shipped.
-
- So: no flag renders the tree only when the tree IS the best round; otherwise this
- refuses and points at ``deliverable/``, which already holds the best round's code,
- sheet and a manifest naming the commit. Rendering a round other than the tree's would
- need that round checked out, which this command does not do — hence a refusal rather
- than a mislabelled image.
+ ``render`` renders the WORKING TREE, which ends at the last round the run wrote
+ (2026-09-22: nothing restores an earlier round). ``--round`` only chose the output
+ folder, so `3dcode render X --round 3` wrote r03-labelled images of round 4's code.
+ Rendering another round would need it checked out, which this command does not do —
+ every round's own renders are already in ``artifacts/renders/rNN/``, and
+ ``3dcode pick <slug> --round N`` packages any round with its GLB.
"""
tree = _latest_round(ws)
- best = _best_round_of_record(ws)
- if round_index is not None:
- if round_index != tree:
- raise C.CliError(
- f"cannot render round {round_index}: `render` renders the working tree, which is at "
- f"round {tree}, and --round only labels the output folder. The best round's code, "
- f"renders and manifest are already packaged in {ws.deliverable} — read "
- f"{ws.deliverable / 'sheet.png'}, or `3dcode resume {ws.root.name}` to keep iterating.",
- code=2)
- return round_index
- if best is not None and best != tree:
+ if round_index is not None and round_index != tree:
raise C.CliError(
- f"refusing to render: this run's BEST round is r{best} but the working tree is at "
- f"r{tree}, so this would render the wrong round — and r{tree} may be why it was not "
- f"chosen. Read {ws.deliverable / 'sheet.png'} (the packaged best round), or pass "
- f"--round {tree} to render the tree anyway.",
+ f"cannot render round {round_index}: `render` renders the working tree, which is at "
+ f"round {tree}, and --round only labels the output folder. Round {round_index}'s renders "
+ f"are in {ws.renders_dir(round_index)}; `3dcode pick {ws.root.name} --round {round_index}` "
+ f"packages it.",
code=2)
return tree
-def _best_round_of_record(ws) -> int | None:
- """``record.best_round``, or None when there is no readable record yet."""
- rec = read_json_or_none(ws.record_path)
- if rec is None:
- return None
- best = rec.get("best_round")
- return best if isinstance(best, int) else None
-
-
def _latest_round(ws) -> int:
rdir = ws.artifacts / "renders"
idxs = (
diff --git a/harness/codeverse3d/cli/layout_cmd.py b/harness/codeverse3d/cli/layout_cmd.py
index f2b96f08..3671a0c3 100644
--- a/harness/codeverse3d/cli/layout_cmd.py
+++ b/harness/codeverse3d/cli/layout_cmd.py
@@ -80,13 +80,15 @@ def _legacy_deliverable_rows(ws: Workspace) -> dict[str, Any]:
def print_deliverable(ws: Workspace, record: RunRecord) -> None:
+ from codeverse3d.addons import select
from codeverse3d.record.deliverable import load_deliverable
_section("DELIVERABLE — what the run produced")
spec = record.spec
+ s = select.summarise(ws.root, record=record)
head = {"prompt": spec.prompt, "track": spec.track.value, "language": spec.language.value,
- "status": record.status.value, "best round": record.best_round,
- "score": fmt_score(record.final_score), "workspace": ws.root}
+ "stop": s.stop_reason, "picked round": f"{s.picked_round} (by {s.method})",
+ "score": fmt_score(s.picked_score), "workspace": ws.root}
d = load_deliverable(ws)
if d is not None:
head["code"] = f"{d.entry or 'deliverable/src/'} @ {d.commit[:12] or '-'} ({d.code_source})"
@@ -97,25 +99,27 @@ def print_deliverable(ws: Workspace, record: RunRecord) -> None:
for name, why in d.skipped.items():
warn(f"not packaged: {name} — {why}")
else:
- console.print(kv_table("artifacts (old layout — no deliverable/)",
+ console.print(kv_table("artifacts (no deliverable/ yet — `3dcode pick` packages a round)",
_legacy_deliverable_rows(ws)))
# --------------------------------------------------------------------------- (b) evidence
def print_evidence(ws: Workspace, record: RunRecord) -> None:
+ from codeverse3d.addons import select
from codeverse3d.addons.dataset.sample import gate_error_summary
- from codeverse3d.record.record import best_round_record, effective_judgment
+ from codeverse3d.record.record import effective_judgment
_section("QUALITY EVIDENCE — why we believe it")
- rnd = best_round_record(record)
+ s = select.summarise(ws.root, record=record)
+ rnd = next((r for r in record.rounds if r.index == s.picked_round), None)
j = effective_judgment(rnd) if rnd is not None else None
gates = gate_error_summary(rnd)
rows: dict[str, Any] = {
- "baseline → best": f"{fmt_score(record.baseline_score)} → {fmt_score(record.final_score)}",
+ "baseline → picked": f"{fmt_score(s.baseline_score)} → {fmt_score(s.picked_score)}",
"rubric": (j.rubric if j else "") or str(record.extra.get("rubric") or "-"),
- "passed": "-" if j is None else j.passed,
+ "judge on that round": "-" if j is None else ("pass" if j.passed else "fail"),
"gate errors": f"{sum(gates.values())} ({', '.join(f'{g}:{n}' for g, n in gates.items() if n) or 'none'})",
- "stop reason": str(record.extra.get("stop_reason") or "-"),
+ "stop reason": s.stop_reason or "-",
}
if j is not None:
acc = j.acceptance_results or {}
diff --git a/harness/codeverse3d/cli/main.py b/harness/codeverse3d/cli/main.py
index 04f14a62..a971a556 100644
--- a/harness/codeverse3d/cli/main.py
+++ b/harness/codeverse3d/cli/main.py
@@ -240,7 +240,7 @@ app.add_typer(
)
app.add_typer(bench_app, name="bench", help="Prompt batteries: run + report.")
app.add_typer(
- cost_app, name="cost", help="Cost audit: per stage/role/model, waste, $ per passing artifact."
+ cost_app, name="cost", help="Cost audit: per stage/role/model, waste, $ per run."
)
app.add_typer(doctor_app, name="doctor", help="Environment checks.")
app.add_typer(
diff --git a/harness/codeverse3d/cost/reconstruct.py b/harness/codeverse3d/cost/reconstruct.py
index 4cc3a6d6..e9d05949 100644
--- a/harness/codeverse3d/cost/reconstruct.py
+++ b/harness/codeverse3d/cost/reconstruct.py
@@ -89,11 +89,9 @@ class RunLedger:
generator: str = ""
status: str = ""
stop_reason: str = ""
- passed: bool = False
- baseline_score: float | None = None
- final_score: float | None = None
n_rounds: int = 0
round_scores: list[float | None] = field(default_factory=list)
+ round_built: list[bool | None] = field(default_factory=list) # per round: did its build (+repair) succeed
recorded_usd: float = 0.0
wall_s: float = 0.0 # time the run was actually working (budget elapsed / stage+round durations)
span_s: float = 0.0 # first to last event — includes time queued behind other runs
@@ -271,7 +269,6 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
track=str(spec.get("track") or ""), language=str(spec.get("language") or ""),
generator=str(backends.get("generator") or ""),
status=str(record.get("status") or ""), stop_reason=str(extra.get("stop_reason") or ""),
- baseline_score=record.get("baseline_score"), final_score=record.get("final_score"),
recorded_usd=float(total.cost_usd),
)
cands = _read_json(root / "rounds" / "candidates.json") or (extra.get("candidates") or {})
@@ -279,7 +276,6 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
led.selected_candidate = f"c{cands['selected']}"
rounds = record.get("rounds") or []
led.n_rounds = len(rounds)
- led.passed = bool(record.get("status") == "passed")
live_rows = _live_rows(root, run=run, recheck=recheck)
led.source = "live" if live_rows else "reconstructed"
@@ -303,6 +299,7 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
idx = int(r.get("index") or 0)
judgment = r.get("judgment") or {}
led.round_scores.append((judgment or {}).get("overall"))
+ led.round_built.append((r.get("build") or {}).get("ok"))
ju = _usage(judgment.get("usage"))
if ju.input_tokens or ju.cost_usd:
judged_rounds.add(idx)
@@ -396,8 +393,8 @@ def reconstruct_cell(cell_dir: str | Path, *, recheck: bool = False) -> RunLedge
run = RunId(battery="", rel=f"{root.parent.name}/{root.name}").slug # <prompt>__<arm>
led = RunLedger(run=run, path=root, track="static_object", language="blender",
generator=str(cell.get("arm") or root.name),
- status=str(cell.get("status") or ""), passed=bool(cell.get("passed")),
- final_score=cell.get("score"), n_rounds=1,
+ status=str(cell.get("status") or ""), round_scores=[cell.get("score")],
+ round_built=[cell.get("build_ok")], n_rounds=1,
wall_s=float(cell.get("wall_s") or 0.0), model_s=float(cell.get("gen_seconds") or 0.0))
live = _live_rows(root, run=run, recheck=recheck)
if live: # the cell wrote its own ledger while it ran: one priced row per real call
diff --git a/harness/codeverse3d/proc.py b/harness/codeverse3d/proc.py
index 17cfa796..8e29eedb 100644
--- a/harness/codeverse3d/proc.py
+++ b/harness/codeverse3d/proc.py
@@ -454,8 +454,7 @@ def read_json_or_none(path: Path | str, *, errors: str | None = None) -> dict[st
The one tolerant reader behind every "use it if it is there" side-car: it
replaced identical ``try: json.loads(read_text()) except (OSError, ValueError)``
copies in ``record/telemetry.read_json``, ``eval/bench/complexity_report._read_json``,
- ``gallery/index._measurement_complexity`` / ``_spec_fields``,
- ``cli/inspect_cmd._best_round_of_record``, ``runlock._holder``,
+ ``gallery/index._measurement_complexity`` / ``_spec_fields``, ``runlock._holder``,
``languages/threejs/__init__.py``'s plan loader, ``cost/reconstruct._read_json``,
``spatial/_render_common.read_json`` and ``cli/_judge.plan_summary_for`` — all
the same algorithm, differing only in whether they also checked ``isinstance(dict)``
diff --git a/harness/tests/core/test_run_layout.py b/harness/tests/core/test_run_layout.py
index 32a3bead..58257992 100644
--- a/harness/tests/core/test_run_layout.py
+++ b/harness/tests/core/test_run_layout.py
@@ -88,7 +88,7 @@ def _moved_run(tmp_path: Path):
"""Create a record whose absolute sheet path predates a workspace move."""
import shutil
- from codeverse3d.contracts.artifacts import RenderSet
+ from codeverse3d.contracts.artifacts import Judgment, RenderSet
from codeverse3d.contracts.common import Backends, Language, Track
from codeverse3d.contracts.run import RoundRecord, RunStatus
@@ -98,8 +98,9 @@ def _moved_run(tmp_path: Path):
sheet_a.write_bytes(b"PNG")
spec = Spec(id="run1", track=Track.STATIC_OBJECT, language=Language.BLENDER,
prompt="a wooden chair", backends=Backends(generator="gemini-cli:gemini-3.6-flash"))
- rec = RunRecord(spec=spec, workspace=str(a), status=RunStatus.MAX_ROUNDS, best_round=0,
+ rec = RunRecord(spec=spec, workspace=str(a), status=RunStatus.MAX_ROUNDS,
rounds=[RoundRecord(index=0, kind="generate",
+ judgment=Judgment(rubric="r", scores={}, overall=0.6, passed=False),
renders=RenderSet(views=[], contact_sheet=str(sheet_a)))])
b = tmp_path / "B" / "run1"
b.parent.mkdir(parents=True)
diff --git a/harness/tests/cost/test_audit_report.py b/harness/tests/cost/test_audit_report.py
index 454a5eaa..35cf4d80 100644
--- a/harness/tests/cost/test_audit_report.py
+++ b/harness/tests/cost/test_audit_report.py
@@ -19,8 +19,7 @@ from codeverse3d.addons.costreport.report import console, markdown
def test_audit_of_one_run(fake_run: Path):
audit = audit_runs([fake_run.parent])
- assert audit.n_runs == 1 and audit.n_passed == 1
- assert audit.usd_per_passing_artifact == pytest.approx(audit.total_usd)
+ assert audit.n_runs == 1 and audit.usd_per_run == pytest.approx(audit.total_usd)
assert audit.summary.dimension("track")["static_object"].cost_usd == pytest.approx(audit.total_usd)
assert audit.calls_per_round() > 0
cached, total, usd = cached_input_share(audit)
@@ -47,25 +46,31 @@ def test_a_run_that_booked_no_money_still_reports(tmp_path: Path):
assert "0%" in console(audit) and "# Cost audit" in markdown(audit)
-def test_waste_finds_a_regression(fake_run: Path, tmp_path: Path):
- """A refine round that scores below the best is money spent on a worse artifact."""
+def test_a_lower_scoring_round_is_not_waste_but_a_repair_that_never_built_is(fake_run: Path, tmp_path: Path):
+ """Every round is kept and any can be handed over (2026-09-22), so scoring below an
+ earlier round is no longer waste; repair money in a round that still did not build is."""
import shutil
ws = tmp_path / "regressed"
shutil.copytree(fake_run, ws)
record = json.loads((ws / "record.json").read_text())
- r0 = record["rounds"][0]
- r1 = json.loads(json.dumps(r0))
- r1["index"] = 1
- r1["kind"] = "refine"
+ r1 = json.loads(json.dumps(record["rounds"][0]))
+ r1.update(index=1, kind="refine")
r1["judgment"]["overall"] = 0.5 # worse than r00's 0.8
- record["rounds"].append(r1)
- record["total_usage"]["cost_usd"] *= 2
+ r2 = json.loads(json.dumps(r1))
+ r2.update(index=2, judgment=None, build={"ok": False, "language": "blender"})
+ record["rounds"] += [r1, r2]
+ record["total_usage"]["cost_usd"] *= 3
(ws / "record.json").write_text(json.dumps(record))
- audit = audit_runs([ws])
- kinds = audit.waste_by_kind()
- assert "regression" in kinds and kinds["regression"][1] > 0
- assert any("scored 0.500 vs 0.800" in w.detail for w in audit.waste)
+ ledger = ws / "telemetry" / "cost.jsonl"
+ ledger.parent.mkdir(exist_ok=True)
+ ledger.write_text("\n".join(json.dumps({"run": ws.name, "round": i, "stage": stage, "role": "generator",
+ "label": label, "model": "gemini-3.7-flash", "cost_usd": 0.05})
+ for i, stage, label in ((1, "refine", "refine"), (2, "repair", "r02_refine_repair1")))
+ + "\n")
+ kinds = audit_runs([ws]).waste_by_kind()
+ assert not {"regression", "zero_delta_round", "unpromoted_judge"} & set(kinds)
+ assert kinds["repair_no_converge"] == (1, pytest.approx(0.05))
def test_report_renders(fake_run: Path):
@@ -73,7 +78,7 @@ def test_report_renders(fake_run: Path):
md = markdown(audit)
for heading in ("Per stage", "Per role", "Where a dollar bought nothing", "Routing"):
assert f"## {heading}" in md
- assert "per passing artifact" in md
+ assert "per passing artifact" not in md and "per run" in md
assert "judge" in console(audit)
diff --git a/harness/tests/cost/test_cli.py b/harness/tests/cost/test_cli.py
index d23634d0..7aee9222 100644
--- a/harness/tests/cost/test_cli.py
+++ b/harness/tests/cost/test_cli.py
@@ -16,7 +16,7 @@ def test_cost_show(fake_run: Path, tmp_path: Path):
out = tmp_path / "report.md"
r = runner.invoke(app, ["cost", "show", str(fake_run), "--md", str(out)])
assert r.exit_code == 0, r.output
- assert "passing" in r.stdout and "judge" in r.stdout
+ assert "per run" in r.stdout and "passing" not in r.stdout and "judge" in r.stdout
assert out.is_file() and "## Per stage" in out.read_text()
diff --git a/harness/tests/cost/test_reconstruct.py b/harness/tests/cost/test_reconstruct.py
index c15db4e9..78c9b8ff 100644
--- a/harness/tests/cost/test_reconstruct.py
+++ b/harness/tests/cost/test_reconstruct.py
@@ -18,7 +18,7 @@ def test_reconstructs_and_reconciles(fake_run: Path):
assert abs(led.ledger_usd - led.recorded_usd) < 1e-6, "rows must reconcile to record.total_usage"
stages = {r.stage for r in led.rows}
assert Stage.BASELINE in stages and Stage.JUDGE in stages and Stage.PLAN in stages
- assert led.track == "static_object" and led.passed is True
+ assert led.track == "static_object" and led.round_scores == [0.8] and led.round_built == [None]
def test_generate_event_does_not_double_count_its_session(fake_run: Path):
diff --git a/harness/tests/flywheel_cli/test_cli.py b/harness/tests/flywheel_cli/test_cli.py
index 0aba3779..92eb7ba8 100644
--- a/harness/tests/flywheel_cli/test_cli.py
+++ b/harness/tests/flywheel_cli/test_cli.py
@@ -489,8 +489,8 @@ def test_texture_is_not_offered_on_tracks_that_have_no_glb(tmp_path: Path):
# --------------------------------------------------- render must not publish the wrong round
-def _round_guard_ws(tmp_path: Path, *, best: int, tree: int):
- """A workspace whose record names `best` while the render tree sits at `tree`."""
+def _round_guard_ws(tmp_path: Path, *, tree: int, best: int | None = None):
+ """A workspace whose render tree sits at `tree` (an old record may still name a `best`)."""
import json as _json
from codeverse3d.workspace import Workspace
@@ -499,28 +499,22 @@ def _round_guard_ws(tmp_path: Path, *, best: int, tree: int):
ws.create()
for i in range(tree + 1):
(ws.artifacts / "renders" / f"r{i:02d}").mkdir(parents=True, exist_ok=True)
- ws.record_path.write_text(_json.dumps({"best_round": best}))
+ if best is not None:
+ ws.record_path.write_text(_json.dumps({"best_round": best}))
return ws
def test_render_only_labels_the_working_tree_round(tmp_path: Path):
- """Implicit and explicit round selection must never label another tree's code."""
+ """The tree ends at the last round (2026-09-22), so a plain render IS that round; a
+ --round that names another one would label another round's code, and is refused."""
from codeverse3d.cli._common import CliError
from codeverse3d.cli.inspect_cmd import _render_round_or_refuse
- ws = _round_guard_ws(tmp_path, best=3, tree=4)
+ ws = _round_guard_ws(tmp_path, tree=4, best=3)
+ assert _render_round_or_refuse(ws, None) == 4, "an old record's best_round no longer matters"
with pytest.raises(CliError) as ei:
- _render_round_or_refuse(ws, None)
- msg = str(ei.value)
- assert "BEST round is r3" in msg and "working tree is at r4" in msg
- assert "deliverable" in msg, "it must point at the packaged best round"
- assert "--round 4" in msg, "and offer the explicit escape"
- assert ei.value.exit_code == 2
- with pytest.raises(CliError):
_render_round_or_refuse(ws, 3)
+ msg = str(ei.value)
+ assert "working tree, which is at round 4" in msg and "3dcode pick" in msg and ei.value.exit_code == 2
assert _render_round_or_refuse(ws, 4) == 4, "rendering the tree's own round is fine"
- # The common case stays quiet: no record, or best == tree, just renders.
- assert _render_round_or_refuse(_round_guard_ws(tmp_path / "a", best=2, tree=2), None) == 2
- ws = _round_guard_ws(tmp_path / "b", best=1, tree=1)
- ws.record_path.unlink() # a run that has not written a record yet
- assert _render_round_or_refuse(ws, None) == 1
+ assert _render_round_or_refuse(_round_guard_ws(tmp_path / "b", tree=1), None) == 1
diff --git a/harness/tests/flywheel_cli/test_quality_gallery.py b/harness/tests/flywheel_cli/test_quality_gallery.py
index bf649397..beaa11aa 100644
--- a/harness/tests/flywheel_cli/test_quality_gallery.py
+++ b/harness/tests/flywheel_cli/test_quality_gallery.py
@@ -94,7 +94,7 @@ def test_export_meta_tiers_duplicates_and_captions_sidecar(runs_dir: Path, tmp_p
assert meta["quality_tier"] == "A" and meta["gate_errors"] == 0 and meta["gate_summary"] == {"lint": 0}
assert meta["acceptance"] == [{"id": "a1", "text": "", "how": "", "priority": "", "passed": True}]
assert len(meta["code_fingerprint"]) == 64 and meta["prompt_hash"] == prompt_hash("a wooden dining chair")
- assert [r["index"] for r in meta["rounds_summary"]] == [0, 1] and meta["best_round"] == 1
+ assert [r["index"] for r in meta["rounds_summary"]] == [0, 1] and meta["round"] == 1
u_meta = json.loads((out / "articulated_object" / "urdf_blender" / "cabinet_urdf" / "meta.json").read_text())
assert "meshes/Body.glb" in u_meta["files"] and (out / "articulated_object" / "urdf_blender" / "cabinet_urdf" / "meshes" / "Body.glb").is_file()
# side-car captions are picked up for lamp_three only
diff --git a/harness/tests/flywheel_cli/test_run_layout_cli.py b/harness/tests/flywheel_cli/test_run_layout_cli.py
index 4cfa1440..ff503ed0 100644
--- a/harness/tests/flywheel_cli/test_run_layout_cli.py
+++ b/harness/tests/flywheel_cli/test_run_layout_cli.py
@@ -209,7 +209,7 @@ def test_show_works_on_an_old_layout_run(tmp_path: Path):
ws, _ = old_layout_run(tmp_path)
r = _show(ws.root.name, ws.root.parent)
assert r.exit_code == 0, r.output
- assert "old layout" in r.output and "computed on the fly" in r.output
+ assert "no deliverable/ yet" in r.output and "computed on the fly" in r.output
assert "COST & SETTINGS" in r.output and "models per role" in r.output
assert not ws.telemetry.exists() # show never writes
diff --git a/harness/tests/gallery/test_index.py b/harness/tests/gallery/test_index.py
index 2ed5f7e4..9285ee03 100644
--- a/harness/tests/gallery/test_index.py
+++ b/harness/tests/gallery/test_index.py
@@ -28,9 +28,9 @@ def test_build_index_sections_and_states(gallery_tree: dict[str, Path]):
def test_entry_fields_and_links(gallery_tree: dict[str, Path]):
entry = entry_for_dir("runs", gallery_tree["runs"] / "wooden_chair_ab12cd34")
- assert entry.state == "ok" and entry.passed is True and entry.tier == "A"
+ assert entry.state == "ok" and entry.verdict == "judged" and entry.tier == "A"
assert entry.score == 0.80 and entry.baseline_score == 0.55 and entry.rounds == 2
- assert entry.best_round == 1 and entry.minutes is not None
+ assert entry.picked_round == 1 and entry.minutes is not None
assert entry.sheet == "artifacts/renders/r01/sheet.png"
labels = [link.label for link in entry.links]
assert labels[:2] == ["workspace", "record.json"]
@@ -68,13 +68,11 @@ def test_filter_and_sort_and_summary(gallery_tree: dict[str, Path]):
threejs = [e for e in entries if match(e, {"lang": "threejs"})]
assert [e.slug for e in threejs] == ["lamp_three"]
assert [e.slug for e in entries if match(e, {"q": "TOASTER"})] == ["ctrl_med_toaster"]
- assert [e.slug for e in entries if match(e, {"pass": "fail"})] == ["lamp_three"]
assert {e.slug for e in entries if match(e, {"battery": "static_v9"})} == {
"art_easy_hinge", "ctrl_med_toaster", "half_written", "not_started"}
s = summarize([e for e in entries if e.state == "ok"])
- assert s.n == 4 and s.n_judged == 4 and s.n_passed == 3
- assert s.pass_rate == 0.75 and s.mean_score is not None and s.median_score is not None
- assert s.total_usd > 0 and s.usd_per_pass == round(s.total_usd / 3, 4)
+ assert s.n == 4 and s.n_judged == 4 and s.mean_score is not None and s.median_score is not None
+ assert s.total_usd > 0 and not hasattr(s, "pass_rate")
facets = index.facets()
assert facets["track"] == ["articulated_object", "static_object"]
assert facets["battery"] == ["runs", "static_v9"]
diff --git a/harness/tests/gallery/test_page.py b/harness/tests/gallery/test_page.py
index d0e3e239..ec846172 100644
--- a/harness/tests/gallery/test_page.py
+++ b/harness/tests/gallery/test_page.py
@@ -75,7 +75,7 @@ def test_static_build_parses_and_is_self_contained(gallery_tree: dict[str, Path]
assert markup.startswith("<!doctype html>")
assert "data:image/jpeg;base64," in markup # --embed inlined the sheets
assert any(u.startswith("file://") for u in checker.urls) # links point at the run dirs
- assert "gallery-data" in checker.ids and "s-perpass" in checker.ids
+ assert "gallery-data" in checker.ids and "s-cost" in checker.ids and "s-perpass" not in checker.ids
assert markup.count("<article class='card") == 6
for slug in ("wooden_chair_ab12cd34", "half_written", "not_started"):
assert slug in markup
diff --git a/harness/tests/gallery/test_triage.py b/harness/tests/gallery/test_triage.py
index ffaa55fc..0c4e7597 100644
--- a/harness/tests/gallery/test_triage.py
+++ b/harness/tests/gallery/test_triage.py
@@ -1,9 +1,10 @@
"""The triage affordances: the status breakdown, the hero view, label humanisation,
multi-select routing (compare / CSV) and the detail page's neighbour links.
-The invariant worth a test is the arithmetic one: the four verdict buckets are
-disjoint and exhaustive, so the strip's "n runs shown = a + b + c + d" is true for
-*any* selection — including one holding half-written records.
+The invariant worth a test is the arithmetic one: the verdict buckets are disjoint and
+exhaustive, so the strip's "n runs shown = a + b + c" is true for *any* selection —
+including one holding half-written records. None of them is a pass or a fail
+(2026-09-22): a run is judged, unjudged or unreadable.
"""
from __future__ import annotations
@@ -35,14 +36,14 @@ from codeverse3d.addons.gallery.urls import UrlMaker
# --------------------------------------------------------------------------- breakdown
-def test_the_four_buckets_are_disjoint_and_sum_to_n(gallery_tree: dict[str, Path]):
+def test_the_buckets_are_disjoint_and_sum_to_n(gallery_tree: dict[str, Path]):
entries = build_index([gallery_tree["runs"], gallery_tree["battery"]]).entries()
s = summarize(entries)
assert s.n == 6
assert set(s.breakdown) == set(VERDICTS)
assert sum(s.breakdown.values()) == s.n
- # the tree holds 3 passing, 1 failing, and 2 runs with no usable record
- assert s.breakdown == {"passed": 3, "failed": 1, "unjudged": 0, "error": 2}
+ # the tree holds 4 judged runs and 2 with no usable record
+ assert s.breakdown == {"judged": 4, "unjudged": 0, "error": 2}
# every entry lands in exactly one bucket
assert sorted(e.verdict for e in entries) == sorted(
k for k, n in s.breakdown.items() for _ in range(n))
@@ -53,7 +54,7 @@ def test_a_run_with_no_record_is_error_not_unjudged(gallery_tree: dict[str, Path
by_slug = {e.slug: e for e in entries}
assert by_slug["half_written"].verdict == "error" # corrupt record
assert by_slug["not_started"].verdict == "error" # no record at all
- assert by_slug["half_written"].passed is None # ...and still unjudged in the old sense
+ assert by_slug["half_written"].score is None # ...and nothing to score
def test_breakdown_of_an_empty_selection_is_all_zeroes():
@@ -68,13 +69,12 @@ def test_the_page_states_the_breakdown_and_it_adds_up(gallery_tree: dict[str, Pa
for bucket in VERDICTS:
assert f"id='vc-{bucket}'" in markup
assert "runs shown" in markup
- # the old un-addable "pass rate / not ok" pair of tiles is gone
- assert "PASS RATE" not in markup.upper()
+ # no pass rate and no pass/fail bucket: a run is not passed or failed
+ assert "PASS RATE" not in markup.upper() and "per pass" not in markup and "vc-passed" not in markup
# --------------------------------------------------------------------------- verdict filter
[email protected](("verdict", "n"), [("passed", 3), ("failed", 1), ("error", 2),
- ("unjudged", 0)])
[email protected](("verdict", "n"), [("judged", 4), ("error", 2), ("unjudged", 0)])
def test_verdict_filter_selects_exactly_its_bucket(gallery_tree: dict[str, Path], verdict: str, n: int):
entries = build_index([gallery_tree["runs"], gallery_tree["battery"]]).entries()
kept = [e for e in entries if match(e, {"verdict": verdict})]
@@ -85,8 +85,8 @@ def test_verdict_filter_reaches_the_api_and_the_page(gallery_tree: dict[str, Pat
app = GalleryApp([gallery_tree["runs"], gallery_tree["battery"]])
payload = json.loads(app.route("/api/runs", {"verdict": "error"}).body)
assert payload["n"] == 2 and {r["state"] for r in payload["runs"]} == {"broken", "pending"}
- page = app.route("/", {"verdict": "failed"}).body.decode()
- assert "<option value='failed' selected>" in page
+ page = app.route("/", {"verdict": "judged"}).body.decode()
+ assert "<option value='judged' selected>" in page
# --------------------------------------------------------------------------- hero view
@@ -108,7 +108,7 @@ def test_hero_falls_back_to_the_sheet_when_a_round_has_no_views(tmp_path: Path):
ws, rec = make_fake_run(tmp_path / "runs", "noviews")
for rnd in rec.rounds:
rnd.renders = None
- assert hero_view(ws, rec) == ("", "", 0)
+ assert hero_view(ws, rec, 1) == ("", "", 0) and hero_view(ws, rec, None) == ("", "", 0)
# --------------------------------------------------------------------------- labels
@@ -210,7 +210,7 @@ def test_export_csv_route_and_columns(gallery_tree: dict[str, Path]):
assert "attachment" in r.headers["Content-Disposition"]
lines = r.body.decode().strip().splitlines()
assert lines[0] == ",".join(CSV_COLUMNS) and len(lines) == 2
- assert "wooden_chair_ab12cd34" in lines[1] and ",passed," in lines[1]
+ assert "wooden_chair_ab12cd34" in lines[1] and ",judged," in lines[1]
def test_export_csv_without_a_selection_exports_the_current_filter(gallery_tree: dict[str, Path]):
diff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.py
index fca4abdf..4b689cab 100644
--- a/harness/tests/judges/test_calibration.py
+++ b/harness/tests/judges/test_calibration.py
@@ -44,7 +44,7 @@ def test_plan_digest_and_load_cases(tmp_path):
cases = load_run_cases(run)
assert [c.round_index for c in cases] == [0, 1] and cases[0].gate_errors == 1 and cases[1].gate_errors == 0
assert cases[0].rubric == "static_object_v1" and len(cases[0].inp.acceptance) == 2
- assert cases[1].is_best and not cases[0].is_best and cases[0].glb is None
+ assert cases[1].is_picked and not cases[0].is_picked and cases[0].glb is None # no usable record: the last round
assert cases[0].inp.plan_summary == "Chair: a chair. Overall 0.50×0.50×0.90 m. Parts: Seat, Leg×4."
assert plan_digest({"title": "Harbour", "summary": "s", "zones": [{"name": "quay"}], "cameras": [{"name": "cam_hero"}], "setting": "dusk"}) \
== "Harbour: s Zones: quay. Cameras: cam_hero. Setting: dusk."