Skip to content

Official Codebase of 3DCodeVerse

Python

tracks: no best round in the core — every round refines the previous one, the workspace ends at the last

gaoypengd4c1ce4c474a
diff --git a/eval/tests/test_degraded.py b/eval/tests/test_degraded.py
index ba93d2e3..9348dd28 100644
--- a/eval/tests/test_degraded.py
+++ b/eval/tests/test_degraded.py
@@ -34,7 +34,7 @@ def _track(status: RunStatus, stop_reason: str, rounds: int, cost: float, aborte
def run(spec, ws, resume):
ws.src.mkdir(parents=True, exist_ok=True)
(ws.root / "src" / "model.py").write_text(GOOD.format(score=0.6))
- rec = RunRecord(spec=spec, workspace=str(ws.root), status=status, final_score=0.6,
+ rec = RunRecord(spec=spec, workspace=str(ws.root), status=status,
rounds=[RoundRecord(index=i, kind="baseline" if i == 0 else "refine") for i in range(rounds)],
total_usage=Usage(cost_usd=cost),
extra={"stop_reason": stop_reason, "aborted_rounds": [{"index": rounds}] * aborted})
diff --git a/harness/codeverse3d/addons/dataset/__init__.py b/harness/codeverse3d/addons/dataset/__init__.py
index 7fb9ef86..74a98be7 100644
--- a/harness/codeverse3d/addons/dataset/__init__.py
+++ b/harness/codeverse3d/addons/dataset/__init__.py
@@ -6,6 +6,6 @@
* ``refine`` the refine-transition rows (``RefineTransition``)
* ``captions`` ``caption_sample(ws, record, model_id)`` → ``Captions``
* ``index`` ``build_index(runs_dir, out_sqlite)`` + query helpers
-* ``sample`` one run → one dataset sample (the code files of its best round)
+* ``sample`` one run → one dataset sample (the code files of its picked round)
* ``quality`` ``quality_tier`` (A/B/C/D) · ``prompt_hash`` · code fingerprints, duplicate groups
"""
diff --git a/harness/codeverse3d/addons/dataset/sample.py b/harness/codeverse3d/addons/dataset/sample.py
index 06591d32..3da9312f 100644
--- a/harness/codeverse3d/addons/dataset/sample.py
+++ b/harness/codeverse3d/addons/dataset/sample.py
@@ -305,7 +305,7 @@ def build_meta(
n_gate_errors = sum(gates.values())
# difficulty next to quality: a tier-A five-box stool and a tier-A machine are
# not the same training sample (eval/docs/COMPLEXITY.md)
- cx = record.extra.get("complexity") or (round_complexity(rnd) if rnd is not None else None)
+ cx = (round_complexity(rnd) if rnd is not None else None) or record.extra.get("complexity")
name = ""
if record.plan is not None:
name = getattr(record.plan, "object_name", "") or getattr(record.plan, "title", "")
diff --git a/harness/codeverse3d/addons/gallery/detail.py b/harness/codeverse3d/addons/gallery/detail.py
index 402ba90a..763f8100 100644
--- a/harness/codeverse3d/addons/gallery/detail.py
+++ b/harness/codeverse3d/addons/gallery/detail.py
@@ -221,10 +221,12 @@ def _complexity_panel(entry: RunEntry, rec: RunRecord) -> str:
v = entry.complexity_axes.get(axis, block.get(axis))
if isinstance(v, (int, float)):
rows.append(_kv(label, spec.format(float(v))))
- if isinstance(block.get("plan_parts"), int) and block["plan_parts"]:
- rows.append(_kv("plan parts", str(block["plan_parts"])))
- if block.get("parts_per_plan_part") is not None:
- rows.append(_kv("built / planned parts", f"{float(block['parts_per_plan_part']):.2f}"))
+ plan_parts = block.get("plan_parts")
+ if isinstance(plan_parts, int) and plan_parts:
+ rows.append(_kv("plan parts", str(plan_parts)))
+ built = entry.complexity_axes.get("part_count", block.get("part_count")) # the picked round's
+ if isinstance(built, (int, float)):
+ rows.append(_kv("built / planned parts", f"{float(built) / plan_parts:.2f}"))
trail = block.get("by_round")
if isinstance(trail, list) and len(trail) > 1:
rows.append(_kv("by round", " → ".join(f"{float(x):.2f}" for x in trail)))
diff --git a/harness/codeverse3d/addons/select.py b/harness/codeverse3d/addons/select.py
index 4901126a..948b7391 100644
--- a/harness/codeverse3d/addons/select.py
+++ b/harness/codeverse3d/addons/select.py
@@ -32,7 +32,7 @@ from typing import Any, Literal
from pydantic import BaseModel, Field
from codeverse3d.contracts.common import TRACK_INFO, Usage
-from codeverse3d.contracts.run import PairwiseNote, RunRecord
+from codeverse3d.contracts.run import RunRecord
from codeverse3d.cost.context import call_context
from codeverse3d.cost.types import Role, Stage
from codeverse3d.judges.base import judged_subset, resolve_paths
@@ -90,6 +90,20 @@ class Selection(BaseModel):
selected_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+class PairwiseNote(BaseModel):
+ """One paid pairwise verdict — does round ``b`` beat round ``a``? — as cached in
+ ``artifacts/judge/rAA_vs_rBB_pairwise.json`` so a second ``3dcode pick`` re-reads it."""
+
+ a: str = Field(description="label of the top-scored round")
+ b: str = Field(description="label of the runner-up")
+ winner: Literal["a", "b", "tie"] = "tie"
+ confidence: float = 0.0
+ accepted: bool = Field(default=False, description="True when the runner-up is picked instead")
+ reasons: list[str] = Field(default_factory=list)
+ usage: Usage = Field(default_factory=Usage)
+ error: str = ""
+
+
# --------------------------------------------------------------------------- rows
def _load(run_dir: Path | str, record: RunRecord | None) -> RunRecord:
return record if record is not None else load_record(Workspace(run_dir))
diff --git a/harness/codeverse3d/cli/flywheel_cmd.py b/harness/codeverse3d/cli/flywheel_cmd.py
index 3e1e8472..98f591b4 100644
--- a/harness/codeverse3d/cli/flywheel_cmd.py
+++ b/harness/codeverse3d/cli/flywheel_cmd.py
@@ -24,7 +24,7 @@ def export_cmd(
only_passed: Annotated[bool, typer.Option("--only-passed")] = False,
pack: Annotated[bool, typer.Option("--pack", help="also pack samples-NNN.tar + byte-range locators")] = False,
tar_prefix: Annotated[str, typer.Option("--tar-prefix", help="repo-root-relative prefix for the tar column")] = "",
- include_unbuilt: Annotated[bool, typer.Option("--include-unbuilt", help="also export runs whose best round never built")] = False,
+ include_unbuilt: Annotated[bool, typer.Option("--include-unbuilt", help="also export runs whose exported round never built")] = False,
captions_dir: Annotated[Path | None, typer.Option("--captions-dir", help="side-car captions written by `caption --out`")] = None,
drop_duplicates: Annotated[bool, typer.Option("--drop-duplicates", help="leave byte-identical duplicates (raw code sha256 + prompt) out of the index, manifest and tars (recorded under the manifest's dropped); normalised ones are only marked near_duplicate_of")] = False,
) -> None:
diff --git a/harness/codeverse3d/contracts/run.py b/harness/codeverse3d/contracts/run.py
index acad4a2b..c4d208aa 100644
--- a/harness/codeverse3d/contracts/run.py
+++ b/harness/codeverse3d/contracts/run.py
@@ -197,28 +197,6 @@ class RunStatus(StrEnum):
return cls.STOPPED if value in ("passed", "plateau") else None
-class PairwiseNote(BaseModel):
- """What a tie-break compared and what it concluded.
-
- Lives on the round record because the comparison is a PAID judge call (~$0.05):
- ``rNN.json`` is the durable artifact, so a resume replays the verdict instead of
- re-ranking on score alone and silently reversing it
- (``tracks.candidates.replay_best_round``)."""
-
- a: str = Field(description="label of the incumbent (current best)")
- b: str = Field(description="label of the challenger (new round / other candidate)")
- winner: Literal["a", "b", "tie"] = "tie"
- confidence: float = 0.0
- accepted: bool = Field(default=False, description="True when the challenger replaces the incumbent")
- reasons: list[str] = Field(default_factory=list)
- usage: Usage = Field(default_factory=Usage)
- error: str = ""
-
- def line(self) -> str:
- verdict = {"a": f"{self.a} wins", "b": f"{self.b} wins", "tie": "tie"}[self.winner]
- return f"pairwise {self.a} vs {self.b}: {verdict} (confidence {self.confidence:.2f}) → {'replace' if self.accepted else 'keep'}"
-
-
class RoundRecord(BaseModel):
index: int
kind: str = Field(description="baseline | refine | repair | texture | asset:<name> ...")
@@ -236,9 +214,6 @@ class RoundRecord(BaseModel):
started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
duration_s: float = 0.0
notes: str = ""
- pairwise: PairwiseNote | None = Field(
- default=None, description="the paid tie-break verdict that ranked this round against the "
- "incumbent best, when one was bought; None = ranked on score alone")
@property
def score(self) -> float | None:
@@ -358,14 +333,16 @@ class RunDeliverable(BaseModel):
class RunRecord(BaseModel):
+ """One run. A record written before 2026-09-22 also names a best round, a baseline and a
+ final score (``best_round`` / ``baseline_score`` / ``final_score``) and a round may carry a
+ ``pairwise`` tie-break note: unknown keys, ignored — ``addons.select`` answers which round
+ to hand over, after the run."""
+
spec: Spec
plan: StaticPlan | ArticulatedPlan | ScenePlan | GraphicsPlan | None = None
workspace: str
status: RunStatus = RunStatus.PLANNING
rounds: list[RoundRecord] = Field(default_factory=list)
- best_round: int | None = None
- baseline_score: float | None = None
- final_score: float | None = None
total_usage: Usage = Field(default_factory=Usage)
environment: dict[str, str] = Field(default_factory=dict, description="tool versions, git sha, host")
prompt_hashes: dict[str, str] = Field(default_factory=dict)
diff --git a/harness/codeverse3d/cost/routing.py b/harness/codeverse3d/cost/routing.py
index f1196387..be7c7b73 100644
--- a/harness/codeverse3d/cost/routing.py
+++ b/harness/codeverse3d/cost/routing.py
@@ -68,11 +68,11 @@ ROUTES: tuple[Route, ...] = (
# ------------------------------------------------------------------ judge
Route(Role.JUDGE, "gemini:gemini-3.1-pro-preview", 0.060,
"σ 0.030, pearson(gate errors, score) +0.63",
- "every decision that persists: best-round selection, pass/fail, dataset tiering",
+ "every verdict that persists: the round a pick hands over, dataset tiering",
default=True),
Route(Role.JUDGE, "gemini:gemini-3.7-flash", 0.027,
"σ 0.083 and pearson −0.33 (anti-correlated with the gates)",
- "in-loop refine hints only, where a wrong rank costs one round, never for pass/fail"),
+ "in-loop refine hints only — its scores are what a pick then ranks, so not where the pick matters"),
# ------------------------------------------------------------------ image
Route(Role.IMAGE, "gemini:gemini-3.1-flash-image", 0.067,
"per 1024² tile; a texture pass is ~2 tiles + plan + gate ≈ $0.13",
diff --git a/harness/codeverse3d/orchestrator.py b/harness/codeverse3d/orchestrator.py
index dc69e75d..8b068619 100644
--- a/harness/codeverse3d/orchestrator.py
+++ b/harness/codeverse3d/orchestrator.py
@@ -1,5 +1,5 @@
"""Orchestration: the stage runner with resume, run state, the round policy
-(RoundPolicy / pick_best_round), refine-task compilation and the budget.
+(RoundPolicy), refine-task compilation and the budget.
One module since 2026-08-28: the seven-file package predates the 1,500-line cap,
nobody ever imported the package itself, and refine_tasks existed only to be
@@ -46,22 +46,15 @@ class StageState(BaseModel):
class RunState(BaseModel):
- """Everything the orchestrator needs to resume a run."""
+ """Everything the orchestrator needs to resume a run. A state saved before 2026-09-22
+ also names a best round (``best_round`` / ``best_commit`` / ``best_score`` /
+ ``best_considered_through``): unknown keys, ignored — a resume goes on from the LAST round."""
status: RunStatus = RunStatus.PLANNING
stages: dict[str, StageState] = Field(default_factory=dict)
current_round: int = Field(default=0, description="index of the round in progress / next to run")
completed_rounds: list[int] = Field(default_factory=list)
round_commits: dict[int, str] = Field(default_factory=dict)
- best_round: int | None = None
- best_commit: str = ""
- best_score: float | None = None
- #: highest round index that has been THROUGH best selection (choose_best_round,
- #: including its paid pairwise comparison). Persisted so a resume can tell
- #: "the best was deliberately kept" from "this round was never considered" —
- #: without it, a state that merely LOOKS consistent with the journal kept a
- #: stale best and delivered the worse round (2026-08-27). -1 = nothing yet.
- best_considered_through: int = -1
materialized_for: str = Field(default="", description="agent kind the workspace was materialised for")
stop_reason: str = ""
error: str = ""
@@ -99,12 +92,6 @@ class RunState(BaseModel):
self.round_commits[index] = commit
self.current_round = index + 1
- def update_best(self, index: int, commit: str, score: float | None) -> bool:
- """Record ``index`` as best; returns True when it changed."""
- changed = self.best_round != index
- self.best_round, self.best_commit, self.best_score = index, commit, score
- return changed
-
class StateCorrupt(RuntimeError):
"""``run_state.json`` exists but does not validate — refuse to guess."""
@@ -453,32 +440,10 @@ class RoundPolicy:
judge_samples: int = 1 # VLM judge samples per round (flash: std ≈ 0.001 between samples → 1 is enough)
-def pick_best_round(rounds: Sequence[RoundRecord]) -> int | None:
- """Best round = highest score, then fewer gate errors, then later.
-
- A round that did not build is never picked (it has no score, and delivering code
- that does not run is never an improvement); when NO round has a score the fallback
- is the built round with the fewest gate errors — later on a tie — so a round that
- broke the gates cannot displace a clean earlier artifact just by being last."""
- scored = [i for i, r in enumerate(rounds) if r.score is not None and not _build_failed(r)]
- if scored:
- return max(scored, key=lambda i: (float(rounds[i].score), -gate_error_count(rounds[i]), i)) # type: ignore[arg-type]
- built = [i for i, r in enumerate(rounds) if r.build is not None and r.build.ok]
- if not built:
- return None
- return min(built, key=lambda i: (gate_error_count(rounds[i]), -i))
-
-
def gate_error_count(r: RoundRecord) -> int:
return sum(len(g.errors) for g in r.gates)
-def _build_failed(r: RoundRecord) -> bool:
- """The round is KNOWN not to build (a record with no build at all is not a failure:
- resumed/rejudged records may carry a score without one)."""
- return r.build is not None and not r.build.ok
-
-
# ===================================================================== budget
#: stage label used when a caller does not say where the money went
diff --git a/harness/codeverse3d/record/__init__.py b/harness/codeverse3d/record/__init__.py
index 56ec43b1..096bc600 100644
--- a/harness/codeverse3d/record/__init__.py
+++ b/harness/codeverse3d/record/__init__.py
@@ -1,8 +1,8 @@
"""What every run WRITES so it can be read later: ``record.json`` and what travels with it.
* ``record`` ``finalize_record(ws, record)`` · ``load_record(ws)`` · ``iter_runs(runs_dir)`` ·
- ``best_round_record`` · ``unique_files``
-* ``deliverable`` the run's ``deliverable/`` directory
+ ``effective_judgment`` · ``unique_files``
+* ``deliverable`` every round's kept build (``artifacts/rNN/``) and ``deliverable/`` for one round
* ``telemetry`` the settings / environment a record carries
* ``_git`` reading the workspace's own git history (files at a round's commit)
diff --git a/harness/codeverse3d/record/_git.py b/harness/codeverse3d/record/_git.py
index b31d345a..258822e7 100644
--- a/harness/codeverse3d/record/_git.py
+++ b/harness/codeverse3d/record/_git.py
@@ -127,7 +127,7 @@ def diff_between(ws: Workspace, before: str, after: str, *, max_bytes: int | Non
``total_bytes`` is what git actually produced, so a capped row still records the
size it was capped from. The useful shas are the ones recorded on the rounds,
- never ``HEAD`` — a finished run ends on a "restore best round rNN" commit — and a
+ never ``HEAD`` — a run recorded before 2026-09-22 may end on a "restore best round rNN" commit — and a
sha the repository no longer holds raises rather than diffing against an empty
tree: ``git diff`` itself refuses an unknown object, so no pre-check is needed.
"""
diff --git a/harness/codeverse3d/record/record.py b/harness/codeverse3d/record/record.py
index 0bda4386..8b780193 100644
--- a/harness/codeverse3d/record/record.py
+++ b/harness/codeverse3d/record/record.py
@@ -1,10 +1,9 @@
"""Run records: finalise / load / iterate.
``finalize_record`` is the last step of every track run: it fills the
-environment (tool versions + harness git sha), derives best/baseline/final
-scores and totals when the track left them empty, adds a compact per-round
-summary under ``record.extra["rounds_summary"]``, writes ``telemetry/`` (see
-docs/RUN_LAYOUT.md) and writes ``record.json`` atomically. ``deliverable/`` is not
+environment (tool versions + harness git sha) and the totals when the track left
+them empty, adds a compact per-round summary under ``record.extra["rounds_summary"]``,
+writes ``telemetry/`` (see docs/RUN_LAYOUT.md) and writes ``record.json`` atomically. ``deliverable/`` is not
the run's: which round to hand over is ``codeverse3d.addons.select``'s question
(2026-09-22). Telemetry is best-effort: a run is never failed by it.
"""
@@ -135,30 +134,6 @@ def effective_score(r: RoundRecord) -> float | None:
return j.overall if j is not None else None
-def best_round_index(record: RunRecord) -> int | None:
- """The round to export: ``record.best_round`` when valid, else the highest
- judged score (ties → fewer gate errors), else the last round with a
- successful build, else the last round. Degraded judgments count as unjudged."""
- by_index = {r.index: r for r in record.rounds}
- if record.best_round is not None and record.best_round in by_index:
- return record.best_round
- if not record.rounds:
- return None
- judged = [r for r in record.rounds if effective_judgment(r) is not None]
- if judged:
- best = max(judged, key=lambda r: (effective_score(r), -_n_gate_errors(r), r.index)) # type: ignore[arg-type]
- return best.index
- built = [r for r in record.rounds if r.build is not None and r.build.ok]
- if built:
- return built[-1].index
- return record.rounds[-1].index
-
-
-def best_round_record(record: RunRecord) -> RoundRecord | None:
- idx = best_round_index(record)
- return next((r for r in record.rounds if r.index == idx), None)
-
-
def _n_gate_errors(r: RoundRecord) -> int:
return sum(len(g.errors) for g in r.gates)
@@ -202,15 +177,9 @@ def _sum_usage(rounds: list[RoundRecord]) -> Usage:
def fill_derived(record: RunRecord) -> RunRecord:
- """Fill best/baseline/final/total_usage/finished_at/environment when absent."""
- if record.best_round is None:
- record.best_round = best_round_index(record)
- scored = [r for r in record.rounds if effective_score(r) is not None]
- if record.baseline_score is None and scored:
- record.baseline_score = effective_score(scored[0])
- if record.final_score is None and record.best_round is not None:
- best = best_round_record(record)
- record.final_score = effective_score(best) if best else None
+ """Fill total_usage/finished_at/environment when absent, the per-round summary and the
+ complexity block. No best round, baseline or final score: which round counts is
+ ``addons.select``'s question (2026-09-22)."""
if record.total_usage.cost_usd == 0 and record.total_usage.input_tokens == 0 and record.rounds:
record.total_usage = _sum_usage(record.rounds)
if record.finished_at is None:
@@ -226,17 +195,12 @@ def fill_derived(record: RunRecord) -> RunRecord:
def complexity_block(record: RunRecord) -> dict[str, Any] | None:
- """``record.extra["complexity"]``: the delivered artifact's complexity vector
- plus what the plan asked for.
-
- The vector is the one measured on the BEST round — the round whose code is
- restored and rebuilt at finalise, so it describes the artifact actually
- shipped. ``plan_parts`` / ``parts_per_plan_part`` say whether the build
- reached the plan's ambition or collapsed it (eval/docs/COMPLEXITY.md)."""
- best = best_round_record(record)
- cx = round_complexity(best) if best is not None else None
- if cx is None:
- cx = next((c for c in (round_complexity(r) for r in reversed(record.rounds)) if c), None)
+ """``record.extra["complexity"]``: the complexity vector of the last measured round —
+ the build ``src/`` and ``artifacts/`` end at — plus what the plan asked for and the
+ per-round trail. ``plan_parts`` / ``parts_per_plan_part`` say whether the build
+ reached the plan's ambition or collapsed it (eval/docs/COMPLEXITY.md). A reader that
+ wants the picked round's vector reads that round's own (:func:`round_complexity`)."""
+ cx = next((c for c in (round_complexity(r) for r in reversed(record.rounds)) if c), None)
if cx is None:
return None
plan_parts = len(record.plan.parts) if record.plan is not None else 0
diff --git a/harness/codeverse3d/texturing/run.py b/harness/codeverse3d/texturing/run.py
index 9e85a543..94d26cfa 100644
--- a/harness/codeverse3d/texturing/run.py
+++ b/harness/codeverse3d/texturing/run.py
@@ -311,7 +311,7 @@ def _finish(ws: Workspace, report: TextureReport, t0: float, events: EventLog, u
def report_path(ws: Workspace) -> Path:
- """THE location of the texture report — the finalise double-buy guard reads it too."""
+ """THE location of the texture report — ``addons.select``'s double-buy guard reads it too."""
return ws.artifacts / TEXTURES_DIR / REPORT_NAME
diff --git a/harness/codeverse3d/tracks/candidates.py b/harness/codeverse3d/tracks/candidates.py
index 13dc5ef6..ca684582 100644
--- a/harness/codeverse3d/tracks/candidates.py
+++ b/harness/codeverse3d/tracks/candidates.py
@@ -29,7 +29,7 @@ from codeverse3d.contracts.artifacts import BuildResult, Measurement, RenderSet
from codeverse3d.contracts.common import Usage
from codeverse3d.contracts.run import RoundRecord
from codeverse3d.conventions import OBJECT_VIEWS_QUICK
-from codeverse3d.orchestrator import BudgetExceeded, gate_error_count, pick_best_round
+from codeverse3d.orchestrator import BudgetExceeded, gate_error_count
from codeverse3d.proc import EventLog, fan_out
from codeverse3d.tracks.common import RunContext
from codeverse3d.tracks.generation import GenerationTask
@@ -196,35 +196,6 @@ def adopt_candidate(ctx: RunContext, sub_ws: Workspace) -> None:
shutil.copytree(sub_ws.trajectories, ctx.ws.trajectories, dirs_exist_ok=True)
-# ----------------------------------------------------------------------------- best round
-def choose_best_round(ctx: RunContext, rounds: list[RoundRecord], new_index: int) -> int | None:
- """Index of the best round after ``rounds[new_index]`` finished (no pairwise tie-break is
- bought any more; verdicts stored by earlier runs are still honoured by the replay)."""
- return replay_best_round(rounds)
-
-
-def replay_best_round(journal: Sequence[RoundRecord]) -> int | None:
- """The best index, walking the journal sequentially and honouring stored verdicts.
-
- Each round is an incumbent-vs-challenger step: a round carrying a
- :class:`PairwiseNote` was decided inside ``policy.pairwise_margin``, where score
- ranking has nothing to say, so the stored verdict — replace or keep — is FINAL
- (a later round can never revive a rejected challenger by global re-ranking, and a
- kill before the state save cannot reverse a paid ~$0.05 judgement — 18 such runs
- on disk, 2026-08-30); every other round advances by the two-way :func:`pick_best_round`
- rule, whose key is a total order — absent verdicts this equals the global pick."""
- best: int | None = None
- for i, rec in enumerate(journal):
- if rec.pairwise is not None and best is not None and best != i:
- if rec.pairwise.accepted:
- best = i
- elif best is None:
- best = pick_best_round(journal[:i + 1])
- elif pick_best_round([journal[best], rec]) == 1:
- best = i
- return best
-
-
# ===================================================================== decision logic
class CandidateRecord(BaseModel):
"""One best-of-N baseline candidate (generated in its own sub-workspace)."""
@@ -251,5 +222,5 @@ def rank_candidates(records: Sequence[CandidateRecord]) -> list[int]:
return [r.index for r in sorted(records, key=lambda r: r.sort_key(), reverse=True)]
-__all__ = ["CAND_DIR", "CandidateRecord", "adopt_candidate", "choose_best_round", "make_candidate_context",
- "quick_render", "rank_candidates", "replay_best_round", "run_best_of_n"]
+__all__ = ["CAND_DIR", "CandidateRecord", "adopt_candidate", "make_candidate_context", "quick_render",
+ "rank_candidates", "run_best_of_n"]
diff --git a/harness/codeverse3d/tracks/lifecycle.py b/harness/codeverse3d/tracks/lifecycle.py
index 70a4fd4e..c3f18781 100644
--- a/harness/codeverse3d/tracks/lifecycle.py
+++ b/harness/codeverse3d/tracks/lifecycle.py
@@ -4,7 +4,8 @@ Concrete tracks (static / articulated / scene / graphics) override the hooks
(``plan_model``, ``make_pipeline``, ``prepare``, ``baseline_tasks``, the planner
knobs ``plan_template``/``plan_example``/``finalise_plan`` and the refine
scaffold's ``_refine_task``). All bookkeeping — workspace, events, budget, run
-state, stage cache, best tracking, the fixed round count, record — lives here once.
+state, stage cache, the fixed round count, record — lives here once. Which round of a
+finished run to hand over is not the core's question: ``codeverse3d.addons.select``.
"""
from __future__ import annotations
@@ -40,7 +41,7 @@ from codeverse3d.orchestrator import (
)
from codeverse3d.proc import EventLog
from codeverse3d.prompts import render
-from codeverse3d.tracks.candidates import choose_best_round, replay_best_round, run_best_of_n
+from codeverse3d.tracks.candidates import run_best_of_n
from codeverse3d.tracks.common import (
RunContext,
Services,
@@ -325,7 +326,7 @@ class BaseTrack:
The journal (``rounds/rNN.json`` + its git commit, written at the end of every
round) is the durable record; the state file is a cache of it that a crash can
- leave stale — round persisted, ``mark_round_done``/best never saved. On resume:
+ leave stale — round persisted, ``mark_round_done`` never saved. On resume:
1. **Spec identity.** ``state.extra["spec_fingerprint"]`` (hash of
:func:`plan_stage_inputs`, stamped on first run) must match the spec we are
@@ -335,12 +336,11 @@ class BaseTrack:
to ``rounds/pre_force/`` — old rounds are never paired with a new spec/plan.
2. **Journal integrity.** Trailing rounds whose commit git does not have are
dropped (half-written journal), with a ``resume.dropped_rounds`` event.
- 3. **Rebuild.** ``completed_rounds``/``round_commits`` come from the journal;
- when the state disagrees with it (rounds it never promoted, a best pointing
- at the wrong commit) the best is recomputed with :func:`replay_best_round`,
- which re-walks the journal applying the pairwise verdicts the rounds stored
- — never buying a new one, never reversing one already paid for.
- A state that already agrees keeps its best untouched.
+ 3. **Rebuild.** ``completed_rounds``/``round_commits`` come from the journal, and
+ the working tree goes back on the LAST round's commit when anything left it
+ elsewhere — a crash mid-round, or the finalise of a run recorded before
+ 2026-09-22, which restored its best round. The next round refines the round
+ before it, always.
Returns the loop's initial round history. A fresh (non-resume) run only
stamps the fingerprint and starts empty."""
@@ -367,7 +367,6 @@ class BaseTrack:
journal = []
state.stages.pop("plan", None) # the plan must be rebuilt from the edited spec
state.completed_rounds, state.round_commits, state.current_round = [], {}, 0
- state.best_round, state.best_commit, state.best_score = None, "", None
state.extra["spec_fingerprint"] = fp
kept: list[RoundRecord] = []
dropped: list[int] = []
@@ -387,32 +386,13 @@ class BaseTrack:
state.completed_rounds = [r.index for r in journal]
state.round_commits = dict(journal_commits)
state.current_round = len(journal)
- best = state.best_round
- best_invalid = best is not None and (best >= len(journal) or journal[best].commit != state.best_commit)
- # a round the journal has but best selection never saw: the state can look
- # perfectly consistent (completed_rounds and commits agree) and still name a
- # stale best, which is then restored and DELIVERED over the newer paid round
- unranked = bool(journal) and state.best_considered_through < journal[-1].index
- best_changed = False
- # an empty journal used to have its own arm; it is subsumed — with journal == []
- # this condition is always true (best_invalid when a best is set, `best is None`
- # otherwise) and replay_best_round([]) is None, which clears the best the same
- # way. The only divergence, blanking a stale best_commit while best_round is
- # None, is unobservable: no reader touches best_commit without best_round.
- if stale or best_invalid or unranked or best is None:
- pick = replay_best_round(journal)
- if pick is None:
- best_changed = best is not None
- state.best_round, state.best_commit, state.best_score = None, "", None
- elif pick != best or journal[pick].commit != state.best_commit:
- state.update_best(pick, journal[pick].commit, journal[pick].score)
- best_changed = True
- if journal:
- state.best_considered_through = journal[-1].index # every journal round is now ranked
+ restored = bool(journal) and self._needs_restore(ws, journal[-1].commit)
+ if restored:
+ ws.restore(journal[-1].commit)
+ ws.commit(f"resume from the last round r{journal[-1].index:02d}")
state.save(ws)
events.emit("resume.reconciled", rounds=len(journal), dropped=dropped, state_was_stale=stale,
- unranked_rounds=unranked, best_round=state.best_round, best_score=state.best_score,
- best_changed=best_changed, spec_fingerprint=fp)
+ restored_last_round=journal[-1].index if restored else None, spec_fingerprint=fp)
return journal
@@ -560,7 +540,7 @@ class BaseTrack:
transport_retried: set[int] = set()
def _stop(reason: RunStatus) -> RunStatus:
- ctx.events.emit("stop", reason=reason.value, rounds=len(rounds), best=ctx.state.best_round)
+ ctx.events.emit("stop", reason=reason.value, rounds=len(rounds))
return reason
while True:
@@ -582,7 +562,7 @@ class BaseTrack:
rejudged.add(last.index)
prev_j = rounds[-2].judgment if len(rounds) > 1 else None
if rejudge_round(ctx, pipeline, last, previous=prev_j):
- self._promote_best(ctx, rounds, last.index)
+ self._save_budget(ctx)
continue # plan the next round from the recovered verdict
tasks, instructions = self.refine_tasks(ctx, last)
kind = "refine"
@@ -627,23 +607,10 @@ class BaseTrack:
ctx.events.emit("round.no_change", round=index, detail=str(e)[:500])
return _stop(RunStatus.NO_CHANGE)
rounds.append(rec)
- # ONE save, AFTER the best is chosen — never a durable "round done" with a
- # not-yet-updated best. Saving the round first (an earlier attempt at
- # shrinking the paid-pairwise crash window) made a crash in that window look
- # SELF-CONSISTENT to reconcile_resume — journal and completed_rounds agreed,
- # so it kept the stale best and delivered the worse round (r0=0.5 kept over
- # r1=0.7). Crashing before this save leaves state behind the journal, which
- # reconcile detects (stale) and repairs by re-ranking; that is the safe side.
+ # a crash before this save leaves the state behind the journal, which
+ # reconcile_resume detects (stale) and repairs from the journal
ctx.state.mark_round_done(index, rec.commit)
- self._promote_best(ctx, rounds, index)
-
- def _promote_best(self, ctx: RunContext, rounds: list[RoundRecord], index: int) -> None:
- """Rank round ``index`` in (pairwise tie-break included), promote the best, save."""
- best = choose_best_round(ctx, rounds, index)
- ctx.state.best_considered_through = index # this round HAS been ranked
- if best is not None and ctx.state.update_best(best, rounds[best].commit, rounds[best].score):
- ctx.events.emit("best.updated", round=best, score=rounds[best].score)
- self._save_budget(ctx)
+ self._save_budget(ctx)
def _salvage_baseline(self, ctx: RunContext, rounds: list[RoundRecord]) -> None:
"""A stage tripped the budget BEFORE round 0 ever ran (the greenhouse scene:
@@ -673,7 +640,7 @@ class BaseTrack:
return
rounds.append(rec)
ctx.state.mark_round_done(0, rec.commit)
- self._promote_best(ctx, rounds, 0)
+ self._save_budget(ctx)
def _save_budget(self, ctx: RunContext) -> None:
snap = ctx.budget.snapshot()
@@ -682,21 +649,25 @@ class BaseTrack:
# ------------------------------------------------------------------ finalise
def finalise(self, ctx: RunContext, rounds: list[RoundRecord], status: RunStatus, *, error: str = "") -> RunRecord:
+ """Write the record. The workspace ENDS AT THE LAST ROUND: a round the budget or a crash
+ cut mid-way can leave src/ past it, so it is put back and rebuilt — the only restore a
+ run does (every round's own build is kept under artifacts/rNN/, and choosing a round
+ to hand over is ``codeverse3d.addons.select``'s job, after the run)."""
self._save_budget(ctx) # aborted-round / stage charges must survive for resume
- best = ctx.state.best_round
+ last = rounds[-1] if rounds else None
rebuild_err = ""
rebuild_ok = False
- if best is not None and best < len(rounds) and rounds[best].commit and self._needs_restore(ctx, rounds[best].commit):
- ctx.ws.restore(rounds[best].commit)
- ctx.ws.commit(f"restore best round r{best:02d}")
+ if last is not None and last.commit and self._needs_restore(ctx.ws, last.commit):
+ ctx.ws.restore(last.commit)
+ ctx.ws.commit(f"back to the last round r{last.index:02d}")
try:
build = ctx.runtime.build(ctx.ws, timeout_s=ctx.settings.limits.build_timeout_s)
- ctx.events.emit("finalise.rebuild", round=best, ok=build.ok)
+ ctx.events.emit("finalise.rebuild", round=last.index, ok=build.ok)
if not build.ok: # the runtime invalidated the canonical artifact FIRST — it is gone
rebuild_err = f"{build.error_type or 'BuildFailed'}: {build.error_message}"[:300]
else:
rebuild_ok = True
- except Exception as e: # noqa: BLE001 — the best round already built once; report, don't fail
+ except Exception as e: # noqa: BLE001 — the round already built once; report, don't fail
rebuild_err = f"{type(e).__name__}: {e}"[:300]
ctx.events.emit("finalise.rebuild_failed", error=rebuild_err)
if rebuild_err:
@@ -712,28 +683,26 @@ class BaseTrack:
self._save_budget(ctx) # saves state too
rec = self._record(ctx, rounds, status, error=error)
self.services.finalize_record(ctx.ws, rec)
- ctx.events.emit("run.done", status=status.value, best_round=rec.best_round,
- final_score=rec.final_score, cost_usd=round(rec.total_usage.cost_usd, 4))
+ ctx.events.emit("run.done", status=status.value, rounds=len(rounds),
+ last_score=last.score if last is not None else None,
+ cost_usd=round(rec.total_usage.cost_usd, 4))
return rec
@staticmethod
- def _needs_restore(ctx: RunContext, best_commit: str) -> bool:
- """src/ != best commit: HEAD moved on, OR an aborted round dirtied the tree
+ def _needs_restore(ws: Workspace, commit: str) -> bool:
+ """src/ != ``commit``: HEAD moved on, OR an aborted round dirtied the tree
without committing (budget stop mid-generation) — HEAD alone cannot see that."""
- if ctx.ws.head() != best_commit:
+ if ws.head() != commit:
return True
try:
- return bool(ctx.ws.changed_files())
+ return bool(ws.changed_files())
except Exception as e: # noqa: BLE001 — a git hiccup must not block finalise
log.warning("changed_files failed in finalise: %s", e)
return False
def _record(self, ctx: RunContext, rounds: list[RoundRecord], status: RunStatus, *, error: str) -> RunRecord:
- best = ctx.state.best_round
- baseline = rounds[0].score if rounds else None
- final = rounds[best].score if best is not None and best < len(rounds) else None
total = sum_usage(rounds)
- # planner, aborted rounds, retried sessions, the texture pass: all in the guard,
+ # planner, aborted rounds, retried sessions: all in the guard,
# none of them in ``rounds`` — the guard is the honest total (docs/COST.md §6).
if ctx.budget.spent.cost_usd > total.cost_usd:
total = ctx.budget.spent
@@ -763,8 +732,7 @@ class BaseTrack:
extra["aborted_rounds"] = prior_aborted + [a for a in ctx.extra["aborted_rounds"]
if a not in prior_aborted]
return RunRecord(
- spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, best_round=best,
- baseline_score=baseline, final_score=final, total_usage=total,
+ spec=ctx.spec, plan=ctx.plan, workspace=str(ctx.ws.root), status=status, rounds=rounds, total_usage=total,
environment={"python": platform.python_version(), "host": platform.node(), "track": self.track.value,
"language": ctx.language.value, "generator": ctx.agent_id},
prompt_hashes={**prior_hashes, **dict(ctx.prompt_hashes)}, started_at=ctx.state.started_at,
diff --git a/harness/codeverse3d/tracks/steps.py b/harness/codeverse3d/tracks/steps.py
index 536fffe7..5b496c9f 100644
--- a/harness/codeverse3d/tracks/steps.py
+++ b/harness/codeverse3d/tracks/steps.py
@@ -109,7 +109,7 @@ def looks_transport(msg: str) -> bool:
An agent that ran fine and chose to change nothing reports ``rc=0;
response=ok`` or plain task notes — none of the markers. A marker therefore
separates "the session never really happened" (worth exactly one retry) from
- "the agent declined" (a plateau verdict to respect).
+ "the agent declined" (a ``no_change`` stop to respect).
"""
low = msg.lower()
return any(m in low for m in _TRANSPORT_MARKS)
@@ -177,7 +177,7 @@ def _run_phase(ctx: RunContext, tasks: Sequence[GenerationTask]) -> list[Generat
return generate_for(ctx, task)
# fan_out also for ONE task: a crashing generator (503 storm, parse error) becomes a failed
- # result → RoundFailed (refine rounds treat that as a plateau) instead of killing the run.
+ # result → RoundFailed (a refine round then stops the run as no_change) instead of killing it.
results = fan_out(list(tasks), _one, max_workers=ctx.settings.limits.max_parallel_agents,
label="generate", item_name=lambda t: t.label)
out: list[GenerationResult] = []
@@ -298,8 +298,8 @@ def _run_round(
# refine round built in 1.2 s, then render_glb.mjs hit its 330 s timeout with
# three articulated runs and two readouts sharing the browser) used to raise
# out of the round and record the whole run `failed` -- with a judged round 0
- # already on disk. The round is kept: build, measurement and gates stand, the
- # judge is skipped (no renders), and the loop delivers the best round so far.
+ # already on disk. The round is kept: build, measurement and gates stand and
+ # the judge is skipped (no renders), so a pick passes over it.
rec.renders = None
notes.append(f"render failed: {str(e)[:200]}")
ctx.events.emit("render.failed", round=index, error=str(e)[:400])
@@ -309,9 +309,7 @@ def _run_round(
n_err = sum(len(g.errors) for g in gates)
ctx.events.emit("gates.done", round=index, n_gates=len(gates), n_errors=n_err,
tri_count=rec.measurement.tri_count if rec.measurement else None)
- # a candidate's verdict is the ONLY thing that picks the code r00 starts from, so it is
- # bought even past the ceiling — unlike a refine verdict, which could promote nothing
- skip = skip_judge_reason(ctx, renders=rec.renders, ignore_budget=kind == "candidate")
+ skip = skip_judge_reason(ctx, renders=rec.renders)
if skip:
notes.append(f"judge skipped ({skip})")
ctx.events.emit("judge.skipped", round=index, reason=skip)
@@ -327,7 +325,7 @@ def _run_round(
if rec.judgment is not None:
# add, never charge: the verdict exists and is paid for — raising here
# would drop a fully judged round before it is committed/recorded
- # (the loop stops at its next budget_ok check instead, AFTER best promotion).
+ # (the loop stops at its next budget check instead).
ctx.budget.add(rec.judgment.usage, stage="judge")
else:
notes.append(f"build failed: {outcome.build.error_type}: {outcome.build.error_message[:200]}")
@@ -351,7 +349,7 @@ def _run_round(
# ----------------------------------------------------------------------------- cost of a round
-def skip_judge_reason(ctx: RunContext, *, renders: RenderSet | None, ignore_budget: bool = False) -> str:
+def skip_judge_reason(ctx: RunContext, *, renders: RenderSet | None) -> str:
"""Why this round must NOT be judged (``""`` = judge it).
Only states in which the verdict is never bought at all — a skip that the
@@ -359,12 +357,9 @@ def skip_judge_reason(ctx: RunContext, *, renders: RenderSet | None, ignore_budg
``judge.retry`` plus a lost score:
* ``no judge configured`` / ``no renders`` — there is nothing to buy.
- * ``budget already exceeded`` — the loop's next ``budget_ok`` check ends the
- run, so this verdict could not promote anything. Measured: 2 verdicts /
- $0.09 in the audit were bought past the run's wall clock (docs/COST.md §5).
- ``ignore_budget`` exempts the best-of-N candidates: their verdict picks the
- code r00 starts from, so skipping it wastes the N generations already paid for.
+ A round that finished past the wall clock is still judged (2026-09-22): its verdict is
+ what lets ``addons.select`` pick it, and the generation it scores is already paid for.
A broken build never gets here (``run_round`` judges only when the build is
ok) and a round in which nothing changed never gets here either
(``run_generation_tasks`` raises :class:`RoundFailed` first)."""
@@ -372,8 +367,6 @@ def skip_judge_reason(ctx: RunContext, *, renders: RenderSet | None, ignore_budg
return "no judge configured"
if renders is None or not renders.views:
return "no renders"
- if not ctx.budget.ok() and not ignore_budget:
- return "budget already exceeded"
return ""
@@ -497,7 +490,7 @@ def _judge(ctx: RunContext, pipeline: RoundPipeline, index: int, build: BuildRes
ctx.ws.write_json(ctx.ws.judge_path(index), judgment)
if is_degraded(judgment):
# a glitch, never a score: keep the raw verdict on disk, pay for it, but do not
- # let 0.0 poison plateau/best/refine (judges/rubrics.degraded_judgment contract)
+ # let 0.0 poison a pick or a refine plan (judges/rubrics.degraded_judgment contract)
ctx.budget.add(judgment.usage, stage="judge")
ctx.events.emit("judge.degraded", round=index, error=judgment.summary[:300],
cost_usd=round(judgment.usage.cost_usd, 4))
diff --git a/harness/codeverse3d/workspace.py b/harness/codeverse3d/workspace.py
index 7dbc1f6c..0bd7b9f1 100644
--- a/harness/codeverse3d/workspace.py
+++ b/harness/codeverse3d/workspace.py
@@ -183,7 +183,7 @@ class Workspace:
@property
def deliverable(self) -> Path:
- """(a) the hand-over folder: best-round code + canonical artifact + sheet + captions."""
+ """(a) the hand-over folder: one round's code + artifact + sheet + captions (``3dcode pick``)."""
return self.root / DELIVERABLE_DIR
@property
diff --git a/harness/tests/core/test_contract_additions.py b/harness/tests/core/test_contract_additions.py
index 6a4720c5..205e073b 100644
--- a/harness/tests/core/test_contract_additions.py
+++ b/harness/tests/core/test_contract_additions.py
@@ -110,8 +110,7 @@ def test_records_written_before_the_2026_08_30_field_retirements_still_load():
job = AgentJob.model_validate({"workspace": "/w", "prompt": "p",
"images": [{"path": "renders/sheet.png", "label": "sheet"}]})
assert job.prompt == "p" and not hasattr(job, "images")
- # and the field this batch ADDED round-trips through the round journal
+ # the in-run pairwise note this batch added went with the in-run best round (2026-09-22)
rec = RoundRecord.model_validate({"index": 1, "kind": "refine",
"pairwise": {"a": "r00", "b": "r01", "winner": "a", "confidence": 0.9}})
- assert rec.pairwise is not None and rec.pairwise.accepted is False
- assert RoundRecord(index=0, kind="baseline").pairwise is None
+ assert rec.index == 1 and not hasattr(rec, "pairwise")
diff --git a/harness/tests/flywheel_cli/conftest.py b/harness/tests/flywheel_cli/conftest.py
index c5fcc4ea..0bd169f1 100644
--- a/harness/tests/flywheel_cli/conftest.py
+++ b/harness/tests/flywheel_cli/conftest.py
@@ -1,5 +1,6 @@
-"""Shared fixtures: a synthetic run (git repo with two commits, tiny PNGs,
-record.json with two judged rounds) — no models, Blender or node required."""
+"""Shared fixtures: a synthetic run (git repo with two commits, tiny PNGs, each round's
+kept GLB under artifacts/rNN/, record.json with two judged rounds) — no models, Blender or
+node required."""
from __future__ import annotations
@@ -78,6 +79,9 @@ def make_fake_run(
(ws.src / "parts").mkdir(exist_ok=True)
(ws.src / "parts" / "leg.py").write_text("LEG = 0.04\n")
commit = c0 if i == 0 else ws.commit(f"round {i}")
+ kept = ws.round_artifacts(i)
+ kept.mkdir(parents=True, exist_ok=True)
+ (kept / "object.glb").write_bytes(b"glTF\x02\x00\x00\x00" + bytes([i]) * 16)
rd = ws.renders_dir(i)
views = [RenderView(name=n, path=str(tiny_png(rd / f"view_{n}.png"))) for n in ("front", "top")]
sheet = tiny_png(rd / "sheet.png", (0, 0, 200))
@@ -107,9 +111,8 @@ def make_fake_run(
index=len(rounds), kind="repair", commit=fixed, instructions=["fix the syntax error at line 2"],
build=BuildResult(ok=True, language=language.value),
))
- (ws.artifacts / "object.glb").write_bytes(b"glTF\x02\x00\x00\x00" + b"\0" * 16)
- rec = RunRecord(spec=spec, workspace=str(ws.root), status=RunStatus.MAX_ROUNDS,
- rounds=rounds, best_round=1, baseline_score=scores[0], final_score=scores[1],
+ (ws.artifacts / "object.glb").write_bytes(b"glTF\x02\x00\x00\x00" + bytes([len(scores) - 1]) * 16) # the last round
+ rec = RunRecord(spec=spec, workspace=str(ws.root), status=RunStatus.MAX_ROUNDS, rounds=rounds,
total_usage=Usage(cost_usd=0.06, input_tokens=2000, output_tokens=1000),
finished_at=datetime.now(UTC),
environment={"blender": "Blender 5.0.1"})
diff --git a/harness/tests/flywheel_cli/test_complexity_record.py b/harness/tests/flywheel_cli/test_complexity_record.py
index 978672f1..2a7f1fbd 100644
--- a/harness/tests/flywheel_cli/test_complexity_record.py
+++ b/harness/tests/flywheel_cli/test_complexity_record.py
@@ -41,7 +41,7 @@ def _round(index: int, score: float, cx: float | None) -> RoundRecord:
)
-def _record(rounds: list[RoundRecord], best: int | None = None) -> RunRecord:
+def _record(rounds: list[RoundRecord]) -> RunRecord:
bbox = BBox(center=(0.0, 0.0, 0.0), extents=(0.1, 0.1, 0.1))
plan = StaticPlan(object_name="Thing", summary="a thing",
overall_bbox=BBox(center=(0.0, 0.5, 0.0), extents=(1.0, 1.0, 1.0)),
@@ -50,15 +50,15 @@ def _record(rounds: list[RoundRecord], best: int | None = None) -> RunRecord:
return RunRecord(
spec=Spec(id="t", track=Track.STATIC_OBJECT, language=Language.BLENDER, prompt="a thing",
backends=Backends()),
- plan=plan, workspace="/tmp/x", status=RunStatus.MAX_ROUNDS, rounds=rounds, best_round=best,
+ plan=plan, workspace="/tmp/x", status=RunStatus.MAX_ROUNDS, rounds=rounds,
started_at=datetime.now(UTC),
)
-def test_record_carries_the_delivered_artifact_complexity() -> None:
- rec = fill_derived(_record([_round(0, 0.60, 0.42), _round(1, 0.80, 0.55)], best=1))
+def test_record_carries_the_last_rounds_complexity_and_the_trail() -> None:
+ rec = fill_derived(_record([_round(0, 0.80, 0.42), _round(1, 0.60, 0.55)]))
block = rec.extra["complexity"]
- assert block["index"] == 0.55 # the BEST round's build, not the last measured
+ assert block["index"] == 0.55 # the build the run ENDS at, whatever it scored
assert block["plan_parts"] == 4
assert block["parts_per_plan_part"] == pytest.approx(2.0)
assert block["by_round"] == [0.42, 0.55]
@@ -66,11 +66,11 @@ def test_record_carries_the_delivered_artifact_complexity() -> None:
def test_complexity_block_falls_back_to_the_last_measured_round() -> None:
- rec = _record([_round(0, 0.6, 0.42), _round(1, 0.5, None)], best=1)
+ rec = _record([_round(0, 0.6, 0.42), _round(1, 0.5, None)])
assert complexity_block(rec)["index"] == 0.42
def test_no_measurement_means_no_block() -> None:
- rec = fill_derived(_record([_round(0, 0.6, None)], best=0))
+ rec = fill_derived(_record([_round(0, 0.6, None)]))
assert "complexity" not in rec.extra
assert round_summary(rec.rounds[0])["complexity"] is None
diff --git a/harness/tests/flywheel_cli/test_flywheel.py b/harness/tests/flywheel_cli/test_flywheel.py
index cd286097..d4cdf79f 100644
--- a/harness/tests/flywheel_cli/test_flywheel.py
+++ b/harness/tests/flywheel_cli/test_flywheel.py
@@ -10,6 +10,7 @@ from pathlib import Path
import pytest
+from codeverse3d.addons import select
from codeverse3d.addons.dataset.export import export_samples
from codeverse3d.addons.dataset.index import build_index, summary
from codeverse3d.addons.dataset.pack import pack_samples, verify_locators
@@ -22,7 +23,6 @@ from codeverse3d.contracts.run import RunRecord
from codeverse3d.record import _git
from codeverse3d.record.record import (
RecordError,
- best_round_index,
finalize_record,
iter_runs,
load_record,
@@ -34,30 +34,18 @@ from codeverse3d.workspace import Workspace
def test_finalize_and_load_record(fake_run):
ws, rec = fake_run
- rec.best_round = None
- rec.final_score = None
rec.environment = {}
path = finalize_record(ws, rec)
assert path == ws.record_path and path.is_file()
loaded = load_record(ws)
- assert loaded.best_round == 1 and loaded.final_score == 0.80 and loaded.baseline_score == 0.55
+ stored = json.loads(path.read_text())
+ assert not {"best_round", "baseline_score", "final_score"} & stored.keys() # the run names no best (2026-09-22)
assert loaded.environment["python"] and loaded.environment["codeverse3d"]
assert "three" in loaded.environment and "node" in loaded.environment
assert len(loaded.extra["rounds_summary"]) == 2
assert loaded.extra["rounds_summary"][1]["score"] == 0.80
-def test_best_round_fallbacks(fake_run):
- ws, rec = fake_run
- rec.best_round = None
- assert best_round_index(rec) == 1
- for r in rec.rounds:
- r.judgment = None
- assert best_round_index(rec) == 1 # last built round
- rec.rounds = []
- assert best_round_index(rec) is None
-
-
def test_iter_runs_and_errors(runs_dir: Path):
(runs_dir / "broken").mkdir()
(runs_dir / "broken" / "record.json").write_text("{not json")
@@ -356,10 +344,10 @@ def test_degraded_round_is_not_a_zero_score(tmp_path: Path):
"judgment": _degrade(rec.rounds[1].judgment)})
last = rec.rounds[1].model_copy(update={"index": 2})
rec.rounds = [rec.rounds[0], mid, last]
- rec.best_round = None
assert effective_judgment(mid) is None and effective_score(mid) is None
assert round_summary(mid)["score"] is None and round_summary(mid)["judge_degraded"] is True
- assert best_round_index(rec) == 2 # 0.64 beats 0.62; the degraded 0.0 never competes
+ ws.write_json(ws.record_path, rec)
+ assert select.pick(ws.root) == 2 # 0.64 beats 0.62; the degraded 0.0 never competes
pairs = preference_pairs(ws, rec, min_delta=0.05)
# old behaviour: chosen=r2 rejected=r1(degraded) with delta 0.64 — a pure noise pair
assert pairs == []
@@ -367,15 +355,13 @@ def test_degraded_round_is_not_a_zero_score(tmp_path: Path):
assert len(pairs) == 1 and pairs[0]["rejected"]["round"] == 0 and pairs[0]["chosen"]["round"] == 2
-def test_degraded_best_round_exports_unscored(tmp_path: Path):
+def test_a_run_with_only_degraded_verdicts_exports_unscored(tmp_path: Path):
from tests.flywheel_cli.conftest import make_fake_run
ws, rec = make_fake_run(tmp_path / "runs", scores=(0.5, 0.9))
for r in rec.rounds:
if r.judgment is not None:
r.judgment = _degrade(r.judgment)
- rec.best_round = 1
- rec.baseline_score = rec.final_score = None
ws.write_json(ws.record_path, rec)
out = tmp_path / "ds"
rep = export_samples(ws.root.parent, out)
diff --git a/harness/tests/flywheel_cli/test_run_layout_cli.py b/harness/tests/flywheel_cli/test_run_layout_cli.py
index ff503ed0..4d852b59 100644
--- a/harness/tests/flywheel_cli/test_run_layout_cli.py
+++ b/harness/tests/flywheel_cli/test_run_layout_cli.py
@@ -32,16 +32,19 @@ runner = CliRunner()
def strip_layout(ws: Workspace) -> Workspace:
- """Turn a freshly created workspace back into a pre-2026-08-23 run directory."""
+ """Turn a freshly created workspace back into a pre-2026-08-23 run directory: no
+ telemetry/ or deliverable/, no per-round builds — only the canonical one, of the best
+ round its record.json names."""
for name, _ in LAYOUT_ALIASES:
p = ws.root / name
if p.is_symlink():
p.unlink()
- for d in (ws.deliverable, ws.telemetry):
+ for d in (ws.deliverable, ws.telemetry, *ws.artifacts.glob("r[0-9][0-9]")):
shutil.rmtree(d, ignore_errors=True)
rec = json.loads(ws.record_path.read_text())
rec.pop("telemetry", None)
rec.pop("deliverable", None)
+ rec["best_round"] = max((r for r in rec["rounds"] if r.get("judgment")), key=lambda r: r["judgment"]["overall"])["index"]
ws.record_path.write_text(json.dumps(rec, indent=2))
return ws
@@ -158,15 +161,25 @@ def test_the_settings_snapshot_records_the_temperature_the_track_planned_at(tmp_
def test_rejected_texture_pass_is_not_delivered_or_linked(fake_run):
"""Belt and braces: a stray canonical object_textured.glb from a rejected pass
is skipped by the deliverable AND by the gallery links (both gate on shipped)."""
+ from codeverse3d.texturing.run import report_path
+
ws, rec = fake_run
(ws.artifacts / "object_textured.glb").write_bytes(b"glTF\x02" + b"\0" * 16)
- rec.extra["texturing"] = {"shipped": False, "glb_textured": "artifacts/object_textured.glb"}
+ report_path(ws).parent.mkdir(parents=True, exist_ok=True)
+
+ def texturing(shipped: bool) -> None: # the pass report names the round GLB it started from
+ rec.extra["texturing"] = {"shipped": shipped, "glb_textured": "artifacts/object_textured.glb"}
+ ws.write_json(report_path(ws), {**rec.extra["texturing"], "glb_in": "artifacts/r01/object.glb"})
+
+ texturing(False)
build_deliverable(ws, rec, 1)
assert not (ws.deliverable / "object_textured.glb").exists()
entry = entry_from_record("runs", ws, rec)
assert "textured glb" not in {ln.label for ln in entry.links}
- # ... and a SHIPPED pass is delivered and linked
- rec.extra["texturing"] = {"shipped": True, "glb_textured": "artifacts/object_textured.glb"}
+ # ... and a SHIPPED pass is delivered and linked — with the round it textured, only
+ texturing(True)
+ build_deliverable(ws, rec, 0)
+ assert not (ws.deliverable / "object_textured.glb").exists()
build_deliverable(ws, rec, 1)
assert (ws.deliverable / "object_textured.glb").is_file()
entry = entry_from_record("runs", ws, rec)
diff --git a/harness/tests/gallery/test_index.py b/harness/tests/gallery/test_index.py
index c53a7cde..91aa2e3f 100644
--- a/harness/tests/gallery/test_index.py
+++ b/harness/tests/gallery/test_index.py
@@ -48,8 +48,6 @@ def test_a_card_links_the_picked_rounds_own_glb(tmp_path: Path):
from tests.flywheel_cli.conftest import make_fake_run
ws, _ = make_fake_run(tmp_path / "runs", "kept_rounds", scores=(0.9, 0.6)) # the pick is r00
- (ws.round_artifacts(0)).mkdir(parents=True)
- (ws.round_artifacts(0) / "object.glb").write_bytes(b"glTF r00")
links = {ln.label: ln.rel for ln in entry_for_dir("runs", ws.root).links}
assert links["glb"] == "artifacts/r00/object.glb"
select.package(ws.root, 0)
diff --git a/harness/tests/gallery/test_page.py b/harness/tests/gallery/test_page.py
index ec846172..d46f58f3 100644
--- a/harness/tests/gallery/test_page.py
+++ b/harness/tests/gallery/test_page.py
@@ -127,7 +127,7 @@ def test_detail_links_are_offered_only_when_the_target_has_them(gallery_tree: di
assert "/run/runs/wooden_chair_ab12cd34" in markup
assert "/file/runs/wooden_chair_ab12cd34/artifacts/renders/r01/sheet.png" in markup
assert "/code/runs/wooden_chair_ab12cd34/src" in markup
- assert "/viewer/runs/wooden_chair_ab12cd34/artifacts/object.glb" in markup
+ assert "/viewer/runs/wooden_chair_ab12cd34/artifacts/r01/object.glb" in markup # the picked round's
# the static form never advertises server-only routes
static = render_static(index, embed=False)
assert "/run/runs/" not in static and "/viewer/" not in static
diff --git a/harness/tests/judges/test_calibration.py b/harness/tests/judges/test_calibration.py
index 4b689cab..92eb28ed 100644
--- a/harness/tests/judges/test_calibration.py
+++ b/harness/tests/judges/test_calibration.py
@@ -140,5 +140,5 @@ def test_calibration_judges_what_the_in_run_judge_saw(tmp_path, monkeypatch):
assert set(made) == {"_Ref"}
r1 = next(inp for inp in seen if inp.round_index == 1)
assert r1.previous is not None and r1.previous.overall == 0.55
- assert r1.geometry_views is not None and r1.glb_path == str(ws.artifacts / "object.glb")
+ assert r1.geometry_views is not None and r1.glb_path == str(ws.round_artifacts(1) / "object.glb") # the round's own
assert next(row for row in table.rows if row.round == 1).geometry_views
diff --git a/harness/tests/orchestrator_tracks/fakes.py b/harness/tests/orchestrator_tracks/fakes.py
index dccf0826..4ec92bfc 100644
--- a/harness/tests/orchestrator_tracks/fakes.py
+++ b/harness/tests/orchestrator_tracks/fakes.py
@@ -236,23 +236,6 @@ class FakeJudge:
improvement_plan=plan, acceptance_results=acc, usage=Usage(backend="fake", cost_usd=self.cost))
-class FakePairwise:
- """Scripted pairwise verdicts: ``verdicts`` is consumed in order (last one repeats)."""
-
- def __init__(self, verdicts: Sequence[tuple[str, float]] = (("tie", 0.3),), cost: float = 0.004):
- self.verdicts = list(verdicts)
- self.cost = cost
- self.calls: list[tuple[Any, Any, Any, str]] = []
-
- def compare(self, spec: Any, renders_a: Any, renders_b: Any, *, rubric: str = "static_object_v1") -> Any:
- from codeverse3d.judges.pairwise import PairwiseResult
-
- i = min(len(self.calls), len(self.verdicts) - 1)
- self.calls.append((spec, renders_a, renders_b, rubric))
- winner, conf = self.verdicts[i]
- return PairwiseResult(winner=winner, confidence=conf, reasons=[f"scripted verdict {i}"], usage=Usage(backend="fake", cost_usd=self.cost))
-
-
# ----------------------------------------------------------------------------- services
def _png(path: Path, size: int = 32) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
diff --git a/harness/tests/orchestrator_tracks/test_budget_snapshot.py b/harness/tests/orchestrator_tracks/test_budget_snapshot.py
index f9dc3c08..f51586fc 100644
--- a/harness/tests/orchestrator_tracks/test_budget_snapshot.py
+++ b/harness/tests/orchestrator_tracks/test_budget_snapshot.py
@@ -245,7 +245,6 @@ def test_the_adopted_best_of_n_winner_survives_a_boundary_budget_stop(tmp_path,
rec = track.run(make_spec(max_rounds=0), ws)
assert rec.status is RunStatus.BUDGET
assert len(rec.rounds) == 1 and rec.rounds[0].score == pytest.approx(0.6)
- assert rec.best_round == 0 and rec.final_score == pytest.approx(0.6)
ev = [e["event"] for e in EventLog(ws.events_path).read()]
assert "budget.salvage" in ev and "budget.salvage_skipped" not in ev
diff --git a/harness/tests/orchestrator_tracks/test_cost_latency.py b/harness/tests/orchestrator_tracks/test_cost_latency.py
index 405a1c0e..c2e0ee4e 100644
--- a/harness/tests/orchestrator_tracks/test_cost_latency.py
+++ b/harness/tests/orchestrator_tracks/test_cost_latency.py
@@ -325,12 +325,11 @@ def test_a_budget_stop_before_round_zero_still_delivers_a_judged_round(tmp_path,
rec = track.run(spec, ws)
assert rec.status is RunStatus.BUDGET and rec.extra["stop_reason"] == "budget"
# …and, unlike the greenhouse run, it has a score
- assert rec.best_round == 0 and rec.final_score == pytest.approx(0.58)
- assert len(rec.rounds) == 1 and rec.rounds[0].renders is not None
+ assert len(rec.rounds) == 1 and rec.rounds[0].renders is not None and rec.rounds[0].score == pytest.approx(0.58)
assert "salvaged" in rec.rounds[0].notes
ev = [json.loads(x) for x in (ws.root / "events.jsonl").read_text().splitlines() if x.strip()]
names = [e["event"] for e in ev]
- assert "budget.salvage" in names and names.count("best.updated") == 1
+ assert "budget.salvage" in names
salvage = next(e for e in ev if e["event"] == "budget.salvage")
assert salvage["grace_minutes"] > 0
@@ -440,7 +439,7 @@ def test_an_imperfect_asset_stays_available_but_a_broken_one_does_not(tmp_path,
assert not bad.ok and bad.fatal and "missing export" in bad.errors[0]
-def test_a_scene_round_judged_at_the_ceiling_is_still_promoted(tmp_path, settings):
+def test_a_scene_round_judged_at_the_ceiling_keeps_its_verdict(tmp_path, settings):
plan = _threejs_scene_plan()
spec = make_spec(Track.SCENE, Language.SCENE_THREEJS, max_rounds=2)
ws = Workspace(tmp_path / "runs" / "ceiling")
@@ -450,10 +449,8 @@ def test_a_scene_round_judged_at_the_ceiling_is_still_promoted(tmp_path, setting
runtime=FakeRuntime(Language.SCENE_THREEJS))
with fake_clock():
rec = track.run(spec, ws)
- assert rec.status is RunStatus.BUDGET and rec.best_round == 0 and rec.final_score == pytest.approx(0.61)
- assert len(rec.rounds) == 1 and rec.rounds[0].judgment is not None
+ assert rec.status is RunStatus.BUDGET and len(rec.rounds) == 1 and rec.rounds[0].score == pytest.approx(0.61)
names = [json.loads(x)["event"] for x in (ws.root / "events.jsonl").read_text().splitlines() if x.strip()]
- assert names.index("best.updated") < names.index("stop")
assert "budget.salvage" not in names, "round 0 already exists: nothing to salvage"
assert len(judge.calls) == 1
diff --git a/harness/tests/orchestrator_tracks/test_fix_batch2.py b/harness/tests/orchestrator_tracks/test_fix_batch2.py
index a91ffb37..e9a3d84b 100644
--- a/harness/tests/orchestrator_tracks/test_fix_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_fix_batch2.py
@@ -58,7 +58,7 @@ class ScriptedJudge(FakeJudge):
return out
-# --------------------------------------------------------------------- finding: judge charge after persist (+ best BEFORE budget check)
+# --------------------------------------------------------------------- finding: judge charge after persist
def test_judged_round_survives_budget_ceiling_crossed_by_judge(tmp_path, chair_plan, settings):
"""steps.py:148 — the judge's own charge used to raise BudgetExceeded BEFORE the round
was committed/recorded, throwing away a complete judged round (scene r2 case)."""
@@ -71,19 +71,16 @@ def test_judged_round_survives_budget_ceiling_crossed_by_judge(tmp_path, chair_p
with fake_clock():
rec = track.run(spec, ws)
assert rec.status is RunStatus.BUDGET and rec.extra["stop_reason"] == "budget"
- assert len(rec.rounds) == 1 and rec.rounds[0].judgment is not None
- assert rec.best_round == 0 and rec.final_score == pytest.approx(0.55) # promoted BEFORE the budget stop
+ assert len(rec.rounds) == 1 and rec.rounds[0].score == pytest.approx(0.55)
assert (ws.root / "rounds" / "r00.json").is_file() and ws.judge_path(0).is_file()
- kinds = [e["event"] for e in EventLog(ws.events_path).read()]
- assert "best.updated" in kinds and kinds.index("best.updated") < kinds.index("stop")
# the judge cost is still in the totals
assert rec.total_usage.cost_usd > 0.05
-# --------------------------------------------------------------------- finding: finalise with a dirty tree at the best commit
-def test_finalise_restores_best_when_aborted_round_dirtied_src(tmp_path, chair_plan, settings):
+# --------------------------------------------------------------------- finding: finalise with a dirty tree at the last commit
+def test_finalise_restores_the_last_round_when_an_aborted_round_dirtied_src(tmp_path, chair_plan, settings):
"""lifecycle.py:263 — a refine round that wrote files but died on the generation charge
- left HEAD at the best commit with foreign src/: the run then shipped unjudged code."""
+ left HEAD at the last round's commit with foreign src/: the run then ended on unjudged code."""
spec = make_spec(max_rounds=3)
ws = Workspace(tmp_path / "runs" / "r")
# planner 0.002 + baseline agent 0.01 + judge 0.003 = 0.015 < 0.02; the refine agent's
@@ -93,7 +90,7 @@ def test_finalise_restores_best_when_aborted_round_dirtied_src(tmp_path, chair_p
runtime=FakeRuntime(Language.THREEJS))
with fake_clock():
rec = track.run(spec, ws)
- assert rec.status is RunStatus.BUDGET and rec.best_round == 0 and len(rec.rounds) == 1
+ assert rec.status is RunStatus.BUDGET and len(rec.rounds) == 1
text = (ws.src / "object.js").read_text()
assert "baseline r0" in text and "refine" not in text # restored, not the aborted round's edits
assert ws.changed_files() == [] # tree clean at the delivered commit
@@ -108,39 +105,45 @@ def test_finalise_restores_best_when_aborted_round_dirtied_src(tmp_path, chair_p
# --------------------------------------------------------------------- finding: finalise rebuild fails after invalidation
def test_a_failed_finalise_rebuild_cannot_finalize_silently(tmp_path, chair_plan, settings, monkeypatch):
"""Every real runtime invalidates the canonical artifact FIRST in build(), so a
- failed rebuild of the restored best round leaves object.glb MISSING — yet the run
- used to finalize with final_score set, ``record.error == ""`` and a deliverable
- with no model file. The earned status + judge scores are kept; the error and the
- ``finalise_rebuild_failed`` flag must say what happened."""
- import subprocess
-
- class RebuildFailsRuntime(FakeRuntime):
- def build(self, ws, *, timeout_s=None):
- msg = subprocess.run(["git", "-C", str(ws.root), "log", "-1", "--format=%s"],
- capture_output=True, text=True).stdout.strip()
- if msg.startswith("restore best round"):
- ws.stage_artifacts("object.glb").invalidate() # what the real runtimes do first
- return BuildResult(ok=False, language=self.language.value, error_type="Timeout",
- error_message="runtime crashed on rebuild", error_file="src/object.js")
- return super().build(ws, timeout_s=timeout_s)
-
- spec = make_spec(max_rounds=2)
+ failed rebuild of the restored last round leaves object.glb MISSING — yet the run
+ used to finalize as if nothing happened. The earned status + judge scores are kept;
+ the ``finalise_rebuild_failed`` flag says what happened, and the round's own kept
+ build (``artifacts/r00/``) is untouched."""
+ spec = make_spec(max_rounds=3)
ws = Workspace(tmp_path / "runs" / "r")
- track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.7, 0.5), targets=("Seat",)),
- agent=FakeAgent(_writer), planner_model=_planner(chair_plan.model_dump(mode="json")),
- settings=settings, runtime=RebuildFailsRuntime(Language.THREEJS))
- rec = track.run(spec, ws)
- assert rec.status is RunStatus.MAX_ROUNDS and rec.best_round == 0 # the earned status stays
- assert rec.final_score == pytest.approx(0.7) # ...and so do the paid scores
- assert rec.error.startswith("finalise rebuild failed: Timeout")
+ # the refine round's agent crosses the wall clock after its files hit the disk: finalise
+ # puts src/ back on r00 and rebuilds it — and that rebuild fails
+ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.7,), targets=("Seat",)),
+ agent=FakeAgent(_writer, minutes=6.0), planner_model=_planner(chair_plan.model_dump(mode="json")),
+ settings=settings, runtime=_RebuildFails(Language.THREEJS))
+ with fake_clock():
+ rec = track.run(spec, ws)
+ assert rec.status is RunStatus.BUDGET and rec.rounds[0].score == pytest.approx(0.7) # the earned status + score stay
assert "runtime crashed on rebuild" in rec.extra["finalise_rebuild_failed"]
- assert not (ws.artifacts / "object.glb").is_file()
+ assert not (ws.artifacts / "object.glb").is_file() and (ws.round_artifacts(0) / "object.glb").is_file()
on_disk = json.loads(ws.record_path.read_text())
- assert on_disk["error"].startswith("finalise rebuild failed") and on_disk["status"] == "max_rounds"
+ assert on_disk["extra"]["finalise_rebuild_failed"] and on_disk["status"] == "budget"
ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "finalise.rebuild"]
assert ev and ev[-1]["ok"] is False
+class _RebuildFails(FakeRuntime):
+ """Fails the finalise rebuild (the build right after a "back to the last round" commit)
+ ``fails`` times, invalidating the canonical GLB first as the real runtimes do."""
+
+ def __init__(self, language, *, fails: int = 1_000):
+ super().__init__(language)
+ self.fails = fails
+
+ def build(self, ws, *, timeout_s=None):
+ if ws._git("log", "-1", "--format=%s").stdout.startswith("back to the last round") and self.fails: # noqa: SLF001
+ self.fails -= 1
+ ws.stage_artifacts("object.glb").invalidate()
+ return BuildResult(ok=False, language=self.language.value, error_type="Timeout",
+ error_message="runtime crashed on rebuild", error_file="src/object.js")
+ return super().build(ws, timeout_s=timeout_s)
+
+
# --------------------------------------------------------------------- finding: spent usage persisted on crash paths
def test_spent_usage_saved_when_a_round_crashes(tmp_path, chair_plan, settings):
class BoomServices(FakeServices):
@@ -178,7 +181,7 @@ def test_degraded_verdict_never_scores_and_run_stops_as_judge_unavailable(tmp_pa
runtime=FakeRuntime(Language.THREEJS))
rec = track.run(spec, ws)
assert rec.rounds[1].judgment is None and rec.rounds[1].score is None # never 0.0
- assert rec.best_round == 0 and rec.final_score == pytest.approx(0.65)
+ assert rec.rounds[0].score == pytest.approx(0.65)
assert rec.extra["stop_reason"] == "judge_unavailable" and rec.status is RunStatus.JUDGE_UNAVAILABLE
kinds = [e["event"] for e in EventLog(ws.events_path).read()]
assert "judge.degraded" in kinds and "judge.retry" in kinds
@@ -347,35 +350,21 @@ def test_a_successful_re_finalise_clears_the_stale_rebuild_failed_flag(tmp_path,
once recorded ``finalise_rebuild_failed`` and is later resumed to a SUCCESSFUL
rebuild must clear the flag explicitly (a no-rebuild resume keeps it: the
artifact may still be the missing one)."""
- import subprocess
-
- fail_once = {"armed": True}
-
- class FlakyRebuildRuntime(FakeRuntime):
- def build(self, ws, *, timeout_s=None):
- msg = subprocess.run(["git", "-C", str(ws.root), "log", "-1", "--format=%s"],
- capture_output=True, text=True).stdout.strip()
- if msg.startswith("restore best round") and fail_once["armed"]:
- fail_once["armed"] = False
- ws.stage_artifacts("object.glb").invalidate()
- return BuildResult(ok=False, language=self.language.value, error_type="Timeout",
- error_message="runtime crashed on rebuild", error_file="src/object.js")
- return super().build(ws, timeout_s=timeout_s)
-
- spec = make_spec(max_rounds=2)
+ spec = make_spec(max_rounds=3)
ws = Workspace(tmp_path / "runs" / "r")
+ runtime = _RebuildFails(Language.THREEJS, fails=1)
def track():
- return StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.7, 0.5), targets=("Seat",)),
- agent=FakeAgent(_writer), planner_model=_planner(chair_plan.model_dump(mode="json")),
- settings=settings, runtime=FlakyRebuildRuntime(Language.THREEJS))
+ return StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.7,), targets=("Seat",)),
+ agent=FakeAgent(_writer, minutes=6.0), planner_model=_planner(chair_plan.model_dump(mode="json")),
+ settings=settings, runtime=runtime)
- rec1 = track().run(spec, ws)
+ with fake_clock():
+ rec1 = track().run(spec, ws)
assert "runtime crashed on rebuild" in rec1.extra["finalise_rebuild_failed"]
- # dirty the tree so the resume's finalise needs a restore (and hence a rebuild)
- (ws.src / "object.js").write_text((ws.src / "object.js").read_text() + "\n// scribble\n")
- ws.commit("scribble after the failed finalise")
- rec2 = track().run(spec, ws, resume=True)
+ # the resume is still past the clock: no round runs, and its finalise rebuilds the last round
+ with fake_clock():
+ rec2 = track().run(spec, ws, resume=True)
assert "finalise_rebuild_failed" not in rec2.extra, "a successful rebuild must clear the stale flag"
assert not rec2.error.startswith("finalise rebuild failed")
assert (ws.artifacts / "object.glb").is_file()
diff --git a/harness/tests/orchestrator_tracks/test_fixed_rounds.py b/harness/tests/orchestrator_tracks/test_fixed_rounds.py
index 5b894cb4..48119e74 100644
--- a/harness/tests/orchestrator_tracks/test_fixed_rounds.py
+++ b/harness/tests/orchestrator_tracks/test_fixed_rounds.py
@@ -72,6 +72,9 @@ def test_every_refine_starts_from_the_previous_rounds_commit(tmp_path, chair_pla
assert rec.rounds[2].instructions, "the tasks come from r01's verdict and gates"
events = [e["event"] for e in EventLog(ws.events_path).read()]
assert "round.refine_from_best" not in events and "strategy.switch" not in events
+ # ... and the run ENDS at its last round: no restore of a better-scored one
+ assert "restore" not in ws._git("log", "--format=%s").stdout # noqa: SLF001
+ assert ws.head() == rec.rounds[-1].commit and (ws.src / "model.py").read_text() == "import bpy # written in r2\n"
class _RoundStampRuntime(FakeRuntime):
@@ -106,6 +109,7 @@ def test_every_round_keeps_its_build_and_any_round_can_be_packaged(tmp_path, cha
assert all(p.is_file() for p in kept) and (ws.round_artifacts(1) / "object.stl").is_file()
assert len({p.read_bytes() for p in kept}) == 3, "each round keeps its OWN build"
assert not (ws.round_artifacts(1) / "census.json").exists(), "only what a hand-over needs is kept"
+ assert (ws.artifacts / "object.glb").read_bytes() == kept[2].read_bytes(), "the canonical build is the last round's"
# any round packages without a rebuild: its commit's code + its own GLB
builds = track._runtime.builds
for i in (1, 0):
diff --git a/harness/tests/orchestrator_tracks/test_orchestrator.py b/harness/tests/orchestrator_tracks/test_orchestrator.py
index e909165f..bf7da9cd 100644
--- a/harness/tests/orchestrator_tracks/test_orchestrator.py
+++ b/harness/tests/orchestrator_tracks/test_orchestrator.py
@@ -2,13 +2,13 @@
from __future__ import annotations
+import json
import time
import pytest
from pydantic import BaseModel
from codeverse3d.contracts.artifacts import (
- BuildResult,
GateFinding,
GateReport,
ImprovementItem,
@@ -17,7 +17,7 @@ from codeverse3d.contracts.artifacts import (
)
from codeverse3d.contracts.common import Budget, Usage
from codeverse3d.contracts.plan import AcceptanceItem
-from codeverse3d.contracts.run import RoundRecord
+from codeverse3d.contracts.run import RunStatus
from codeverse3d.orchestrator import (
BudgetExceeded,
BudgetGuard,
@@ -26,7 +26,6 @@ from codeverse3d.orchestrator import (
StageRunner,
build_refine_instructions,
hash_inputs,
- pick_best_round,
plan_parallel_groups,
)
from codeverse3d.proc import EventLog, fan_out
@@ -68,11 +67,22 @@ def test_fan_out_preserves_order_and_captures_exceptions():
def test_run_state_roundtrip(tmp_ws):
st = RunState()
st.mark_round_done(0, "abc")
- st.update_best(0, "abc", 0.5)
st.save(tmp_ws)
again = RunState.load(tmp_ws)
- assert again is not None and again.best_round == 0 and again.current_round == 1 and again.round_commits[0] == "abc"
- assert RunState.load_or_new(tmp_ws, resume=False).best_round is None
+ assert again is not None and again.current_round == 1 and again.round_commits[0] == "abc"
+ assert RunState.load_or_new(tmp_ws, resume=False).completed_rounds == []
+
+
+def test_a_run_state_saved_before_2026_09_22_still_loads(tmp_ws):
+ """It named a best round and could end ``passed`` / ``plateau``: the keys are ignored
+ and the status reads as ``stopped`` — a resume goes on from the last round."""
+ tmp_ws.state_path.write_text(json.dumps({
+ "status": "passed", "stop_reason": "pass", "current_round": 2, "completed_rounds": [0, 1],
+ "round_commits": {"0": "a0", "1": "a1"}, "best_round": 0, "best_commit": "a0", "best_score": 0.81,
+ "best_considered_through": 1}))
+ st = RunState.load(tmp_ws)
+ assert st is not None and st.status is RunStatus.STOPPED and st.round_commits == {0: "a0", 1: "a1"}
+ assert "best_round" not in st.model_dump() and RunStatus("plateau") is RunStatus.STOPPED
def test_stage_runner_caches_by_input_hash(tmp_ws):
@@ -145,22 +155,6 @@ def test_stage_runner_raises_and_records_nothing(tmp_ws):
assert hash_inputs({"a": [1, 2]}) == hash_inputs({"a": (1, 2)})
-# ----------------------------------------------------------------------------- best round
-def _round(i: int, score: float | None, errors: int = 0, build_ok: bool = True) -> RoundRecord:
- j = Judgment(rubric="r", scores={"a": score}, overall=score, passed=score >= 0.8) if score is not None else None
- gates = [GateReport(gate="g", passed=errors == 0, findings=[GateFinding(gate="g", severity=Severity.ERROR, message="e")] * errors)]
- return RoundRecord(index=i, kind="x", judgment=j, gates=gates, build=BuildResult(ok=build_ok, language="l"), commit=f"c{i}")
-
-
-def test_pick_best_round_prefers_score_then_fewer_errors_then_recency():
- rounds = [_round(0, 0.5), _round(1, 0.7, errors=2), _round(2, 0.7, errors=0), _round(3, 0.6)]
- assert pick_best_round(rounds) == 2
- assert pick_best_round([_round(0, 0.7, errors=1), _round(1, 0.7, errors=1)]) == 1 # full tie → the later one
- assert pick_best_round([_round(0, 0.9, errors=3), _round(1, 0.8)]) == 0 # score outranks gate errors
- assert pick_best_round([]) is None
- assert pick_best_round([_round(0, None, build_ok=False), _round(1, None, build_ok=True)]) == 1
-
-
# ----------------------------------------------------------------------------- refine instructions
def test_build_refine_instructions_merges_and_groups(chair_plan):
judgment = Judgment(rubric="r", scores={}, overall=0.6, passed=False, improvement_plan=[
diff --git a/harness/tests/orchestrator_tracks/test_resume_reconcile.py b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
index 060ade50..98aa35ae 100644
--- a/harness/tests/orchestrator_tracks/test_resume_reconcile.py
+++ b/harness/tests/orchestrator_tracks/test_resume_reconcile.py
@@ -1,8 +1,9 @@
"""reconcile_resume: on resume, the durable round journal is the truth.
-One test per audited failure mode — the two crash windows between a round write and the
-state save, a planner outage wiping ``record.rounds``, ``--force``/no-``--force`` after a
-spec edit, judge notes lost to a local list, and spend the budget snapshot missed.
+One test per audited failure mode — the crash window between a round write and the state
+save, a run recorded before 2026-09-22 (it restored its best round), a planner outage wiping
+``record.rounds``, ``--force``/no-``--force`` after a spec edit, judge notes lost to a local
+list, and spend the budget snapshot missed.
"""
from __future__ import annotations
@@ -14,6 +15,7 @@ from types import SimpleNamespace
import pytest
from codeverse3d.contracts.common import Language
+from codeverse3d.contracts.run import RunStatus
from codeverse3d.orchestrator import RoundPolicy, RunState
from codeverse3d.proc import EventLog
from codeverse3d.tracks.lifecycle import SpecChanged
@@ -66,88 +68,67 @@ def completed_run(tmp_path, chair_plan, settings):
# --------------------------------------------------------------------- (a) crash window
-def test_crash_between_round_write_and_state_save_delivers_the_newer_round(completed_run):
- """rNN.json + its commit are durable but mark_round_done/update_best lived only in
- memory until _save_budget — with a possible PAID pairwise call in between. A crash in
- that window plus an immediate stop on resume delivered the STALE best."""
+def test_crash_between_round_write_and_state_save_ends_at_the_newer_round(completed_run):
+ """rNN.json + its commit are durable but mark_round_done lived only in memory until
+ the state save. A crash in that window left the state one round behind the journal
+ (and here the tree on the older round): the resume rebuilds the state from the journal
+ and the run ends on the LAST round."""
run = completed_run(max_rounds=1)
rec1, ws = run.record, run.ws
- assert rec1.best_round == 1 and len(rec1.rounds) == 2
+ assert len(rec1.rounds) == 2
# rewind state.json to what a crash between the r01 write and the state save leaves
state = json.loads(ws.state_path.read_text())
state["completed_rounds"], state["current_round"] = [0], 1
state["round_commits"] = {"0": rec1.rounds[0].commit}
- state["best_round"], state["best_commit"], state["best_score"] = 0, rec1.rounds[0].commit, 0.5
ws.state_path.write_text(json.dumps(state))
- ws.restore(rec1.rounds[0].commit) # ...and the tree at the stale best, as delivered
- ws.commit("stale delivery")
+ ws.restore(rec1.rounds[0].commit)
+ ws.commit("stale tree")
rec2 = run.rerun(resume=True)
- # the immediate stop (max_rounds) now delivers the NEWER paid round
- assert rec2.best_round == 1 and rec2.final_score == pytest.approx(0.7)
assert [r.index for r in rec2.rounds] == [0, 1]
assert "refine" in (ws.src / "model.py").read_text(), "round 1's code must be restored"
st = RunState.load(ws)
- assert st.completed_rounds == [0, 1] and st.best_round == 1
- assert st.best_commit == rec1.rounds[1].commit
+ assert st.completed_rounds == [0, 1] and st.round_commits[1] == rec1.rounds[1].commit
ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "resume.reconciled"]
- assert ev and ev[-1]["state_was_stale"] is True and ev[-1]["best_changed"] is True
+ assert ev and ev[-1]["state_was_stale"] is True and ev[-1]["restored_last_round"] == 1
-def test_a_self_consistent_state_with_an_unranked_round_still_re_ranks(completed_run):
- """The sharper crash shape (reproduced by review, 2026-08-27): the round was saved
- BEFORE best selection, so completed_rounds/round_commits matched the journal exactly
- while best_round still pointed at the older round — nothing looked stale, reconcile
- believed it, and the run DELIVERED r0=0.5 over the paid r1=0.7. ``best_considered_
- through`` makes the repair independent of write ordering."""
+def test_an_old_run_that_restored_its_best_round_resumes_from_its_last(completed_run, settings):
+ """A run recorded before 2026-09-22 ended on a "restore best round rNN" commit, with its
+ run_state.json and record.json naming that best round and status ``passed``. Both still
+ load; the resume puts src/ back on the LAST round and the next round refines THAT one."""
run = completed_run(max_rounds=1)
rec1, ws = run.record, run.ws
- assert rec1.best_round == 1 and rec1.rounds[1].score == pytest.approx(0.7)
-
- state = json.loads(ws.state_path.read_text()) # journal-consistent, best stale
- assert state["completed_rounds"] == [0, 1]
- state["best_round"], state["best_commit"], state["best_score"] = 0, rec1.rounds[0].commit, 0.5
- state["best_considered_through"] = 0 # r1 was written but never ranked
+ state = json.loads(ws.state_path.read_text())
+ state.update(status="passed", stop_reason="pass", best_round=0, best_commit=rec1.rounds[0].commit,
+ best_score=0.5, best_considered_through=1)
ws.state_path.write_text(json.dumps(state))
- ws.restore(rec1.rounds[0].commit)
- ws.commit("stale delivery")
+ record = json.loads(ws.record_path.read_text())
+ record.update(status="passed", best_round=0, baseline_score=0.5, final_score=0.5)
+ ws.record_path.write_text(json.dumps(record))
+ ws.restore(rec1.rounds[0].commit) # what the old finalise did
+ ws.commit("restore best round r00")
+
+ seen: list[tuple[int, str]] = []
+
+ def writer(job, ws_):
+ seen.append((job.round, (ws_.src / "model.py").read_text()))
+ return _writer(job, ws_)
+
+ track = StaticObjectTrack(services=FakeServices(), judge=FakeJudge(scores=(0.6,)), agent=FakeAgent(writer),
+ planner_model=_planner(run.plan), settings=settings,
+ runtime=FakeRuntime(Language.BLENDER), policy=RoundPolicy(max_rounds=2))
+ rec2 = track.run(make_spec(language=Language.BLENDER, max_rounds=2), ws, resume=True)
+ assert [r.index for r in rec2.rounds] == [0, 1, 2] and rec2.status is RunStatus.MAX_ROUNDS
+ assert seen and all(rnd == 2 for rnd, _ in seen), "only the new round generated"
+ assert seen[0][1] == _code_at(ws, rec1.rounds[1].commit) != _code_at(ws, rec1.rounds[0].commit), "round 2 refines round 1"
+ ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "resume.reconciled"]
+ assert ev[-1]["restored_last_round"] == 1
+ assert ws.head() == rec2.rounds[-1].commit and _code_at(ws, ws.head()) == (ws.src / "model.py").read_text()
- rec2 = run.rerun(resume=True)
- assert rec2.best_round == 1 and rec2.final_score == pytest.approx(0.7)
- ev = [e for e in EventLog(ws.events_path).read() if e["event"] == "resume.reconciled"][-1]
- assert ev["state_was_stale"] is False and ev["unranked_rounds"] is True and ev["best_changed"] is True
-
-
-def test_a_stored_pairwise_rejection_is_final(chair_plan):
- """A paid pairwise REJECTION of a challenger (r1 within margin, judge said keep r0)
- used to be silently overturned once ANY later round landed: both the live
- ``choose_best_round`` and the resume ``replay_best_round`` re-ranked the WHOLE
- journal on score alone and crowned the rejected r1. The verdict is final: live
- and replay agree, whether the later round is worse or failed to score at all."""
- from codeverse3d.contracts.artifacts import BuildResult, Judgment
- from codeverse3d.contracts.run import PairwiseNote, RoundRecord
- from codeverse3d.tracks.candidates import choose_best_round, replay_best_round
-
- def rec(i, score, pairwise=None):
- j = None if score is None else Judgment(rubric="r", scores={}, overall=score, passed=False)
- return RoundRecord(index=i, kind="baseline" if i == 0 else "refine", commit=f"c{i}",
- build=BuildResult(ok=score is not None, language="blender", entrypoint="src/main.py"),
- judgment=j, pairwise=pairwise)
-
- r0 = rec(0, 0.70)
- rejected = PairwiseNote(a="r00", b="r01", winner="a", confidence=0.9, accepted=False)
- r1 = rec(1, 0.72, pairwise=rejected) # outscores r0, but the paid verdict said keep r0
- r2 = rec(2, 0.50)
- assert replay_best_round([r0, r1]) == 0
- assert replay_best_round([r0, r1, r2]) == 0, "a worse later round must not revive the rejected r1"
- ctx = SimpleNamespace(state=SimpleNamespace(best_round=0),
- policy=SimpleNamespace(pairwise_margin=0.03, pairwise_min_confidence=0.6))
- assert choose_best_round(ctx, [r0, r1, r2], 2) == 0, "live must agree with replay"
- r2b = rec(2, None) # the new round never scored (build crash)
- assert choose_best_round(ctx, [r0, r1, r2b], 2) == 0
- # an ACCEPTED verdict still promotes the challenger, and survives later worse rounds
- accepted = PairwiseNote(a="r00", b="r01", winner="b", confidence=0.9, accepted=True)
- assert replay_best_round([r0, rec(1, 0.72, pairwise=accepted), r2]) == 1
+
+def _code_at(ws: Workspace, commit: str) -> str:
+ return ws._git("show", f"{commit}:src/model.py").stdout # noqa: SLF001
# --------------------------------------------------------------------- (b) history wipe
@@ -218,7 +199,7 @@ def test_a_spec_edit_without_force_is_refused_before_anything_runs(completed_run
assert not (ws.root / "rounds" / "pre_force").exists()
assert (ws.root / "rounds" / "r00.json").is_file()
st = RunState.load(ws)
- assert st.best_round == 0 and st.completed_rounds == [0]
+ assert st.completed_rounds == [0]
assert json.loads(ws.record_path.read_text())["status"] != "failed"
# the sanctioned budget raise (outside the fingerprint) still resumes plainly
diff --git a/harness/tests/orchestrator_tracks/test_round_economics.py b/harness/tests/orchestrator_tracks/test_round_economics.py
index 14040fd2..c9747952 100644
--- a/harness/tests/orchestrator_tracks/test_round_economics.py
+++ b/harness/tests/orchestrator_tracks/test_round_economics.py
@@ -169,14 +169,13 @@ def test_skip_judge_reasons_are_only_states_where_the_verdict_is_never_bought(tm
ctx = _ctx(tmp_path, spec, settings)
# gate errors are no reason: the gates say what is broken, the verdict says whether
- # the shape is right, and only the verdict can promote the round
+ # the shape is right, and only the verdict lets a pick choose the round
assert skip_judge_reason(ctx, renders=_renders()) == ""
assert skip_judge_reason(ctx, renders=None) == "no renders"
- # ... the budget/clock stop: the loop's next budget_ok check ends the run, so this
- # verdict cannot promote anything (audit: 2 verdicts / $0.09 bought past the clock)
- ctx.budget.add(Usage(cost_usd=99.0), stage="refine")
+ # a round that finished past the clock is still judged: its generation is paid for and
+ # an unjudged round can never be picked (2026-09-22)
ctx.budget._active_s = (ctx.budget.budget.max_minutes + 1) * 60 # noqa: SLF001
- assert skip_judge_reason(ctx, renders=_renders()) == "budget already exceeded"
+ assert not ctx.budget.ok() and skip_judge_reason(ctx, renders=_renders()) == ""
ctx.judge = None
assert skip_judge_reason(ctx, renders=_renders()) == "no judge configured"
@@ -204,20 +203,6 @@ def test_a_round_with_gate_errors_whose_verdict_was_lost_is_rejudged(tmp_path, s
assert rejudge_round(ctx, _pipeline(), rec) is True and len(ctx.judge.calls) == 1
-def test_a_round_that_broke_the_gates_does_not_displace_a_clean_one():
- from codeverse3d.orchestrator import pick_best_round
-
- def r(i, *, build_ok=True, errors=0):
- return RoundRecord(index=i, kind="refine", build=BuildResult(ok=build_ok, language="l"),
- gates=_gates(errors), commit=f"c{i}")
-
- assert pick_best_round([r(0), r(1, build_ok=False)]) == 0 # broken build never wins
- assert pick_best_round([r(0), r(1, errors=4)]) == 0 # clean r00 keeps the crown
- assert pick_best_round([r(0, errors=4), r(1)]) == 1
- assert pick_best_round([r(0), r(1)]) == 1 # tie → the later one
- assert pick_best_round([r(0, build_ok=False)]) is None
-
-
class _Pipeline:
"""The minimum ``RoundPipeline`` the judge path touches."""
@@ -299,7 +284,6 @@ def test_a_lint_stuck_run_keeps_every_score_instead_of_deferring_the_verdict(tmp
assert [e for e in events if e["event"] == "judge.skipped"] == []
assert [e for e in events if e["event"] == "judge.retry"] == []
assert [r.score for r in rec.rounds] == [0.5, 0.6, 0.7]
- assert rec.best_round == 2 and rec.final_score == pytest.approx(0.7)
# one verdict per round, bought once
assert len([e for e in events if e["event"] == "judge.done"]) == len(rec.rounds) == len(judge.calls)
diff --git a/harness/tests/orchestrator_tracks/test_tracks.py b/harness/tests/orchestrator_tracks/test_tracks.py
index b9d4f8fc..e737910d 100644
--- a/harness/tests/orchestrator_tracks/test_tracks.py
+++ b/harness/tests/orchestrator_tracks/test_tracks.py
@@ -6,6 +6,7 @@ import json
import pytest
+from codeverse3d.addons import select
from codeverse3d.contracts.common import Language, Track
from codeverse3d.contracts.plan import ArticulatedPlan, ScenePlan
from codeverse3d.contracts.run import RunRecord, RunStatus
@@ -56,14 +57,14 @@ def test_static_track_end_to_end_agent_path(tmp_path, chair_plan, settings):
# FIXED rounds: 0.85 would once have stopped the run as a pass; the baseline + 3 refine rounds run
assert isinstance(rec, RunRecord) and rec.status is RunStatus.MAX_ROUNDS
assert [r.kind for r in rec.rounds] == ["baseline", "refine", "refine", "refine"]
- assert rec.baseline_score == pytest.approx(0.55) and rec.final_score == pytest.approx(0.85) and rec.best_round == 3
+ assert [r.score for r in rec.rounds] == [0.55, 0.7, 0.85, 0.85]
assert rec.total_usage.cost_usd > 0 and rec.extra["stop_reason"] == "max_rounds"
assert ws.record_path.is_file() and ws.plan_path.is_file() and ws.state_path.is_file()
state = RunState.load(ws)
assert state.status is RunStatus.MAX_ROUNDS and state.completed_rounds == [0, 1, 2, 3]
assert services.materialized == ["fake"] and (ws.root / "AGENTS.md").is_file()
kinds = [e["event"] for e in EventLog(ws.events_path).read()]
- for k in ("run.start", "stage.done", "plan.done", "round.start", "build.done", "gates.done", "judge.done", "round.done", "best.updated", "stop", "run.done"):
+ for k in ("run.start", "stage.done", "plan.done", "round.start", "build.done", "gates.done", "judge.done", "round.done", "stop", "run.done"):
assert k in kinds, k
# per-round artifacts
assert (ws.root / "rounds" / "r00.json").is_file() and ws.judge_path(2).is_file() and (ws.gates_dir(1) / "contract.json").is_file()
@@ -310,8 +311,8 @@ def test_get_track_dispatch():
def test_a_render_timeout_degrades_the_round_instead_of_failing_the_run(tmp_path, chair_plan, settings):
"""Measured 2026-08-26 (art_verify camera_tripod): the refine round built in 1.2 s, then
render_glb.mjs hit its 330 s timeout under load and the run was recorded `failed` with a
- judged round 0 on disk. The round keeps build/gates, skips the judge, and the run
- delivers its best round."""
+ judged round 0 on disk. The round keeps build/gates and skips the judge, so a pick
+ hands over the judged round 0."""
from codeverse3d.spatial.render import RenderError
class _Services(FakeServices):
@@ -326,7 +327,7 @@ def test_a_render_timeout_degrades_the_round_instead_of_failing_the_run(tmp_path
agent=FakeAgent(_agent_writer), planner_model=_planner(chair_plan.model_dump(mode="json")),
settings=settings, runtime=FakeRuntime(Language.THREEJS))
rec = track.run(spec, ws)
- assert rec.status is not RunStatus.FAILED and rec.best_round == 0 and rec.final_score == pytest.approx(0.55)
+ assert rec.status is not RunStatus.FAILED and select.pick(ws.root) == 0
r1 = rec.rounds[1]
assert r1.build is not None and r1.build.ok and r1.renders is None and r1.judgment is None
assert "render failed" in r1.notes