Official Codebase of 3DCodeVerse
Python
flywheel: a reader of recorded runs, not a writer — delete the model clients, live accounting, captioner, VLM judge, calibration
gaoypeng09f7d9157f44
diff --git a/flywheel/c3d_flywheel/calibration.py b/flywheel/c3d_flywheel/calibration.pydeleted file mode 100644index f71a4759..00000000--- a/flywheel/c3d_flywheel/calibration.py+++ /dev/null@@ -1,292 +0,0 @@-"""Judge calibration smoke: re-judge finished runs and check the judge separates good from bad."""--from __future__ import annotations--import argparse-import json-import logging-import statistics-import time-from concurrent.futures import ThreadPoolExecutor-from pathlib import Path-from typing import Any--from pydantic import BaseModel, Field--from c3d_flywheel import select-from c3d_flywheel.cli._judge import build_judge_input, make_judge, rubric_for-from codeverse3d.config import get_settings-from codeverse3d.contracts.artifacts import Judgment, RenderSet-from codeverse3d.contracts.run import RoundRecord, RunRecord-from codeverse3d.contracts.spec import Spec-from codeverse3d.judges.base import JudgeInput-from c3d_flywheel.vlm_judge import VlmJudge-from c3d_flywheel.record.record import RecordError, load_record-from codeverse3d.workspace import Workspace--log = logging.getLogger(__name__)----def _run_label(run_dir: Path) -> str:- parts = run_dir.resolve().parts- tail = [q for q in parts[-5:] if q not in ("runs", "cells")]- return "_".join(tail) if tail[-1] == "run" else tail[-1]---# --------------------------------------------------------------------------- data-class RoundCase(BaseModel):-- run: str- round_index: int- kind: str- rubric: str- inp: JudgeInput- gate_errors: int- gate_warnings: int- stored: Judgment | None = None- is_picked: bool = False- glb: str | None = None---def _stored_weighted(case: RoundCase) -> float | None:- from codeverse3d.judges.rubrics import load_rubric-- if not case.stored or not case.stored.scores:- return None- try:- return round(load_rubric(case.rubric).weighted_overall(case.stored.scores), 3)- except Exception: # noqa: BLE001- return None---class CalibrationRow(BaseModel):- run: str- round: int- kind: str- gate_errors: int- gate_warnings: int- stored_overall: float | None = None- stored_weighted: float | None = None- stored_backend: str = ""- mean: float- std: float- uncapped: float- defect_penalty: float- per_criterion_mean: dict[str, float] = Field(default_factory=dict)- per_criterion_std: dict[str, float] = Field(default_factory=dict)- defects_present: list[str] = Field(default_factory=list)- caps: list[str] = Field(default_factory=list)- passed: bool- n_used: int- geometry_views: bool = False- cost_usd: float = 0.0- duration_s: float = 0.0- summary: str = ""- error: str = ""---class CalibrationTable(BaseModel):- model_id: str- n_samples: int- fixed_order: bool = False- rows: list[CalibrationRow]- pearson_errors_vs_score: float | None = None- spearman_errors_vs_score: float | None = None- pearson_stored_vs_new: float | None = None- mean_std: float = 0.0- mean_criterion_std: float = 0.0- total_cost_usd: float = 0.0-- def to_markdown(self) -> str:- head = ("| run | round | kind | gate err/warn | stored | new mean ± std | uncapped | penalty | defects | caps | pass | geo | cost |\n"- "|---|---|---|---|---|---|---|---|---|---|---|---|---|")- lines = [head]- for r in self.rows:- stored = f"{r.stored_overall:.3f}" if r.stored_overall is not None else "-"- if (r.stored_overall is not None and r.stored_weighted is not None- and abs(r.stored_overall - r.stored_weighted) > 0.05):- stored += f" (w {r.stored_weighted:.3f}!)"- lines.append(- f"| {r.run} | r{r.round:02d} | {r.kind} | {r.gate_errors}/{r.gate_warnings} | {stored} | "- f"{r.mean:.3f} ± {r.std:.3f} | {r.uncapped:.3f} | -{r.defect_penalty:.2f} | "- f"{', '.join(r.defects_present) or '-'} | {', '.join(r.caps) or '-'} | {'Y' if r.passed else 'n'} | "- f"{'Y' if r.geometry_views else '-'} | ${r.cost_usd:.3f} |" + (f" ERROR {r.error}" if r.error else "")- )- crit = sorted({k for r in self.rows for k in r.per_criterion_mean})- if crit:- lines.append("")- lines.append("| run/round | " + " | ".join(crit) + " |")- lines.append("|---|" + "---|" * len(crit))- for r in self.rows:- cells = [f"{r.per_criterion_mean.get(c, float('nan')):.2f}±{r.per_criterion_std.get(c, 0.0):.2f}" if c in r.per_criterion_mean else "-" for c in crit]- lines.append(f"| {r.run}/r{r.round:02d} | " + " | ".join(cells) + " |")- fmt = lambda v: "n/a" if v is None else f"{v:+.3f}" # noqa: E731- lines.append("")- lines.append(- f"model {self.model_id}, n_samples {self.n_samples}{' (fixed montage order: σ is the re-judge noise)' if self.fixed_order else ''}: "- f"pearson(gate errors, new score) = {fmt(self.pearson_errors_vs_score)}; "- f"spearman = {fmt(self.spearman_errors_vs_score)}; pearson(stored, new) = {fmt(self.pearson_stored_vs_new)}; "- f"mean overall std {self.mean_std:.3f}; mean per-criterion std {self.mean_criterion_std:.3f}; cost ${self.total_cost_usd:.3f}"- )- return "\n".join(lines)---# --------------------------------------------------------------------------- loading runs-def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[RoundCase]:- run_dir = Path(run_dir)- ws = Workspace(run_dir)- saved = [RoundRecord.model_validate_json(p.read_text()) for p in sorted((run_dir / "rounds").glob("r*.json"))]- try:- run = load_record(ws)- picked = select.summarise(run_dir, record=run).round- except RecordError:- run = RunRecord(spec=Spec.model_validate_json(ws.spec_path.read_text()), workspace=str(run_dir))- picked = select.fallback_round(saved)- cases: list[RoundCase] = []- for rec in saved:- if rounds is not None and rec.index not in rounds:- continue- if rec.renders is None or not rec.renders.views:- continue- cases.append(RoundCase(- run=_run_label(run_dir), round_index=rec.index, kind=rec.kind, rubric=rubric_for(run, rec, None),- inp=build_judge_input(ws, run, rec),- gate_errors=sum(len(g.errors) for g in rec.gates),- gate_warnings=sum(1 for g in rec.gates for f in g.findings if f.severity.value == "warn"),- stored=rec.judgment, is_picked=picked == rec.index,- glb=str(glb) if (glb := select.round_file(ws, rec)) is not None else None,- ))- return cases---def render_geometry_views(case: RoundCase, out_dir: Path, mode: str) -> RenderSet | None:- if case.inp.spec.track.value == "scene":- return None- if not case.glb:- log.warning("no geometry views for %s r%02d: the round kept no GLB of its own", case.run, case.round_index)- return None- from codeverse3d.conventions import OBJECT_CLAY_VIEWS- from codeverse3d.spatial.render import render_glb-- views = list(OBJECT_CLAY_VIEWS)- try:- return render_glb(case.glb, out_dir / case.run / f"geometry_{mode}", views=views, mode=mode, sheet=False)- except Exception as e: # noqa: BLE001- log.warning("geometry render failed for %s: %s", case.run, e)- return None---# --------------------------------------------------------------------------- statistics-def pearson(xs: list[float], ys: list[float], method: str = "linear") -> float | None:- if len(xs) < 2 or len(set(xs)) < 2 or len(set(ys)) < 2:- return None- return round(statistics.correlation(xs, ys, method=method), 4)---def spearman(xs: list[float], ys: list[float]) -> float | None:- return pearson(xs, ys, method="ranked")---# --------------------------------------------------------------------------- main entry-def _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, out: Path) -> CalibrationRow:- t0 = time.time()- inp = case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp- j = judge.judge(inp)- raw = json.loads(j.raw) if j.raw else {}- jdir = out / "judgments"- jdir.mkdir(parents=True, exist_ok=True)- (jdir / f"{case.run}_r{case.round_index:02d}_{judge.model_id.replace(':', '_')}.json").write_text(j.model_dump_json(indent=1))- return CalibrationRow(- run=case.run, round=case.round_index, kind=case.kind, gate_errors=case.gate_errors, gate_warnings=case.gate_warnings,- stored_overall=case.stored.overall if case.stored else None,- stored_weighted=_stored_weighted(case),- stored_backend=case.stored.judge_backend if case.stored else "",- mean=j.overall, std=j.score_std, uncapped=float(raw.get("overall_uncapped", j.overall)),- defect_penalty=float(raw.get("defect_penalty", 0.0)),- per_criterion_mean=dict(j.scores), per_criterion_std=dict(raw.get("per_criterion_std", {})),- defects_present=[d for d, on in raw.get("defects", {}).items() if on],- caps=[c["rule"] for c in raw.get("caps", {}).get("caps_applied", [])],- passed=j.passed, n_used=j.n_samples, geometry_views=inp.geometry_views is not None,- cost_usd=j.usage.cost_usd, duration_s=round(time.time() - t0, 1), summary=j.summary,- error=j.summary if j.degraded else "",- )---def calibrate(- run_dirs: list[Path | str],- *,- model_id: str | None = None,- n_samples: int = 3,- out_dir: Path | str | None = None,- geometry_mode: str | None = "clay",- rounds: list[int] | None = None,- thinking: str | None = None,- max_workers: int = 4,- chat_model: Any = None,- fixed_order: bool = False,-) -> CalibrationTable:- model_id = model_id or get_settings().default_judge- thinking = thinking or get_settings().judge.thinking- out = Path(out_dir) if out_dir else Path.cwd()- out.mkdir(parents=True, exist_ok=True)- cases: list[RoundCase] = []- for rd in run_dirs:- cases.extend(load_run_cases(Path(rd), rounds=rounds))- if not cases:- raise ValueError(f"no judgeable rounds found under {list(map(str, run_dirs))}")- geometry: dict[int, RenderSet | None] = {}- for i, c in enumerate(cases):- want = geometry_mode and c.is_picked and c.inp.geometry_views is None- geometry[i] = render_geometry_views(c, out, geometry_mode) if want else None-- def kind(c: RoundCase) -> tuple[str, str, bool]:- return c.rubric, c.inp.spec.track.value, bool(c.inp.spec.references)-- judges = {kind(c): make_judge(c.inp.spec, c.rubric, model_id, n_samples, thinking=thinking,- cache_dir=out / "cache", label="calibrate", chat_model=chat_model,- fixed_order=fixed_order) for c in cases}- with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:- rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[kind(ic[1])], geometry[ic[0]], out), enumerate(cases)))- ok = [r for r in rows if not r.error]- table = CalibrationTable(- model_id=model_id, n_samples=n_samples, fixed_order=fixed_order, rows=rows,- pearson_errors_vs_score=pearson([float(r.gate_errors) for r in ok], [r.mean for r in ok]),- spearman_errors_vs_score=spearman([float(r.gate_errors) for r in ok], [r.mean for r in ok]),- pearson_stored_vs_new=pearson([r.stored_overall for r in ok if r.stored_overall is not None],- [r.mean for r in ok if r.stored_overall is not None]),- mean_std=round(statistics.fmean([r.std for r in ok]), 4) if ok else 0.0,- mean_criterion_std=round(statistics.fmean([v for r in ok for v in r.per_criterion_std.values()] or [0.0]), 4),- total_cost_usd=round(sum(r.cost_usd for r in rows), 4),- )- stem = "calibration_" + model_id.replace(":", "_").replace("/", "_")- (out / f"{stem}.json").write_text(json.dumps(table.model_dump(mode="json"), indent=1, ensure_ascii=False))- (out / f"{stem}.md").write_text(table.to_markdown() + "\n")- return table---def main(argv: list[str] | None = None) -> int:- ap = argparse.ArgumentParser(description="Re-judge finished runs to calibrate the VLM judge.")- ap.add_argument("run_dirs", nargs="+")- ap.add_argument("--model", default=None, help="chat model id, e.g. gemini:gemini-4-argon")- ap.add_argument("--n", type=int, default=3, help="samples per round")- ap.add_argument("--out", default=".", help="output directory (json + md)")- ap.add_argument("--geometry", default="clay", help="clay | normals | none (geometry montage for the picked round)")- ap.add_argument("--rounds", default="", help="comma-separated round indices (default all)")- ap.add_argument("--thinking", default=None, choices=("off", "low", "medium", "high"),- help="judge thinking level (default: Settings.judge.thinking / C3D_JUDGE__THINKING)")- ap.add_argument("--workers", type=int, default=4)- ap.add_argument("--fixed-order", action="store_true",- help="same montage order for every sample: measures the model's re-judge σ, not order robustness")- a = ap.parse_args(argv)- logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")- rounds = [int(x) for x in a.rounds.split(",") if x.strip()] or None- table = calibrate(a.run_dirs, model_id=a.model, n_samples=a.n, out_dir=a.out,- geometry_mode=None if a.geometry == "none" else a.geometry, rounds=rounds,- thinking=a.thinking, max_workers=a.workers, fixed_order=a.fixed_order)- print(table.to_markdown())- return 0---if __name__ == "__main__": # pragma: no cover- raise SystemExit(main())diff --git a/flywheel/c3d_flywheel/cli/_judge.py b/flywheel/c3d_flywheel/cli/_judge.pydeleted file mode 100644index 01750361..00000000--- a/flywheel/c3d_flywheel/cli/_judge.py+++ /dev/null@@ -1,97 +0,0 @@-"""``3dcodeverse judge`` implementation: re-judge a stored round with the inputs the in-run judge-saw, built by ``judges.base.round_input`` from the run directory."""--from __future__ import annotations--from typing import Any--from pydantic import ValidationError--from codeverse3d.cli import _common as C-from codeverse3d.contracts.artifacts import BuildResult, RenderSet, RenderView-from codeverse3d.contracts.plan import Plan-from codeverse3d.contracts.run import RoundRecord, RunRecord-from codeverse3d.contracts.spec import Spec-from codeverse3d.judges.base import judged_subset, resolve_paths, round_input-from codeverse3d.proc import read_json_or_none-from c3d_flywheel.record.record import previous_verdict-from codeverse3d.workspace import Workspace---def load_round(ws: Workspace, rec: RunRecord, index: int) -> RoundRecord | None:- p = ws.root / "rounds" / f"r{index:02d}.json"- if p.is_file():- try:- return RoundRecord.model_validate_json(p.read_text())- except Exception: # noqa: BLE001 — fall through to the record copy- pass- return next((r for r in rec.rounds if r.index == index), None)---def rubric_for(rec: RunRecord, rnd: RoundRecord, override: str | None) -> str:- from c3d_flywheel.record.record import rubric_of-- if override:- return override- if rnd.judgment is not None and rnd.judgment.rubric:- return rnd.judgment.rubric- return rubric_of(rec)---def stored_plan(ws: Workspace, rec: RunRecord) -> Plan | None:- """The run's typed plan: record.json's, else ``plan.json``."""- if rec.plan is not None:- return rec.plan- from codeverse3d.tracks import get_track-- data = read_json_or_none(ws.plan_path)- try:- return get_track(rec.spec.track).plan_model.model_validate(data) if data else None- except ValidationError:- return None---def extra_context_for(ws: Workspace, rec: RunRecord, plan: Plan | None, rnd: RoundRecord) -> str:- from codeverse3d.tracks import get_track-- build = rnd.build if rnd.build is not None else BuildResult(ok=False, language=rec.spec.language.value)- try:- return get_track(rec.spec.track).make_pipeline().judge_context(ws, plan, rnd.index, build, list(rnd.gates))- except Exception as e: # noqa: BLE001 — extra context is optional judge input- return f"(judge context unavailable: {type(e).__name__}: {e})"---def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:- d = ws.renders_dir(index) / "clay"- if not d.is_dir():- return None- views = [RenderView(name=p.stem.removeprefix("view_"), path=str(p), mode="clay")- for p in sorted(d.glob("view_*.png"))]- return RenderSet(views=views, renderer="stored") if views else None---def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:- plan = stored_plan(ws, rec)- return round_input(rec.spec, plan, rnd, renders=judged_subset(resolve_paths(ws, rnd.renders)), gates=rnd.gates,- previous=previous_verdict([load_round(ws, rec, rnd.index - 1)], rnd.index), extra_context=extra_context_for(ws, rec, plan, rnd),- geometry_views=clay_geometry_views(ws, rnd.index), glb_path=stored_glb_path(ws, rnd))---def stored_glb_path(ws: Workspace, rnd: RoundRecord) -> str | None:- """The round's own GLB (never the canonical one, which is the last build's)."""- from c3d_flywheel.select import round_file-- glb = round_file(ws, rnd)- return str(glb) if glb is not None else None---def make_judge(spec: Spec, rubric_name: str, model_id: str, n: int, **options: Any) -> Any:- from codeverse3d.judges import vlm_judge- from codeverse3d.judges.rubrics import load_rubric-- try:- rubric = load_rubric(rubric_name)- except Exception as e: # unknown rubric name / bad yaml- raise C.CliError(f"cannot load rubric {rubric_name!r}: {e}") from e- cls = vlm_judge.judge_for(spec.track, rubric, references=bool(spec.references))- return cls(model_id=model_id, n_samples=n, rubric=rubric_name, **options)diff --git a/flywheel/c3d_flywheel/cli/inspect_cmd.py b/flywheel/c3d_flywheel/cli/inspect_cmd.pydeleted file mode 100644index 63c098e0..00000000--- a/flywheel/c3d_flywheel/cli/inspect_cmd.py+++ /dev/null@@ -1,88 +0,0 @@-"""``3dcodeverse judge`` — re-judge a round of one existing run into ``artifacts/judge/rNN_cli.json``."""--from __future__ import annotations--from pathlib import Path-from typing import Annotated--import typer--from codeverse3d.cli import _common as C-from codeverse3d.cli._common import RunsDirOpt, console, kv_table-from codeverse3d.config import get_settings---def judge(- slug: str,- round_index: Annotated[int | None, typer.Option("--round")] = None,- rubric: Annotated[- str | None, typer.Option("--rubric", help="default: the rubric the round was judged with")- ] = None,- model: Annotated[str | None, typer.Option("--model", help="judge chat model id")] = None,- n: Annotated[int | None, typer.Option("--n", min=1, help="samples (default: Settings.judge.samples)")] = None,- thinking: Annotated[- str | None, typer.Option("--thinking", help="off | low | medium | high (default: Settings.judge.thinking, "- "C3D_JUDGE__THINKING; gemini-4-argon thinks only from medium)"),- ] = None,- runs_dir: RunsDirOpt = None,-) -> None:- """Re-judge a round with the same inputs as the in-run judge; writes artifacts/judge/rNN_cli.json."""- from c3d_flywheel import select- from codeverse3d.cli import _judge as J- from c3d_flywheel.record.record import RecordError, join_post_run, load_record-- ws = C.open_workspace(slug, runs_dir)- try:- rec = load_record(ws)- except RecordError as e:- raise C.CliError(str(e), code=2) from e- idx = round_index if round_index is not None else select.summarise(ws.root, record=rec).round- if idx is None:- raise C.CliError(f"no round of {ws.root.name} built with renders: nothing to judge (--round N names one)")- rnd = J.load_round(ws, rec, idx)- if rnd is None or rnd.renders is None or not rnd.renders.views:- raise C.CliError(f"round {idx} has no renders (rounds/r{idx:02d}.json / record.json)")- rubric_name = J.rubric_for(rec, rnd, rubric)- inp = J.build_judge_input(ws, rec, rnd)- missing = [v.path for v in inp.renders.views if not Path(v.path).is_file()]- if missing:- raise C.CliError(f"round {idx}: {len(missing)} of its {len(inp.renders.views)} judged renders are not on disk "- f"(e.g. {missing[0]}): nothing to re-judge it from", code=2)- try:- judge_obj = J.make_judge(rec.spec, rubric_name, model or rec.spec.backends.judge,- n or get_settings().judge.samples,- **({"thinking": thinking} if thinking else {}))- except ValueError as e:- raise C.CliError(str(e), code=2) from e- # writes into the run: one writer per run dir- with C.mutating(ws, what=f"3dcodeverse judge {ws.root.name}", action="judge"):- try:- with join_post_run(ws):- verdict = judge_obj.judge(inp)- except ValueError as e:- raise C.CliError(f"judge failed: {e}") from e- except Exception as e:- from c3d_flywheel.vlm_judge import ReferenceJudgeError-- if isinstance(e, ReferenceJudgeError):- raise C.CliError(f"judge failed: {e}") from e- raise- ws.write_json(ws.judge_path(idx, "_cli"), verdict)- console.print(- kv_table(- "judgment",- {- "rubric": verdict.rubric,- "round": idx,- "overall": f"{verdict.overall:.3f}",- "passed": verdict.passed,- "std": verdict.score_std,- "thinking": getattr(judge_obj, "thinking", "-"),- "stored score": f"{rnd.judgment.overall:.3f}" if rnd.judgment else "-",- "summary": verdict.summary,- "cost": f"${verdict.usage.cost_usd:.4f}",- },- )- )- for it in verdict.improvement_plan:- console.print(f" [{it.priority}] {it.target}: {it.instruction}")diff --git a/flywheel/c3d_flywheel/cost/__init__.py b/flywheel/c3d_flywheel/cost/__init__.pyindex eaf26a17..0836d8b1 100644--- a/flywheel/c3d_flywheel/cost/__init__.py+++ b/flywheel/c3d_flywheel/cost/__init__.py@@ -1,20 +1,10 @@-"""Cost accounting: a typed ledger of every model call, and a price-provenance layer on-top of ``models/pricing.py``. (The audit of FINISHED runs and-its report are ``c3d_flywheel.costreport``.)--This package re-exports only the names other packages actually import through it.-Everything else lives in — and is imported from — its own module: ``cost.instrument``-(metered_chat_model/MeteredAgent), ``cost.types`` (CallCost/CostBucket/Summary), ``cost.tally`` (the step clock).-Reading the ledgers of finished runs back is ``c3d_flywheel.costreport``.--Nothing here imports tracks/ or orchestrator/, so any layer may use it.-"""+"""Reading a run's cost ledger (``telemetry/cost.jsonl``) back: ``ledger`` (rows, re-pricing, buckets),+``types`` (CallCost / CostBucket / Summary, the stage → role map), ``pricing`` (the USD table + provenance).+The audit over many runs and its report are ``c3d_flywheel.costreport``."""from __future__ import annotations-from c3d_flywheel.cost.context import call_context-from c3d_flywheel.cost.instrument import run_ledger-from c3d_flywheel.cost.ledger import load_ledger, record_call+from c3d_flywheel.cost.ledger import load_ledgerfrom c3d_flywheel.cost.types import Role, Stage-__all__ = ["Role", "Stage", "call_context", "load_ledger", "record_call", "run_ledger"]+__all__ = ["Role", "Stage", "load_ledger"]diff --git a/flywheel/c3d_flywheel/cost/context.py b/flywheel/c3d_flywheel/cost/context.pydeleted file mode 100644index e84d7164..00000000--- a/flywheel/c3d_flywheel/cost/context.py+++ /dev/null@@ -1,169 +0,0 @@-"""Where a call sits in a run: the attribution the ledger needs.--Precedence: explicit arguments > a label naming a job of its own (:data:`SELF_DESCRIBING`)-> the ambient context (:func:`call_context`) > the rest of the label. The run binding-(:func:`bound_run`) is a plain ``ContextVar`` copied into fan-out workers, with no-process-wide fallback.-"""--from __future__ import annotations--from collections.abc import Iterator-from contextlib import contextmanager-from contextvars import ContextVar-from dataclasses import dataclass, replace--from c3d_flywheel.cost.types import Role, Stage, role_for_stage, stage_for_label---@dataclass(frozen=True)-class CallContext:- """Attribution for one call (all fields optional; ``None`` = "not stated")."""-- run: str = ""- round: int | None = None- stage: Stage | None = None- role: Role | None = None- label: str = ""-- def merge(self, other: CallContext) -> CallContext:- """``other`` wins field by field where it says something."""- return CallContext(- run=other.run or self.run,- round=self.round if other.round is None else other.round,- stage=other.stage or self.stage,- role=other.role or self.role,- label=other.label or self.label,- )-- def resolved(self) -> CallContext:- """Fill role from stage (and stage from label) so the row is never ``other`` needlessly."""- stage = self.stage or (stage_for_label(self.label) if self.label else None)- role = self.role or (role_for_stage(stage) if stage else None)- return replace(self, stage=stage, role=role)---_EMPTY = CallContext()-_ctx: ContextVar[CallContext] = ContextVar("c3d_cost_ctx", default=_EMPTY)--#: the run this execution context belongs to ("" = nothing bound here)-_run_var: ContextVar[str] = ContextVar("c3d_cost_run", default="")---@contextmanager-def bound_run(run: str) -> Iterator[None]:- """Name the run every row written inside the block belongs to (concurrent blocks unwind independently)."""- token = _run_var.set(run or "")- try:- yield- finally:- _run_var.reset(token)---def run_binding() -> CallContext:- """The run bound in this execution context (``run=""`` when none is)."""- return CallContext(run=_run_var.get())---def current() -> CallContext:- """Run binding merged with the ambient (thread-local) context."""- return run_binding().merge(_ctx.get())---@contextmanager-def call_context(- *,- run: str = "",- round: int | None = None, # noqa: A002 - matches the record/round vocabulary- stage: Stage | str | None = None,- role: Role | str | None = None,- label: str = "",-) -> Iterator[CallContext]:- """Set the ambient attribution for the block; nesting merges."""- ctx = _ctx.get().merge(CallContext(run=run, round=round, stage=_stage(stage), role=_role(role), label=label))- token = _ctx.set(ctx)- try:- yield ctx- finally:- _ctx.reset(token)---def _stage(value: Stage | str | None) -> Stage | None:- if value is None or isinstance(value, Stage):- return value- try:- return Stage(value)- except ValueError:- return stage_for_label(str(value))---def _role(value: Role | str | None) -> Role | None:- if value is None or isinstance(value, Role):- return value- try:- return Role(value)- except ValueError:- return None---# --------------------------------------------------------------------------- labels-#: request-label prefix → (stage, role); longest prefix wins, the rest goes through ``stage_for_label``.-_LABEL_HINTS: tuple[tuple[str, Stage, Role], ...] = (- ("planner", Stage.PLAN, Role.PLANNER),- ("plan", Stage.PLAN, Role.PLANNER),- ("judge", Stage.JUDGE, Role.JUDGE),- ("caption", Stage.CAPTION, Role.CAPTIONER),- ("calibrate", Stage.JUDGE, Role.JUDGE),-)---def context_from_label(label: str) -> CallContext:- """Best-effort attribution from a ``ChatRequest.label`` (head names the job, ``r<NN>`` the round)."""- low = (label or "").strip()- if not low:- return _EMPTY- round_index = _round_from_label(low)- head = low.split(":", 1)[0].lower()- best: tuple[int, Stage, Role] | None = None- for prefix, stage, role in _LABEL_HINTS:- if head.startswith(prefix) and (best is None or len(prefix) > best[0]):- best = (len(prefix), stage, role)- if best is not None:- return CallContext(round=round_index, stage=best[1], role=best[2], label=low)- return CallContext(round=round_index, stage=_stated(stage_for_label(head)), label=low)---def _stated(stage: Stage | None) -> Stage | None:- """``Stage.OTHER`` states nothing, so it must not outrank the ambient context."""- return None if stage is Stage.OTHER else stage---def _round_from_label(label: str) -> int | None:- """``judge:static_object:r02:s1`` → 2 (``None`` when the label has no round tag)."""- for token in label.replace("/", ":").split(":"):- t = token.strip()- if len(t) >= 2 and t[0] == "r" and t[1:].isdigit():- return int(t[1:])- return None---#: Stages whose label names a job of its own: they outrank the ambient session context.-SELF_DESCRIBING: frozenset[Stage] = frozenset(- {Stage.PLAN, Stage.JUDGE, Stage.CAPTION})---def attribute(explicit: CallContext | None = None, *, label: str = "") -> CallContext:- """The final attribution for one call (precedence in the module docstring)."""- ctx = current()- if label:- from_label = context_from_label(label)- ctx = ctx.merge(from_label) if from_label.stage in SELF_DESCRIBING else from_label.merge(ctx)- if explicit is not None:- ctx = ctx.merge(explicit)- if label and not (explicit is not None and explicit.label):- ctx = replace(ctx, label=label)- return ctx.resolved()---__all__ = ["CallContext", "attribute", "bound_run", "call_context", "context_from_label", "current",- "run_binding"]diff --git a/flywheel/c3d_flywheel/cost/guard.py b/flywheel/c3d_flywheel/cost/guard.pydeleted file mode 100644index 5bab3180..00000000--- a/flywheel/c3d_flywheel/cost/guard.py+++ /dev/null@@ -1,14 +0,0 @@-"""Prompt-size estimate (~4 chars/token, +-15%) for the skills' token accounting."""--from __future__ import annotations--from collections.abc import Iterable--CHARS_PER_TOKEN = 4.0---def text_tokens(text: str | Iterable[str]) -> int:- """Estimated prompt tokens for one string or a bag of blocks."""- if isinstance(text, str):- return int(len(text) / CHARS_PER_TOKEN + 0.5)- return sum(text_tokens(t) for t in text)diff --git a/flywheel/c3d_flywheel/cost/instrument.py b/flywheel/c3d_flywheel/cost/instrument.pydeleted file mode 100644index 06e9bc84..00000000--- a/flywheel/c3d_flywheel/cost/instrument.py+++ /dev/null@@ -1,176 +0,0 @@-"""Meter every model call and agent session into the ledger.--:class:`MeteredChatModel` appends one ``CallCost`` row per ``generate``; :class:`MeteredAgent`-sets the attribution for a session and records its one session row. Accounting never fails-a run: ledger errors are logged and swallowed, and other attributes forward to the real object.-"""--from __future__ import annotations--import logging-import time-from collections.abc import Iterator-from contextlib import contextmanager-from pathlib import Path-from typing import Any--from codeverse3d.contracts.agent import AgentJob, AgentResult-from codeverse3d.contracts.chat import ChatRequest, ChatResponse-from codeverse3d.contracts.common import Usage-from c3d_flywheel.cost.context import (- bound_run,- call_context,-)-from c3d_flywheel.cost.ledger import (- CostLedger,- bound_ledger,- existing_ledger_path,- open_run_ledger,- record_call,-)-from c3d_flywheel.cost.tally import book_time-from c3d_flywheel.cost.types import Role, Stage, stage_for_label--log = logging.getLogger(__name__)---class _Proxy:-- def __init__(self, inner: Any):- self._inner = inner-- @property- def model(self) -> str:- return getattr(self._inner, "model", "")-- @property- def id(self) -> str:- return str(self._inner.id)-- def __getattr__(self, name: str) -> Any: # everything else is the real object's- return getattr(self._inner, name)-- def __repr__(self) -> str: # pragma: no cover - debugging aid- return f"{type(self).__name__}({self._inner!r})"---class MeteredChatModel(_Proxy):-- @property- def provider(self) -> str:- return getattr(self._inner, "provider", "")-- # -- the meter --------------------------------------------------------- def generate(self, request: ChatRequest) -> ChatResponse:- t0 = time.perf_counter()- try:- resp = self._inner.generate(request)- except BaseException as exc: # noqa: BLE001 - record the attempt, then re-raise- # a failed call may still have been billed (ModelError.usage)- ms = int((time.perf_counter() - t0) * 1000)- usage = getattr(exc, "usage", None)- usage = usage if isinstance(usage, Usage) else Usage(backend=self.provider, model=self.model)- book_time(ms / 1000, _lost_s(ms, usage.latency_ms))- self._record(usage, request, outcome=_outcome(exc), ms=ms, attempts=getattr(exc, "attempts", 0))- raise- ms = int((time.perf_counter() - t0) * 1000)- book_time(ms / 1000, _lost_s(ms, resp.usage.latency_ms))- self._record(resp.usage, request, outcome="ok", ms=ms,- key=resp.raw.get("key"), attempts=resp.raw.get("attempts", 0))- return resp-- def _record(self, usage: Usage, request: ChatRequest, *, outcome: str, ms: int,- key: object = None, attempts: object = 0) -> None:- try:- record_call(usage, label=request.label, backend=usage.backend or self.provider,- model=usage.model or self.model, outcome=outcome,- latency_ms=usage.latency_ms or ms,- key=_key_suffix(key), attempts=int(attempts or 0))- except Exception as e: # pragma: no cover - accounting must never break a call- log.debug("cost: could not record %s: %s", request.label, e)---def _lost_s(wall_ms: int, answered_ms: int) -> float:- """Seconds lost to provider errors: the call's wall time outside the answered round-trips."""- return max(0, wall_ms - answered_ms) / 1000---def _key_suffix(key: object) -> str:- """The last 4 chars of an API key as ``…ab12`` — the ledger never holds more."""- s = str(key or "").strip()- return f"…{s[-4:]}" if s else ""---def _outcome(exc: BaseException) -> str:- name = type(exc).__name__.lower()- if "timeout" in name:- return "timeout"- return "error"---class MeteredAgent(_Proxy):- """CodingAgent proxy: attribution for the session + its one ledger row."""-- @property- def kind(self) -> str:- return str(getattr(self._inner, "kind", ""))-- def run(self, job: AgentJob) -> AgentResult:- stage = stage_for_label(job.kind or job.label)- t0 = time.perf_counter()- with call_context(round=job.round, stage=stage, role=Role.GENERATOR, label=job.label):- result = self._inner.run(job)- wall_s = time.perf_counter() - t0- book_time(wall_s, float(result.provider_wait_s or 0.0))- self._record_session(job, result, stage, int(wall_s * 1000))- return result-- def _record_session(self, job: AgentJob, result: AgentResult, stage: Stage, ms: int) -> None:- usage = result.usage- if not (usage.input_tokens or usage.output_tokens or usage.cost_usd):- return- try:- record_call(usage, round=job.round, stage=stage, role=Role.GENERATOR, label=job.label,- backend=usage.backend or self.kind, model=usage.model or self.model,- outcome="ok" if result.ok else (result.exit_reason or "error"),- latency_ms=usage.latency_ms or ms,- n_calls=max(1, int(result.tool_calls or 1)),- source="estimate" if result.usage_estimated else "session",- **({"effort": result.effort} if result.effort else {}))- except Exception as e: # pragma: no cover- log.debug("cost: could not record agent session %s: %s", job.label, e)---# --------------------------------------------------------------------------- factories-def metered_chat_model(model: Any) -> Any:- """Wrap ``model`` unless it is already metered. Metering cannot be switched off."""- if model is None or isinstance(model, MeteredChatModel):- return model- return MeteredChatModel(model)---def metered_agent(agent: Any) -> Any:- if agent is None or isinstance(agent, MeteredAgent):- return agent- return MeteredAgent(agent)---# --------------------------------------------------------------------------- run activation-@contextmanager-def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) -> Iterator[CostLedger | None]:- """Meter one run into ``<workspace>/telemetry/cost.jsonl``.-- ``create=False`` (work after a run finished) appends only to an existing ledger, else rows go- to the per-process log, so a re-judge never masquerades as the run's whole cost.- Nests and parallelises: bindings unwind by ``ContextVar`` token."""- ws = Path(workspace)- if not create and existing_ledger_path(ws) is None:- with bound_run(run or ws.name):- yield None- return- ledger = open_run_ledger(ws)- with bound_run(run or ws.name), bound_ledger(ledger.path):- yield ledger---__all__ = ["MeteredAgent", "MeteredChatModel", "metered_agent", "metered_chat_model", "run_ledger"]diff --git a/flywheel/c3d_flywheel/cost/ledger.py b/flywheel/c3d_flywheel/cost/ledger.pyindex 4b5ee851..c5a100bd 100644--- a/flywheel/c3d_flywheel/cost/ledger.py+++ b/flywheel/c3d_flywheel/cost/ledger.py@@ -1,40 +1,26 @@-"""Append-only cost ledger: one JSONL row per model / agent call.--``record_call`` prices and appends a row (never raises); ``load_ledger`` reads rows back-tolerantly; ``summarise`` buckets them by dimension.-"""+"""The cost ledger of a finished run (``telemetry/cost.jsonl``): read tolerantly, re-price, bucket by dimension."""from __future__ import annotations-import jsonimport logging-import os-import threading-import time-from collections.abc import Iterable, Iterator, Sequence-from contextlib import contextmanager-from contextvars import ContextVar+from collections.abc import Iterable, Sequencefrom pathlib import Pathfrom typing import Anyfrom codeverse3d.contracts.common import Usage-from c3d_flywheel.cost.context import CallContext, _stage, attribute-from c3d_flywheel.cost.tally import book_usage+from codeverse3d.proc import iter_jsonl_lines++from c3d_flywheel.cost.pricing import (+ estimate_cost,+ price_provenance,+ unit_prices,+)from c3d_flywheel.cost.types import (CallCost,CostBucket,- Role,- Stage,Summary,normalise_ids,- role_for_stage,-)-from c3d_flywheel.models.pricing import (- estimate_cost,- price_provenance,- unit_prices,)-from codeverse3d.proc import iter_jsonl_lineslog = logging.getLogger(__name__)@@ -48,91 +34,12 @@ DIMENSIONS = ("run", "stage", "role", "backend", "model", "provider", "round", "PROVIDER_PRICED_BACKENDS = frozenset({"claude-code"})-class CostLedger:- """Append-only JSONL ledger. Thread-safe; one line per call, flushed on write."""-- def __init__(self, path: str | Path):- self.path = Path(path)- self._lock = threading.Lock()-- def append(self, row: CallCost) -> CallCost:- line = json.dumps(row.model_dump(mode="json"), ensure_ascii=False, default=str)- try:- self.path.parent.mkdir(parents=True, exist_ok=True)- with self._lock, self.path.open("a", encoding="utf-8") as fh:- fh.write(line + "\n")- fh.flush()- except OSError as e: # pragma: no cover - disk full / read-only run dir- log.warning("cost ledger append failed (%s): %s", self.path, e)- return row---_fallback_lock = threading.Lock()-_fallback: CostLedger | None = None--#: sentinel: "this execution context did not state a ledger"-_UNSET: Any = object()-#: context-local ONLY override of the default ledger, so parallel runs never share a file-_default_var: ContextVar[Any] = ContextVar("c3d_cost_default_ledger", default=_UNSET)---@contextmanager-def bound_ledger(path: str | Path) -> Iterator[CostLedger]:- """Point :func:`record_call` at ``path`` for the block."""- led = CostLedger(path)- token = _default_var.set(led)- try:- yield led- finally:- _default_var.reset(token)---def default_ledger() -> CostLedger:- """The ledger :func:`record_call` writes to: the bound one, else the per-process fallback- log, so a call outside any run is still accounted for. There is no off switch."""- global _fallback- led = _default_var.get()- if led is not _UNSET:- return led- with _fallback_lock:- if _fallback is None:- _fallback = CostLedger(process_ledger_path())- return _fallback---def process_ledger_path() -> Path:- """The fallback log for calls with no run context: one file per process under ``<cache_dir>/cost/``."""- from codeverse3d.config import get_settings-- base = Path(get_settings().cache_dir)- stamp = time.strftime("%Y%m%d", time.localtime())- return base / "cost" / f"p{os.getpid()}-{stamp}.jsonl"--def existing_ledger_path(workspace: str | Path) -> Path | None:"""The run's ``telemetry/cost.jsonl`` if it exists, else ``None``."""p = Path(workspace) / TELEMETRY_LEDGERreturn p if p.is_file() else None-def open_run_ledger(workspace: str | Path) -> CostLedger:- """The run's ledger (the file is created by its first row)."""- return CostLedger(Path(workspace) / TELEMETRY_LEDGER)---#: ``_as_role`` differs from ``context._role``: it returns Role.OTHER (truthy) where that returns None.-_as_stage = _stage---def _as_role(value: Role | str | None) -> Role | None:- if value is None or isinstance(value, Role):- return value- try:- return Role(value)- except ValueError:- return Role.OTHER--def price_call(usage: Usage,*,@@ -173,70 +80,6 @@ def price_call(return cost, fields-def record_call(- usage: Usage | None = None,- *,- run: str = "",- stage: Stage | str | None = None,- role: Role | str | None = None,- round: int | None = None, # noqa: A002 - matches the record/round vocabulary- label: str = "",- backend: str = "",- model: str = "",- outcome: str = "ok",- latency_ms: int | None = None,- n_calls: int = 1,- source: str = "live",- ledger: CostLedger | str | Path | None = None,- ts: float | None = None,- reprice: bool = False,- **extra: Any,-) -> CallCost:- """Append one priced call to the ledger, book it into the open tallies, return the row.-- Unstated ``run`` / ``round`` / ``stage`` / ``role`` resolve from the ambient context and label.- Never raises."""- u = usage or Usage()- ctx = attribute(CallContext(run=run, round=round, stage=_as_stage(stage), role=_as_role(role),- label=label), label=label)- st = ctx.stage or Stage.OTHER- rl = ctx.role or role_for_stage(st)- run, round, label = ctx.run, ctx.round, ctx.label # noqa: A001 - see the signature- try:- cost, price_fields = price_call(u, backend=backend, model=model, reprice=reprice)- except Exception as e: # pragma: no cover - pricing must never break a run- log.warning("cost: pricing failed for %s/%s: %s", backend, model, e)- cost, price_fields = float(u.cost_usd), {}- row = CallCost(- ts=ts if ts is not None else time.time(),- run=run,- round=round,- stage=st,- role=rl,- label=label,- input_tokens=int(u.input_tokens),- cached_tokens=int(u.cached_tokens),- output_tokens=int(u.output_tokens),- thoughts_tokens=int(u.thoughts_tokens),- tool_calls=int(u.tool_calls),- cost_usd=cost,- recorded_usd=float(u.cost_usd),- latency_ms=int(u.latency_ms if latency_ms is None else latency_ms),- cache_hit=int(u.cached_tokens) > 0,- outcome=outcome,- n_calls=max(1, int(n_calls)),- source=source,- **price_fields,- **extra,- )- (ledger if isinstance(ledger, CostLedger) else CostLedger(ledger) if ledger else default_ledger()).append(row)- book_usage(Usage(backend=row.backend, model=row.model, input_tokens=row.input_tokens,- output_tokens=row.output_tokens, cached_tokens=row.cached_tokens,- thoughts_tokens=row.thoughts_tokens, tool_calls=row.tool_calls,- cost_usd=row.cost_usd, latency_ms=row.latency_ms))- return row--def load_ledger(path: str | Path) -> list[CallCost]:"""Read a ledger file, or a run directory holding one; bad lines are skipped."""p = Path(path)diff --git a/flywheel/c3d_flywheel/models/pricing.py b/flywheel/c3d_flywheel/cost/pricing.pysimilarity index 99%rename from flywheel/c3d_flywheel/models/pricing.pyrename to flywheel/c3d_flywheel/cost/pricing.pyindex cbe60d19..c84466b0 100644--- a/flywheel/c3d_flywheel/models/pricing.py+++ b/flywheel/c3d_flywheel/cost/pricing.py@@ -136,7 +136,7 @@ def lookup_price(provider: str, model: str) -> Price | None:@dataclass(frozen=True)class PriceRow:- """A resolved price plus its audit trail — what ``3dcodeverse cost prices`` prints."""+ """A resolved price plus its audit trail — what ``c3d-flywheel cost prices`` prints."""provider: strmodel: strdiff --git a/flywheel/c3d_flywheel/cost/tally.py b/flywheel/c3d_flywheel/cost/tally.pydeleted file mode 100644index e6f6bb6b..00000000--- a/flywheel/c3d_flywheel/cost/tally.py+++ /dev/null@@ -1,83 +0,0 @@-"""What the meters saw inside one block of work: its money and its lost time.--A :class:`Tally` opened by :func:`tally` nests; the ledger books priced rows and the-``cost.instrument`` proxies book wall/lost seconds into every open tally. :func:`timed`-turns a block into a ``StepTime`` row. The binding is a ``ContextVar`` that fan-out copies.-"""--from __future__ import annotations--import threading-import time-from collections.abc import Iterator-from contextlib import contextmanager-from contextvars import ContextVar--from codeverse3d.contracts.common import Usage-from codeverse3d.contracts.run import StepTime---class Tally:-- def __init__(self) -> None:- self.usage = Usage()- self._time: dict[int, list[float]] = {} # thread id → [wall_s, lost_s] of its metered calls- self._lock = threading.Lock()-- def add_usage(self, usage: Usage) -> None:- with self._lock:- self.usage = self.usage + usage-- def add_time(self, wall_s: float, lost_s: float) -> None:- with self._lock:- t = self._time.setdefault(threading.get_ident(), [0.0, 0.0])- t[0] += wall_s- t[1] += min(lost_s, wall_s)-- def lost_s(self) -> float:- """Seconds lost to provider errors: losses add within a thread; across parallel threads- it is ``max(wall) - max(wall - lost)``."""- with self._lock:- if not self._time:- return 0.0- return max(w for w, _ in self._time.values()) - max(w - lost for w, lost in self._time.values())---_open: ContextVar[tuple[Tally, ...]] = ContextVar("c3d_cost_tallies", default=())---@contextmanager-def tally() -> Iterator[Tally]:- t = Tally()- token = _open.set(_open.get() + (t,))- try:- yield t- finally:- _open.reset(token)---def book_usage(usage: Usage) -> None:- for t in _open.get():- t.add_usage(usage)---def book_time(wall_s: float, lost_s: float) -> None:- for t in _open.get():- t.add_time(max(0.0, wall_s), max(0.0, lost_s))---@contextmanager-def timed(step: str, into: list[StepTime], *, round_index: int | None = None) -> Iterator[Tally]:- """Time ``step`` into ``into`` whether the block returns or raises; a ~0 s step leaves no row."""- t0 = time.monotonic()- with tally() as t:- try:- yield t- finally:- wall = time.monotonic() - t0- if wall >= 0.01:- into.append(StepTime(step=step, round=round_index, wall_s=round(wall, 2),- lost_s=round(min(wall, t.lost_s()), 2)))---__all__ = ["Tally", "book_time", "book_usage", "tally", "timed"]diff --git a/flywheel/c3d_flywheel/dataset/captions.py b/flywheel/c3d_flywheel/dataset/captions.pydeleted file mode 100644index f968e336..00000000--- a/flywheel/c3d_flywheel/dataset/captions.py+++ /dev/null@@ -1,189 +0,0 @@-"""Caption one finished run: ``{detailed, instruction, factory}``."""--from __future__ import annotations--import json-import re-from datetime import UTC, datetime-from pathlib import Path--from pydantic import BaseModel, Field--from c3d_flywheel.dataset.sample import code_files_for_round, exported_round-from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart-from codeverse3d.contracts.common import ENTRY_FILE, Language, Track-from codeverse3d.contracts.run import RunRecord-from c3d_flywheel.models import get_chat_model-from codeverse3d.proc import write_json_atomic-from codeverse3d.prompts import prompt_hash-from c3d_flywheel.record import git_history-from codeverse3d.workspace import Workspace--MAX_CODE_CHARS = 12_000-N_VIEWS = 2-MOTION_VIEW_HINTS = ("articulation", "pose_")--LANGUAGE_PHRASE: dict[Language, tuple[str, str]] = {- Language.BLENDER: ("a Blender Python script", "blender"),- Language.CADQUERY: ("a CadQuery script", "cadquery"),- Language.THREEJS: ("a Three.js module", "three.js"),- Language.URDF_BLENDER: ("a URDF model with Blender-Python-built link meshes", "urdf"),- Language.SCENE_THREEJS: ("a Three.js scene", "three.js"),- Language.SCENE_BLENDER: ("a Blender Python scene", "blender"),- Language.GLSL_SHADER: ("a GLSL fragment shader", "glsl"),- Language.OPENGL_PYTHON: ("an OpenGL Python program", "opengl"),-}-#: leaked API / platform tokens banned from captions; ``THREE.`` is checked case-sensitively because "Three.js" is required.-_FORBIDDEN = re.compile(- r"\b(bpy|bmesh|mathutils|cq\.|cadquery\.|GLTFLoader|ShaderMaterial|MeshStandardMaterial|"- r"Workplane|moderngl|Shadertoy|Sketchfab|Gemini|Claude|OpenAI|GPT|Antigravity)\b",- re.IGNORECASE,-)-_FORBIDDEN_THREE_NS = re.compile(r"\bTHREE\.(?!js\b)")---class Captions(BaseModel):- detailed: str = Field(min_length=20, description="objective, image-grounded description (2-5 sentences)")- instruction: str = Field(min_length=15, description="one natural user request asking to generate this as code")- factory: str = Field(min_length=20, description="code-grounded build recipe: parts + construction order")---class CaptionProvenance(BaseModel):- captioner: str- round_index: int | None- images_used: list[str]- code_chars: int- prompt_hash: str- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))- cost_usd: float = 0.0---class CaptionError(RuntimeError):- pass---def _system_prompt() -> str:- return (- "You label 3D assets for a code-generation dataset. You receive rendered views of an asset, "- "the user's original request, and the raw source code that built it. Return STRICT JSON with "- "exactly the keys detailed, instruction, factory (string values).\n"- "- detailed: an objective description of what is VISIBLE in the renders: the object/scene, its "- "main parts, proportions, materials/colours, and — if it articulates or animates — how its parts "- "move. 2-5 sentences, grounded in the images.\n"- "- instruction: ONE natural user request (1-2 sentences, imperative) asking an AI to generate this "- "asset as {language_phrase}; mention the key parts/requirements a user would state.\n"- "- factory: a technical build recipe (3-6 sentences) derived from the CODE: which parts are built, "- "in what order, from which primitives/operations (extrusions, booleans, lofts, instancing, joints, "- "shader effects...), how they are positioned and finished.\n"- "Rules: be specific to THIS asset; describe shapes in plain words; do NOT mention API, module, "- "class or function names, file names, model/vendor names or platforms; no markdown."- )---def _round_images(ws: Workspace, record: RunRecord) -> tuple[list[ImagePart], list[str]]:- rnd = exported_round(ws, record)- images: list[ImagePart] = []- used: list[str] = []- if rnd is None or rnd.renders is None:- return images, used-- def _add(path: str | None, label: str) -> None:- if not path or len(images) >= N_VIEWS + 1:- return- p = ws.rebase(path)- if p.is_file():- images.append(ImagePart(path=str(p), label=label))- used.append(p.relative_to(ws.root).as_posix() if p.is_relative_to(ws.root) else str(p))-- _add(rnd.renders.contact_sheet, "contact sheet (all views, labelled)")- views = list(rnd.renders.views)- motion = [v for v in views if any(h in v.name for h in MOTION_VIEW_HINTS)]- for v in (motion + [v for v in views if v not in motion])[:N_VIEWS]:- _add(v.path, f"view: {v.name}")- return images, used---def _code_excerpt(ws: Workspace, record: RunRecord) -> str:- rnd = exported_round(ws, record)- files, _src = code_files_for_round(ws, rnd)- entry = ENTRY_FILE[record.spec.language]- ordered = {k: files[k] for k in sorted(files, key=lambda k: (k != entry, k))}- text, _skipped = git_history.decode_text_files(ordered, max_total=MAX_CODE_CHARS)- return "\n\n".join(f"### {p}\n{t}" for p, t in text.items())---def _user_prompt(record: RunRecord, code: str) -> str:- kind = {Track.STATIC_OBJECT: "static object", Track.ARTICULATED_OBJECT: "articulated object",- Track.SCENE: "scene", Track.GRAPHICS: "animated procedural graphics (judged from sampled frames)"}[record.spec.track]- return (- f"Asset kind: {kind}.\nOriginal user request: {record.spec.prompt}\n\n"- f"Source code (raw, may be truncated):\n{code}\n\nReturn the JSON now."- )---def validate_captions(caps: Captions, language: Language) -> list[str]:- problems = []- for field in ("detailed", "instruction", "factory"):- m = _FORBIDDEN.search(getattr(caps, field)) or _FORBIDDEN_THREE_NS.search(getattr(caps, field))- if m:- problems.append(f"{field}: must not mention '{m.group(0)}' (API/platform name)")- phrase, word = LANGUAGE_PHRASE[language]- if word not in caps.instruction.lower():- problems.append(f"instruction: must name the target language ({phrase})")- return problems---def caption_sample(- ws: Workspace, record: RunRecord, model_id: str, *, model: object | None = None,- out_dir: Path | str | None = None, slug: str | None = None-) -> Captions:- if model is None:- model = get_chat_model(model_id)- images, used = _round_images(ws, record)- code = _code_excerpt(ws, record)- if not code and not images:- raise CaptionError(f"{ws.root}: no code and no renders to caption")- system = _system_prompt().replace("{language_phrase}", LANGUAGE_PHRASE[record.spec.language][0])- user = _user_prompt(record, code)- messages = [ChatMessage.user(user, images=images)]- schema = Captions.model_json_schema()- cost = 0.0- caps: Captions | None = None- problems: list[str] = []- for _attempt in range(2):- resp = model.generate(ChatRequest(messages=messages, system=system, response_schema=schema,- temperature=0.3, thinking="low", max_wait_s=900.0, label="captioner")) # type: ignore[attr-defined]- cost += resp.usage.cost_usd- data = resp.parsed- try:- caps = Captions.model_validate(data)- except Exception as e:- problems = [f"invalid JSON/schema: {e}"]- else:- problems = validate_captions(caps, record.spec.language)- if not problems:- break- messages = messages + [ChatMessage.assistant(resp.text or json.dumps(data)),- ChatMessage.user("Fix these problems and return the JSON again:\n- " + "\n- ".join(problems))]- if caps is None or problems:- raise CaptionError(f"{ws.root}: captions rejected after retry: {problems}")- rnd = exported_round(ws, record)- prov = CaptionProvenance(captioner=model_id, round_index=rnd.index if rnd else None,- images_used=used, code_chars=len(code),- prompt_hash=prompt_hash(system), cost_usd=cost)- payload = {**caps.model_dump(), "provenance": prov.model_dump(mode="json")}- record.extra["captions"] = payload- if out_dir is not None:- write_json_atomic(Path(out_dir) / f"{slug or ws.root.name}.json", payload)- return caps- from c3d_flywheel.record.deliverable import build_deliverable, load_deliverable- from c3d_flywheel.record.record import package_run-- ws.write_json(ws.root / "captions.json", payload)- package_run(ws, record)- ws.write_json(ws.record_path, record)- handed = load_deliverable(ws)- if handed is not None and handed.round is not None:- build_deliverable(ws, record, handed.round)- return caps-diff --git a/flywheel/c3d_flywheel/legacy.py b/flywheel/c3d_flywheel/legacy.pynew file mode 100644index 00000000..99c0f13c--- /dev/null+++ b/flywheel/c3d_flywheel/legacy.py@@ -0,0 +1,273 @@+"""The v0.0.x run record (``record.json``) and everything stored inside it, read leniently.++The plugin no longer writes runs: these models exist to READ the recorded ones. Spec, plan and the+artifact blocks are tolerant of what later versions dropped (``Budget``/``Backends`` were removed+from the plugin's contracts; the plan is read as plain data)."""++from __future__ import annotations++from datetime import UTC, datetime+from enum import StrEnum+from pathlib import PurePosixPath+from typing import Any++from codeverse3d.contracts.artifacts import (+ BuildResult,+ GateReport,+ Judgment,+ Measurement,+ RenderSet,+)+from codeverse3d.contracts.common import Usage+from codeverse3d.contracts.spec import Spec+from pydantic import BaseModel, ConfigDict, Field+++# ===================================================================== spec + plan (as recorded)+class Budget(BaseModel):+ """The v0.0.x run ceilings (the plugin has no budget)."""++ max_rounds: int = 4+ max_minutes: float = 60.0+ max_repair_attempts: int = 3+++class Backends(BaseModel):+ """Who did which job in a v0.0.x run (``<kind>:<model>`` ids)."""++ planner: str = "gemini:gemini-4-argon"+ generator: str = "agy:gemini-4-argon"+ judge: str = "gemini:gemini-4-argon"+ captioner: str = "gemini:gemini-4-argon"+++class LegacySpec(Spec):+ budget: Budget = Field(default_factory=Budget)+ backends: Backends = Field(default_factory=Backends)+ tags: list[str] = Field(default_factory=list)+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))+++class AcceptanceItem(BaseModel):+ model_config = ConfigDict(extra="ignore")++ id: str+ text: str = ""+ how: str = "visual"+ priority: str = "must"+++class LegacyPlan(BaseModel):+ """The plan as the gallery and the dataset read it: a name, a part count, the acceptance checklist."""++ model_config = ConfigDict(extra="ignore")++ object_name: str = ""+ title: str = ""+ parts: list[Any] = Field(default_factory=list)+ acceptance: list[AcceptanceItem] = Field(default_factory=list)+++# ===================================================================== run+# --------------------------------------------------------------------------- identity+#: battery-layout path segments that are pure plumbing, never part of a run's identity+RUN_PATH_NOISE = frozenset({"arms", "cells", "runs"})+++class RunId(BaseModel):+ """Where a run directory sits inside its scan root, and the one name to call it.++ ``battery`` labels the scan root; ``rel`` is the run dir's posix path relative to it.+ Only the scanner knows the root, so only it mints one (``record.record.run_id_for``)."""++ model_config = ConfigDict(frozen=True)++ battery: str = Field(description="label of the scan root this run was found under")+ rel: str = Field(description="posix path of the run dir relative to the scan root")++ def _parts(self) -> tuple[str, ...]:+ return PurePosixPath(self.rel).parts++ @property+ def slug(self) -> str:+ """The stable, filesystem-safe name of this run.++ A flat ``rel`` keeps its basename; a nested one joins its meaningful segments with ``__``+ (dropping :data:`RUN_PATH_NOISE` and a trailing ``run``)."""+ parts = self._parts()+ if len(parts) <= 1:+ return parts[0] if parts else self.battery+ meaningful = [p for p in parts if p not in RUN_PATH_NOISE]+ if len(meaningful) > 1 and meaningful[-1] == "run":+ meaningful.pop() # the literal `run` dir a battery cell wraps its harness run in+ if not meaningful:+ meaningful = [parts[-1]]+ return "__".join(meaningful)+++class RunStatus(StrEnum):+ """Where a run is, and once stopped, why it stopped. Never a verdict."""++ PLANNING = "planning"+ GENERATING = "generating"+ REFINING = "refining"+ MAX_ROUNDS = "max_rounds" # every round the budget asked for ran+ BUDGET = "budget" # the wall clock (``max_minutes``) ran out+ AGENT_QUOTA = "agent_quota" # the vendor's usage limit, not ours+ NO_CHANGE = "no_change" # a refine round's sessions all failed / changed nothing+ NO_REFINE_TASKS = "no_refine_tasks" # the last round left nothing to ask for+ JUDGE_UNAVAILABLE = "judge_unavailable" # the last round has no verdict, even after a re-judge+ FAILED = "failed"+++class StepTime(BaseModel):+ """One timed step: its wall clock and the part lost to provider errors (503 / 429 / timeouts).++ Parallel work is ONE step, so no second is counted twice."""++ step: str = Field(description="plan | skeleton | assets+env+layouts | zones | assemble"+ " | generate | build | gates | render | judge")+ round: int | None = Field(default=None, description="the round the step belongs to; None = a run-level stage")+ wall_s: float = 0.0+ lost_s: float = Field(default=0.0, description="seconds of wall_s lost to provider errors")++ @property+ def active_s(self) -> float:+ return max(0.0, self.wall_s - self.lost_s)+++class RoundRecord(BaseModel):+ index: int+ kind: str = Field(description="baseline | refine | repair | asset:<name> ...")+ commit: str = Field(default="", description="git commit of src/ after this round")+ agent_backend: str = ""+ instructions: list[str] = Field(default_factory=list, description="what the agent was asked to change")+ build: BuildResult | None = None+ gates: list[GateReport] = Field(default_factory=list)+ measurement: Measurement | None = None+ renders: RenderSet | None = None+ judgment: Judgment | None = None+ usage: Usage = Field(default_factory=Usage)+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))+ duration_s: float = Field(default=0.0, description="the round's clock, provider errors included")+ steps: list[StepTime] = Field(+ default_factory=list, description="the round's timed steps (generate / build / gates / render / judge); "+ "empty until timing is recorded")+ notes: str = ""++ @property+ def score(self) -> float | None:+ return self.judgment.overall if self.judgment else None++ @property+ def minutes(self) -> float:+ """The round's timed steps, excluding provider wait."""+ return sum(s.active_s for s in self.steps) / 60.0+++# --------------------------------------------------------------------------- telemetry (settings + cost)+class RoleSettings(BaseModel):+ """Resolved model settings for ONE role of a run.++ ``source``: ``run`` (stamped as it ran) or ``spec`` (only the model is known)."""++ role: str+ model: str = ""+ backend: str = Field(default="", description="gemini | anthropic | agy | claude-code | codex | …")+ thinking: str = Field(default="", description="off | low | medium | high; '' = not recorded")+ temperature: float | None = None+ n_samples: int | None = Field(default=None, description="judge samples per verdict")+ source: str = "spec"+++class SettingsSnapshot(BaseModel):+ """Everything that decided how the run ran."""++ roles: list[RoleSettings] = Field(default_factory=list)+ budget: dict[str, Any] = Field(default_factory=dict, description="max_rounds / max_minutes / max_repair_attempts")+ rubric: str = ""+ rubric_hash: str = Field(default="", description="content hash of the judge rubric YAML")+ prompt_hashes: dict[str, str] = Field(default_factory=dict, description="contract / generate / refine / repair / skill:<name> / judge …")+ tool_versions: dict[str, str] = Field(default_factory=dict, description="python / blender / node / three / chrome / harness sha")+ harness_version: str = ""+ harness_git_sha: str = ""+ key_pool_size: int = Field(default=0, description="number of API keys in the rotating pool")+ price_table_version: str = Field(default="", description="content hash of the price table the run was billed with")+ render: dict[str, Any] = Field(default_factory=dict, description="resolved render settings (size, gpu, sheet)")+++class StageCost(BaseModel):++ stage: str+ calls: int = 0+ input_tokens: int = 0+ output_tokens: int = 0+ cached_tokens: int = 0+ thoughts_tokens: int = 0+ cost_usd: float = 0.0+ seconds: float = 0.0+++class CostSummary(BaseModel):+ """The run's ledger rows summarised (``record.telemetry.cost``)."""++ total_usd: float = Field(default=0.0, description="the ledger's sum at list price = record.total_usage")+ minutes: float | None = Field(default=None, description="RunRecord.minutes when the record was written")+ max_minutes: float = 0.0+ n_calls: int = 0+ tokens: Usage = Field(default_factory=Usage, description="run totals (record.total_usage)")+ by_stage: list[StageCost] = Field(default_factory=list)+ by_role: dict[str, float] = Field(default_factory=dict, description="planner/generator/judge/… → USD")+ by_model: dict[str, float] = Field(default_factory=dict, description="model → USD")+ by_round: list[dict[str, Any]] = Field(default_factory=list, description="index / kind / cost_usd / minutes / score")+++class RunTelemetry(BaseModel):++ settings: SettingsSnapshot | None = None+ cost: CostSummary | None = None+++# --------------------------------------------------------------------------- deliverable+class DeliverableFile(BaseModel):++ path: str = Field(description="run-relative path (under deliverable/)")+ role: str = Field(description="code | model | urdf | mesh | sheet | preview | frames | captions | bundle | manifest")+ bytes: int = 0+ sha256: str = ""+++class RunDeliverable(BaseModel):+ """``deliverable/manifest.json``: the handed-over round as paths + hashes + sizes."""++ round: int | None = Field(default=None, description="the packaged round")+ commit: str = ""+ entry: str = Field(default="", description="run-relative entry file of the code snapshot")+ files: list[DeliverableFile] = Field(default_factory=list)+++class RunRecord(BaseModel):+ """One run."""++ spec: LegacySpec+ plan: LegacyPlan | None = None+ workspace: str+ status: RunStatus = RunStatus.PLANNING+ rounds: list[RoundRecord] = Field(default_factory=list)+ total_usage: Usage = Field(default_factory=Usage)+ environment: dict[str, str] = Field(default_factory=dict, description="tool versions, git sha, host")+ prompt_hashes: dict[str, str] = Field(default_factory=dict)+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))+ finished_at: datetime | None = None+ error: str = ""+ telemetry: RunTelemetry | None = Field(+ default=None, description="settings snapshot + cost summary (written by finalize_record)")+ steps: list[StepTime] = Field(+ default_factory=list, description="the run-level timed steps (plan, the pre-round stages, best-of-N, the part of "+ "an aborted round that ran)")+ extra: dict[str, Any] = Field(default_factory=dict)++ @property+ def minutes(self) -> float:+ """The run's minutes: ``wall_s - lost_s`` over every step (provider errors excluded)."""+ return sum(s.active_s for s in self.steps) / 60.0 + sum(r.minutes for r in self.rounds)diff --git a/flywheel/c3d_flywheel/models/__init__.py b/flywheel/c3d_flywheel/models/__init__.pydeleted file mode 100644index ddeee53a..00000000--- a/flywheel/c3d_flywheel/models/__init__.py+++ /dev/null@@ -1,5 +0,0 @@-"""ChatModel backends (API). Use ``get_chat_model("gemini:gemini-4-argon")``."""--from c3d_flywheel.models.registry import get_chat_model--__all__ = ["get_chat_model"]diff --git a/flywheel/c3d_flywheel/models/anthropic.py b/flywheel/c3d_flywheel/models/anthropic.pydeleted file mode 100644index 1b42fe5a..00000000--- a/flywheel/c3d_flywheel/models/anthropic.py+++ /dev/null@@ -1,248 +0,0 @@-"""ChatRequest ⇄ Anthropic Messages API dicts (messages, submit tool, thinking, parse).--Adaptive-thinking models take ``thinking={"type":"adaptive"}`` + ``output_config.effort``;-some always think (``disabled`` is a 400) and reject a forced ``tool_choice``, so "off" is-effort ``low``. Older models take ``budget_tokens`` (< max_tokens) and need default temperature.-"""--from __future__ import annotations--import json-import time-from typing import Any--from codeverse3d.config import get_settings-from codeverse3d.contracts.chat import (- ChatMessage,- ChatRequest,- ChatResponse,- ImagePart,-)-from codeverse3d.contracts.common import Usage-from c3d_flywheel.models.base import ModelError-from c3d_flywheel.models.parts import (- SdkModel,- Stopwatch,- attempt_timeout_s,- classify_sdk_exception,- image_b64,- message_blocks,- retry_budget_s,- retry_one_key,-)-from c3d_flywheel.models.pricing import cache_write_surcharge, estimate_cost-from c3d_flywheel.models.schema_utils import (- JsonParseError,- parse_json_lenient,- strip_control_chars,- to_anthropic_schema,-)--SUBMIT_TOOL = "submit"-THINKING_BUDGET: dict[str, int] = {"off": 0, "low": 1024, "medium": 4096, "high": 16000}-EFFORT: dict[str, str] = {"off": "low", "low": "low", "medium": "medium", "high": "high"}--_ADAPTIVE_MARKERS = (- "opus-5",- "sonnet-5",- "haiku-5",- "fable-5",- "mythos",- "opus-4-6",- "opus-4-7",- "opus-4-8",- "sonnet-4-6",-)-_ALWAYS_THINKING = ("fable-5", "mythos", "opus-5-5", "sonnet-5-5")-_NO_SAMPLING = ("opus-5", "sonnet-5", "haiku-5", "fable-5", "mythos", "opus-4-7", "opus-4-8")---def uses_adaptive_thinking(model: str) -> bool:- m = model.lower()- return any(tok in m for tok in _ADAPTIVE_MARKERS)---def thinking_always_on(model: str) -> bool:- m = model.lower()- return any(tok in m for tok in _ALWAYS_THINKING)---def sampling_allowed(model: str) -> bool:- m = model.lower()- return not any(tok in m for tok in _NO_SAMPLING)---# ------------------------------------------------------------------ messages-def _image_block(p: ImagePart) -> dict[str, Any]:- data, mime = image_b64(p)- return {"type": "image", "source": {"type": "base64", "media_type": mime, "data": data}}---def to_messages(messages: list[ChatMessage]) -> list[dict[str, Any]]:- """ChatMessages → Anthropic ``messages``; consecutive same-role turns are merged."""- out: list[dict[str, Any]] = []- for msg in messages:- role = "assistant" if msg.role == "assistant" else "user"- blocks = message_blocks(msg, text=lambda t: {"type": "text", "text": t}, image=_image_block)- if not blocks:- continue- if out and out[-1]["role"] == role:- out[-1]["content"].extend(blocks)- else:- out.append({"role": role, "content": blocks})- if not out:- raise ModelError("ChatRequest has no content to send")- if out[0]["role"] != "user":- out.insert(0, {"role": "user", "content": [{"type": "text", "text": "(continue)"}]})- return out---# --------------------------------------------------------------------- tools-def submit_tool(schema: dict[str, Any]) -> dict[str, Any]:- return {- "name": SUBMIT_TOOL,- "description": "Submit the final answer. Call exactly once with the complete JSON object.",- "input_schema": to_anthropic_schema(schema),- }---# ------------------------------------------------------------------- request-def build_kwargs(request: ChatRequest, model: str) -> dict[str, Any]:- """Build ``client.messages.create(**kwargs)`` (without ``model``/``timeout``); a- response schema becomes a forced ``submit`` tool."""- kw: dict[str, Any] = {- "messages": to_messages(request.messages),- "max_tokens": request.max_output_tokens,- }- system = request.system- thinking_on = request.thinking != "off" or thinking_always_on(model)-- if uses_adaptive_thinking(model):- if request.thinking != "off":- kw["thinking"] = {"type": "adaptive"}- kw["output_config"] = {"effort": EFFORT[request.thinking]}- elif not thinking_always_on(model):- kw["thinking"] = {"type": "disabled"}- else:- kw["output_config"] = {"effort": "low"}- elif request.thinking != "off":- budget = THINKING_BUDGET[request.thinking]- kw["thinking"] = {"type": "enabled", "budget_tokens": budget}- kw["max_tokens"] = max(request.max_output_tokens, budget + 2048)-- if sampling_allowed(model) and not thinking_on:- kw["temperature"] = request.temperature-- if request.response_schema is not None:- kw["tools"] = [submit_tool(request.response_schema)]- if not thinking_on:- kw["tool_choice"] = {"type": "tool", "name": SUBMIT_TOOL}- else:- # forced tool_choice is incompatible with thinking- system = (- (system + "\n\n" if system else "")- + f"When you have the final answer, call the `{SUBMIT_TOOL}` tool exactly once with the complete JSON object."- )- if system:- kw["system"] = system- return kw---# ------------------------------------------------------------------ response-def parse_content(content: list[Any]) -> tuple[str, Any]:- """→ ``(text, submit_input)`` from response blocks."""- texts: list[str] = []- submit: Any = None- for block in content:- btype = getattr(block, "type", None)- if btype == "text":- texts.append(block.text)- elif btype == "tool_use" and block.name == SUBMIT_TOOL:- submit = (- block.input- if isinstance(block.input, dict)- else json.loads(json.dumps(block.input))- )- return "".join(texts), submit---# ===================================================================== anthropic-def classify_exception(exc: BaseException) -> ModelError:- """Map anthropic SDK exceptions onto ``ModelError``."""- import anthropic-- return classify_sdk_exception(exc, anthropic, "Anthropic")---class AnthropicModel(SdkModel):- """ChatModel for ``anthropic:<model>``."""-- provider = "anthropic"-- def _make_client(self) -> Any:- import anthropic--- key = get_settings().anthropic_api_key- if not key:- raise ModelError("ANTHROPIC_API_KEY is not configured")- return anthropic.Anthropic(api_key=key, max_retries=0, timeout=self.timeout_s)-- # -------------------------------------------------------------- generate- def generate(self, request: ChatRequest) -> ChatResponse:- kwargs = build_kwargs(request, self.model)- deadline = time.monotonic() + retry_budget_s(request.max_wait_s)- return retry_one_key(lambda: self._once(kwargs, request, deadline), label=f"anthropic {self.model}",- classify=classify_exception, attempts=self.max_attempts, sleep=self._sleep,- max_wait_s=request.max_wait_s)-- def _once(self, kwargs: dict[str, Any], request: ChatRequest, deadline: float) -> ChatResponse:- client = self.client()- with Stopwatch() as sw:- msg = client.messages.create(- model=self.model, timeout=attempt_timeout_s(deadline, self.timeout_s), **kwargs)- text, submit = parse_content(msg.content)- stop = str(msg.stop_reason or "")- usage = self._usage(msg.usage, sw.ms)- if stop == "refusal":- raise ModelError(- f"Anthropic refused the request: {getattr(msg, 'stop_details', None)}",- retryable=False, usage=usage, # a refusal is billed like any other reply- )- parsed: Any = None- if request.response_schema is not None:- if submit is not None:- # tool input skips parse_json_lenient; strip NULs here or they reach subprocess.Popen- parsed = strip_control_chars(submit)- if not text:- text = json.dumps(parsed)- else:- try:- parsed = parse_json_lenient(text)- except JsonParseError as exc:- raise ModelError(- f"structured output missing (stop_reason={stop}): {exc}",- retryable=stop != "max_tokens", usage=usage,- ) from exc- if not text and parsed is None:- raise ModelError(f"Anthropic returned no content (stop_reason={stop})",- retryable=True, usage=usage)- raw: dict[str, Any] = {"stop_reason": stop, "id": msg.id, "model": msg.model}- return ChatResponse(text=text, parsed=parsed, finish_reason=stop, usage=usage, raw=raw)-- def _usage(self, u: Any, latency_ms: int) -> Usage:- cache_read = int(getattr(u, "cache_read_input_tokens", 0) or 0)- cache_write = int(getattr(u, "cache_creation_input_tokens", 0) or 0)- usage = Usage(- backend="anthropic",- model=self.model,- input_tokens=int(u.input_tokens or 0) + cache_read + cache_write,- output_tokens=int(u.output_tokens or 0),- cached_tokens=cache_read,- thoughts_tokens=0, # thinking is billed inside output_tokens- latency_ms=latency_ms,- )- usage.cost_usd = estimate_cost("anthropic", self.model, usage) + cache_write_surcharge(- "anthropic", self.model, cache_write- )- return usagediff --git a/flywheel/c3d_flywheel/models/base.py b/flywheel/c3d_flywheel/models/base.pydeleted file mode 100644index 26554009..00000000--- a/flywheel/c3d_flywheel/models/base.py+++ /dev/null@@ -1,49 +0,0 @@-"""The ChatModel protocol every API backend implements."""--from __future__ import annotations--from typing import Protocol, runtime_checkable--from codeverse3d.contracts.chat import ChatRequest, ChatResponse-from codeverse3d.contracts.common import Usage---@runtime_checkable-class ChatModel(Protocol):- """One provider+model. Implementations must be thread-safe (the- orchestrator fans out judge/caption calls across threads)."""-- provider: str # gemini | anthropic | openai- model: str-- @property- def id(self) -> str: ...-- def generate(self, request: ChatRequest) -> ChatResponse:- """Blocking call. Must:- * honour ``request.response_schema`` (return ``parsed`` as a dict/list),- * attach images (``ImagePart``) inline in order,- * fill ``usage`` including ``cost_usd`` (see ``pricing.py``),- * raise ``ModelError`` (retryable=True/False) rather than provider exceptions.- """- ...---class ModelError(RuntimeError):- """A model call failed. ``attempts`` = round-trips issued (0 = unknown); ``usage`` =- what the provider already billed for the failed call (empty when nothing was)."""-- def __init__(- self,- message: str,- *,- retryable: bool = False,- status: int | None = None,- attempts: int = 0,- usage: Usage | None = None,- ):- super().__init__(message)- self.retryable = retryable- self.status = status- self.attempts = attempts- self.usage = usage if usage is not None else Usage()diff --git a/flywheel/c3d_flywheel/models/gemini.py b/flywheel/c3d_flywheel/models/gemini.pydeleted file mode 100644index 7fd4d545..00000000--- a/flywheel/c3d_flywheel/models/gemini.py+++ /dev/null@@ -1,429 +0,0 @@-"""Gemini backend: ``ChatRequest`` ⇄ google-genai conversion, and ``GeminiModel`` (shared-``KeyPool``, ``rotate_with_retries``, streaming with stall detection)."""--from __future__ import annotations--import logging-import random-import threading-import time-from collections.abc import Callable, Iterable-from typing import Any--import httpx-from google import genai-from google.genai import errors as genai_errors-from google.genai import types--from codeverse3d.config import get_settings-from codeverse3d.contracts.chat import (- ChatMessage,- ChatRequest,- ChatResponse,- ImagePart,-)-from codeverse3d.contracts.common import Usage-from c3d_flywheel.models.base import ModelError-from c3d_flywheel.models.parts import (- FailedBills,- Stopwatch,- attempt_timeout_s,- default_timeout_s,- image_bytes,- message_blocks,- retry_budget_s,- retryable_status,-)-from c3d_flywheel.models.pricing import estimate_cost-from c3d_flywheel.models.retry import KeyPool, KeyPoolExhausted, rotate_with_retries-from c3d_flywheel.models.schema_utils import JsonParseError, parse_json_lenient, to_gemini_schema--#: thinking level → token budget-THINKING_BUDGET: dict[str, int] = {"off": 0, "low": 1024, "medium": 4096, "high": 16384}--#: finish reasons worth one more attempt-RETRYABLE_FINISH = {"RECITATION", "MALFORMED_FUNCTION_CALL", "OTHER", "UNEXPECTED_TOOL_CALL"}-#: finish reasons retrying cannot fix-FATAL_FINISH = {"SAFETY", "PROHIBITED_CONTENT", "BLOCKLIST", "SPII", "IMAGE_SAFETY"}---# ------------------------------------------------------------------ contents-def _image_part(p: ImagePart) -> types.Part:- raw, mime = image_bytes(p)- return types.Part.from_bytes(data=raw, mime_type=mime)---def to_contents(messages: list[ChatMessage]) -> list[types.Content]:- """ChatMessages → Gemini ``Content`` list."""- out: list[types.Content] = []- for msg in messages:- role = "model" if msg.role == "assistant" else "user"- parts: list[types.Part] = message_blocks(msg, text=lambda t: types.Part.from_text(text=t), image=_image_part)- if not parts:- continue- # Gemini wants strict role alternation- if out and out[-1].role == role:- out[-1].parts.extend(parts) # type: ignore[union-attr]- else:- out.append(types.Content(role=role, parts=parts))- if not out:- raise ModelError("ChatRequest has no content to send")- return out---# -------------------------------------------------------------------- config-def build_config(- request: ChatRequest,- *,- timeout_ms: int,- use_thinking: bool = True,-) -> types.GenerateContentConfig:- """Map a ChatRequest onto ``GenerateContentConfig``."""- cfg: dict[str, Any] = {- "temperature": request.temperature,- "max_output_tokens": request.max_output_tokens,- "http_options": types.HttpOptions(timeout=timeout_ms),- }- if request.system:- cfg["system_instruction"] = request.system- if use_thinking:- cfg["thinking_config"] = types.ThinkingConfig(- thinking_budget=THINKING_BUDGET[request.thinking], include_thoughts=False- )- if request.response_schema is not None:- cfg["response_mime_type"] = "application/json"- cfg["response_schema"] = to_gemini_schema(request.response_schema)- return types.GenerateContentConfig(**cfg)---def clip_timeout(- config: types.GenerateContentConfig, timeout_ms: int-) -> types.GenerateContentConfig:- """``config`` with its HTTP timeout set to ``timeout_ms`` (same object if unchanged)."""- current = config.http_options.timeout if config.http_options is not None else None- if current == timeout_ms:- return config- return config.model_copy(update={"http_options": types.HttpOptions(timeout=timeout_ms)})---# ------------------------------------------------------------------ response-def parse_usage(resp: types.GenerateContentResponse, model: str) -> Usage:- um = resp.usage_metadata- if um is None:- return Usage(backend="gemini", model=model)- return Usage(- backend="gemini",- model=resp.model_version or model,- input_tokens=(um.prompt_token_count or 0) + (um.tool_use_prompt_token_count or 0),- output_tokens=um.candidates_token_count or 0,- cached_tokens=um.cached_content_token_count or 0,- thoughts_tokens=um.thoughts_token_count or 0,- )---def _guard_candidates(resp: types.GenerateContentResponse, usage: Usage | None, what: str) -> Any:- """The first candidate, or a blocked-prompt (fatal) / no-candidates (retryable) error."""- pf = resp.prompt_feedback- if pf is not None and pf.block_reason:- raise ModelError(- f"{what} prompt blocked: {pf.block_reason} {pf.block_reason_message or ''}".strip(),- retryable=False, usage=usage,- )- if not resp.candidates:- raise ModelError(f"{what} returned no candidates", retryable=True, usage=usage)- return resp.candidates[0]---def extract_candidate(- resp: types.GenerateContentResponse, usage: Usage | None = None-) -> tuple[str, str]:- """→ ``(text, finish_reason)``; raises ``ModelError`` carrying ``usage`` (these failures are billed)."""- cand = _guard_candidates(resp, usage, "Gemini")- finish = str(cand.finish_reason.value if cand.finish_reason else "") or "UNKNOWN"- texts: list[str] = []- for part in cand.content.parts if cand.content and cand.content.parts else []:- if part.text and not part.thought:- texts.append(part.text)- text = "".join(texts)- if not text:- if finish in FATAL_FINISH:- raise ModelError(- f"Gemini produced no content (finish_reason={finish})", retryable=False, usage=usage- )- if finish == "MAX_TOKENS":- # Gemini 3 always thinks; a small max_output_tokens is eaten by thoughts. The- # "finish_reason=MAX_TOKENS" text is matched by the planner's grow-the-budget retry.- thoughts = resp.usage_metadata.thoughts_token_count if resp.usage_metadata else None- raise ModelError(- f"Gemini produced no content (finish_reason=MAX_TOKENS): max_output_tokens exhausted by thinking "- f"(thoughts_tokens={thoughts}); raise max_output_tokens",- retryable=False, usage=usage,- )- raise ModelError(f"Gemini produced no content (finish_reason={finish})",- retryable=True, usage=usage)- return text, finish---# ===================================================================== gemini-log = logging.getLogger(__name__)--#: ceiling of the inter-chunk read timeout under streaming: a hung socket dies within it,-#: while a long healthy stream is bounded only by its attempt budget-STREAM_STALL_S = 300.0--_pools: dict[tuple[str, ...], KeyPool] = {}-_clients: dict[tuple[str, int], genai.Client] = {}-_registry_lock = threading.Lock()---def shared_pool(keys: list[str], *, max_in_flight: int | None = None) -> KeyPool:- """One ``KeyPool`` per (key list, in-flight cap) per process; slots are machine-wide- (``<cache_dir>/slots/gemini/NN.lock``)."""- cap = _rate().max_in_flight if max_in_flight is None else max_in_flight- sig = (*keys, f"|{cap}")- with _registry_lock:- pool = _pools.get(sig)- if pool is None:- slots = get_settings().cache_dir / "slots" / "gemini" if cap else None- # random first key, so parallel processes do not all start on key 0- pool = KeyPool(keys, max_in_flight=cap or 0, slots_dir=slots, start=random.randrange(len(keys)))- _pools[sig] = pool- return pool---def _rate() -> Any:- return get_settings().rate---def _default_keys() -> list[str]:- return list(get_settings().gemini_api_keys)---def _keys_or_raise(keys: list[str] | None, pool: KeyPool | None) -> list[str]:- keys = list(keys) if keys is not None else _default_keys()- if pool is None and not keys:- raise ModelError("no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)")- return keys---def _client_for(key: str, timeout_s: float, factory: Callable[[str], Any] | None) -> Any:- """One ``genai.Client`` per (key, timeout), process-wide; ``factory`` bypasses the cache."""- if factory is not None:- return factory(key)- ck = (key, int(timeout_s * 1000))- with _registry_lock:- client = _clients.get(ck)- if client is None:- client = genai.Client(api_key=key, http_options=types.HttpOptions(- timeout=ck[1],- # the WSL2 IPv6 path silently hangs reads: bind the transport to IPv4- client_args={"transport": httpx.HTTPTransport(local_address="0.0.0.0")}))- _clients[ck] = client- return client---def classify_exception(exc: BaseException) -> ModelError:- """Map provider / transport exceptions onto ``ModelError``."""- if isinstance(exc, ModelError):- return exc- if isinstance(exc, genai_errors.APIError):- code = int(exc.code or 0)- msg = f"Gemini API error {code}: {exc.message}"- return ModelError(msg, retryable=retryable_status(code), status=code)- if isinstance(exc, (httpx.TimeoutException, TimeoutError)):- return ModelError(f"Gemini request timed out: {exc}", retryable=True, status=408)- if isinstance(exc, (httpx.TransportError, ConnectionError)):- return ModelError(f"Gemini transport error: {exc}", retryable=True)- if isinstance(exc, KeyPoolExhausted):- return ModelError(f"Gemini key pool exhausted: {exc}", retryable=False, status=429)- return ModelError(f"Gemini unexpected error: {type(exc).__name__}: {exc}", retryable=False)---def _drain_stream(- it: Iterable[types.GenerateContentResponse],- deadline: float,- clock: Callable[[], float] = time.monotonic,-) -> list[types.GenerateContentResponse]:- """Collect every chunk, enforcing the attempt ``deadline`` between chunks (the wire- timeout only bounds silence between chunks)."""- chunks: list[types.GenerateContentResponse] = []- for ch in it:- chunks.append(ch)- if clock() > deadline:- raise ModelError(- f"Gemini stream exceeded its attempt budget after {len(chunks)} chunks",- retryable=True,- )- return chunks---def _merge_stream_chunks(- chunks: list[types.GenerateContentResponse],-) -> types.GenerateContentResponse:- """One response from a chunk sequence: parts concatenate; finish_reason, usage_metadata- and prompt_feedback come from the last chunk carrying each."""- if not chunks:- raise ModelError("Gemini stream yielded no chunks", retryable=True)- parts: list[types.Part] = []- finish = None- usage_md = None- feedback = None- base = None- for ch in chunks:- if ch.usage_metadata is not None:- usage_md = ch.usage_metadata- if ch.prompt_feedback is not None and ch.prompt_feedback.block_reason:- feedback = ch.prompt_feedback- if ch.candidates:- base = ch- cand = ch.candidates[0]- if cand.content and cand.content.parts:- parts.extend(cand.content.parts)- if cand.finish_reason is not None:- finish = cand.finish_reason- if base is None: # no candidates: extract_candidate raises- base = chunks[-1]- else:- base.candidates[0].content = types.Content(role="model", parts=parts)- base.candidates[0].finish_reason = finish- base.usage_metadata = usage_md- if feedback is not None:- base.prompt_feedback = feedback- return base---class GeminiModel:- """ChatModel for ``gemini:<model>``."""-- provider = "gemini"-- def __init__(- self,- model: str,- *,- keys: list[str] | None = None,- pool: KeyPool | None = None,- timeout_s: float | None = None,- max_attempts: int = 6,- storm_attempts: int | None = None,- sleep: Callable[[float], None] = time.sleep,- client_factory: Callable[[str], Any] | None = None,- ) -> None:- self.model = model- #: None = rotate_with_retries' default; 0 = a 503 is final (the storm branch ignores max_attempts)- self.storm_attempts = storm_attempts- keys = _keys_or_raise(keys, pool)- self.pool = pool or shared_pool(keys)- self.timeout_s = default_timeout_s() if timeout_s is None else timeout_s- self.max_attempts = max(1, max_attempts)- self._sleep = sleep- self._client_factory = client_factory- self._thinking_ok = True # flipped when the model rejects ThinkingConfig- self._lock = threading.Lock()-- # ------------------------------------------------------------------ proto- @property- def id(self) -> str:- return f"gemini:{self.model}"-- # --------------------------------------------------------------- generate- def generate(self, request: ChatRequest) -> ChatResponse:- contents = to_contents(request.messages)- state = {"config": self._config(request)}-- def downgrade_thinking(err: ModelError) -> bool:- """Model rejects ThinkingConfig → retry without it, once."""- if not (self._thinking_ok and err.status == 400 and "thinking" in str(err).lower()):- return False- with self._lock:- self._thinking_ok = False- log.warning("%s rejected thinking config; retrying without it: %s", self.model, err)- state["config"] = self._config(request)- return True-- budget = retry_budget_s(request.max_wait_s)- deadline = time.monotonic() + budget- stats: dict[str, Any] = {}- bills = FailedBills()- try:- resp = rotate_with_retries(- self.pool,- bills.track(lambda key: self._once(- key, contents, self._attempt_config(state["config"], deadline),- request,- )),- classify=classify_exception,- max_attempts=self.max_attempts,- max_total_s=budget,- sleep=self._sleep,- on_free_retry=downgrade_thinking,- label=f"gemini {self.model}",- stats=stats,- **({} if self.storm_attempts is None else {"storm_attempts": self.storm_attempts}),- )- except ModelError as e:- bills.charge(e)- raise- resp = bills.join(resp)- resp.raw["attempts"] = int(stats.get("attempts", 0))- return resp-- def _config(self, request: ChatRequest) -> types.GenerateContentConfig:- return build_config(- request,- timeout_ms=int(self.timeout_s * 1000),- use_thinking=self._thinking_ok,- )-- def _attempt_config(- self, config: types.GenerateContentConfig, deadline: float- ) -> types.GenerateContentConfig:- """Config whose HTTP timeout is the remaining retry budget (see ``attempt_timeout_s``)."""- return clip_timeout(config, int(attempt_timeout_s(deadline, self.timeout_s) * 1000))-- def _once(- self,- key: str,- contents: list[types.Content],- config: types.GenerateContentConfig,- request: ChatRequest,- ) -> ChatResponse:- client = _client_for(key, self.timeout_s, self._client_factory)-- # the attempt budget becomes an in-loop deadline; the wire timeout drops to the stall ceiling- cfg_ms = config.http_options.timeout if config.http_options is not None else None- attempt_s = (float(cfg_ms) / 1000.0) if cfg_ms else self.timeout_s- wire = clip_timeout(config, int(min(attempt_s, STREAM_STALL_S) * 1000))- with Stopwatch() as sw:- chunks = _drain_stream(- client.models.generate_content_stream(- model=self.model, contents=contents, config=wire- ),- time.monotonic() + attempt_s,- )- resp = _merge_stream_chunks(chunks)- usage = parse_usage(resp, self.model) # before extract_candidate, whose raises carry it- usage.latency_ms = sw.ms- usage.cost_usd = estimate_cost("gemini", self.model, usage)- text, finish = extract_candidate(resp, usage)- parsed: Any = None- if request.response_schema is not None:- try:- parsed = parse_json_lenient(text)- except JsonParseError as exc:- if finish in FATAL_FINISH or finish == "MAX_TOKENS":- raise ModelError(- f"structured output unavailable (finish_reason={finish}; "- f"raise max_output_tokens if truncated): {exc}",- retryable=False,- usage=usage,- ) from exc- raise ModelError(- f"structured output is not valid JSON (finish_reason={finish}): {exc}",- retryable=True,- usage=usage,- ) from exc- elif finish in RETRYABLE_FINISH and not text.strip():- raise ModelError(f"Gemini finish_reason={finish}", retryable=True, usage=usage)- return ChatResponse(text=text, parsed=parsed, finish_reason=finish, usage=usage,- raw={"key": f"…{key[-4:]}"})diff --git a/flywheel/c3d_flywheel/models/health.py b/flywheel/c3d_flywheel/models/health.pydeleted file mode 100644index c3ad6d7d..00000000--- a/flywheel/c3d_flywheel/models/health.py+++ /dev/null@@ -1,109 +0,0 @@-"""Is a model actually serving right now? ``probe`` asks once per key, in parallel, with a-short timeout and no retries; callers use it as a preflight gate before spending real money.-The probe prompt is workload-sized: a tiny prompt can pass while large calls still fail.-"""--from __future__ import annotations--import concurrent.futures as cf-import logging-import random-from dataclasses import dataclass-from typing import Any--from codeverse3d.config import get_settings-from codeverse3d.contracts.common import Backends-from c3d_flywheel.models.registry import build_chat_model--log = logging.getLogger(__name__)--#: pool keys sampled per probe-DEFAULT_SAMPLE = 6-DEFAULT_TIMEOUT_S = 30.0-#: probe prompt size in tokens, sized like a real planner/generator call-PROBE_TOKENS = 8000-#: below this share of answering calls the model counts as down (a battery needs many calls to land)-DEFAULT_MIN_OK = 0.75---@dataclass(frozen=True)-class Health:- model: str- n_ok: int- n_tried: int- reasons: tuple[str, ...] = ()-- @property- def ok(self) -> bool:- return self.n_tried > 0 and self.n_ok / self.n_tried >= DEFAULT_MIN_OK-- def __str__(self) -> str:- head = f"{self.model}: {self.n_ok}/{self.n_tried} keys answering"- return (- head if self.ok else f"{head} — {'; '.join(self.reasons[:2]) or 'no reason reported'}"- )---def probe(- model: str = Backends().planner,- *,- sample: int = DEFAULT_SAMPLE,- timeout_s: float = DEFAULT_TIMEOUT_S,-) -> Health:- """One workload-sized call (``PROBE_TOKENS``) per sampled key, in parallel, no retries. Never raises."""- from codeverse3d.contracts.chat import ChatMessage, ChatRequest-- prompt = _probe_prompt()-- def one(key: str | None) -> str | None:- try:- m = _bare_model(model, timeout_s, key)- m.generate(- ChatRequest(- messages=[ChatMessage.user(prompt)],- temperature=0.0,- max_output_tokens=256,- thinking="off",- label="health",- max_wait_s=timeout_s,- )- )- return None- except Exception as e: # noqa: BLE001 — a probe reports, it does not propagate- return f"{type(e).__name__}: {str(e)[:100]}"-- keys = _sampled_keys(model, max(1, sample))- n = len(keys)- with cf.ThreadPoolExecutor(max_workers=n) as pool:- out = list(pool.map(one, keys))- reasons = tuple(dict.fromkeys(r for r in out if r)) # de-duplicated, order kept- return Health(model=model, n_ok=sum(r is None for r in out), n_tried=n, reasons=reasons)---def _probe_prompt(tokens: int = PROBE_TOKENS) -> str:- """A workload-sized prompt whose answer is still one word."""- filler = "The quick brown fox jumps over the lazy dog. " * max(1, tokens // 9)- return f"{filler}\n\nIgnore the text above. Reply with the single word: pong"---def _sampled_keys(model_id: str, n: int) -> list[str | None]:- """One pool key per probe for a gemini model (distinct while the pool allows); ``[None] * n`` otherwise."""- if ":" in model_id and not model_id.startswith("gemini:"):- return [None] * n- keys = list(get_settings().gemini_api_keys)- if not keys:- return [None] * n- chosen = random.sample(keys, min(n, len(keys)))- return [chosen[i % len(chosen)] for i in range(n)]---def _bare_model(model_id: str, timeout_s: float, key: str | None = None):- """The model with every retry off, on ONE key when given; a bare name is a gemini model."""- if ":" not in model_id:- model_id = f"gemini:{model_id}"- extra: dict[str, Any] = {}- if model_id.startswith("gemini:"):- extra = {"storm_attempts": 0}- if key is not None:- extra["keys"] = [key]- return build_chat_model(model_id, timeout_s=timeout_s, max_attempts=1, **extra)diff --git a/flywheel/c3d_flywheel/models/openai.py b/flywheel/c3d_flywheel/models/openai.pydeleted file mode 100644index 4e36778f..00000000--- a/flywheel/c3d_flywheel/models/openai.py+++ /dev/null@@ -1,227 +0,0 @@-"""ChatRequest ⇄ OpenAI Chat Completions dicts (messages, response_format).--Chat Completions is used because every OpenAI-compatible endpoint (``OPENAI_BASE_URL``) speaks it.-"""--from __future__ import annotations--import logging-import time-from typing import Any--from codeverse3d.config import get_settings-from codeverse3d.contracts.chat import (- ChatMessage,- ChatRequest,- ChatResponse,- ImagePart,- TextPart,-)-from codeverse3d.contracts.common import Usage-from c3d_flywheel.models.base import ModelError-from c3d_flywheel.models.parts import (- SdkModel,- Stopwatch,- attempt_timeout_s,- classify_sdk_exception,- image_b64,- message_blocks,- retry_budget_s,- retry_one_key,-)-from c3d_flywheel.models.pricing import openai_usage-from c3d_flywheel.models.retry import cause_for-from c3d_flywheel.models.schema_utils import (- JsonParseError,- inline_refs,- parse_json_lenient,- to_openai_strict_schema,-)--SCHEMA_NAME = "response"---# ------------------------------------------------------------- model families-def is_reasoning_model(model: str) -> bool:- return model.lower().startswith("gpt-6")---def reasoning_effort(model: str, thinking: str) -> str | None:- """Map ``thinking`` onto ``reasoning_effort`` (None for non-reasoning models). gpt-6 has no- off switch, so "off" is ``low``."""- if not is_reasoning_model(model):- return None- return "low" if thinking == "off" else thinking---# ------------------------------------------------------------------ messages-def _image_block(p: ImagePart) -> dict[str, Any]:- data, mime = image_b64(p)- return {- "type": "image_url",- "image_url": {"url": f"data:{mime};base64,{data}", "detail": "auto"},- }---def to_messages(messages: list[ChatMessage], system: str) -> list[dict[str, Any]]:- """ChatMessages → Chat Completions messages."""- out: list[dict[str, Any]] = []- if system:- out.append({"role": "system", "content": system})- for msg in messages:- if msg.role == "assistant":- out.append(_assistant_message(msg))- continue- content = message_blocks(msg, text=lambda t: {"type": "text", "text": t}, image=_image_block)- if content:- out.append({"role": "user", "content": content})- if not out or all(m["role"] == "system" for m in out):- raise ModelError("ChatRequest has no content to send")- return out---def _assistant_message(msg: ChatMessage) -> dict[str, Any]:- text = "".join(p.text for p in msg.parts if isinstance(p, TextPart))- return {"role": "assistant", "content": text or None}---def response_format(schema: dict[str, Any], *, strict: bool) -> dict[str, Any]:- body = to_openai_strict_schema(schema) if strict else inline_refs(schema)- return {- "type": "json_schema",- "json_schema": {"name": SCHEMA_NAME, "schema": body, "strict": strict},- }---# ------------------------------------------------------------------- request-def build_kwargs(request: ChatRequest, model: str, *, strict_schema: bool) -> dict[str, Any]:- """``client.chat.completions.create(**kwargs)`` minus ``model``."""- kw: dict[str, Any] = {- "messages": to_messages(request.messages, request.system),- "max_completion_tokens": request.max_output_tokens,- }- effort = reasoning_effort(model, request.thinking)- if effort is not None:- kw["reasoning_effort"] = effort- else:- kw["temperature"] = request.temperature- if request.response_schema is not None:- kw["response_format"] = response_format(request.response_schema, strict=strict_schema)- return kw---# ------------------------------------------------------------------ response-def parse_choice(choice: Any) -> tuple[str, str]:- """→ ``(text, finish_reason)``."""- msg = choice.message- text = msg.content or ""- if getattr(msg, "refusal", None):- raise ModelError(f"OpenAI refusal: {msg.refusal}", retryable=False)- return text, str(choice.finish_reason or "")---# ===================================================================== openai-log = logging.getLogger(__name__)---def classify_exception(exc: BaseException) -> ModelError:- """Map openai SDK exceptions onto ``ModelError``."""- import openai-- return classify_sdk_exception(exc, openai, "OpenAI")---def _is_schema_rejection(err: ModelError) -> bool:- s = str(err).lower()- return err.status == 400 and ("schema" in s or "response_format" in s or "strict" in s)---class OpenAIModel(SdkModel):- """ChatModel for ``openai:<model>`` (and OpenAI-compatible endpoints)."""-- provider = "openai"- _strict_ok = True # flipped (per instance) when the endpoint rejects strict schemas-- def _make_client(self) -> Any:- import openai--- s = get_settings()- key = s.openai_api_key- base_url = s.openai_base_url or None- if not key and not base_url:- raise ModelError("OPENAI_API_KEY is not configured")- return openai.OpenAI(- api_key=key or "sk-local",- base_url=base_url,- max_retries=0,- timeout=self.timeout_s,- )-- # -------------------------------------------------------------- generate- def generate(self, request: ChatRequest) -> ChatResponse:- deadline = time.monotonic() + retry_budget_s(request.max_wait_s)-- def attempt() -> ChatResponse:- strict = self._strict_ok and request.response_schema is not None- kwargs = build_kwargs(request, self.model, strict_schema=strict)- try:- return self._once(kwargs, request, deadline)- except Exception as exc: # noqa: BLE001 - classified- err = classify_exception(exc)- if strict and _is_schema_rejection(err):- log.warning(- "openai %s rejected strict json_schema (%s); falling back to strict=false",- self.model,- err,- )- with self._lock:- self._strict_ok = False- kwargs = build_kwargs(request, self.model, strict_schema=False)- try:- return self._once(kwargs, request, deadline)- except Exception as exc2: # noqa: BLE001- err2 = classify_exception(exc2)- raise err2 from cause_for(err2, exc2)- raise err from cause_for(err, exc)-- return retry_one_key(attempt, label=f"openai {self.model}", classify=classify_exception,- attempts=self.max_attempts, sleep=self._sleep, max_wait_s=request.max_wait_s)-- def _once(self, kwargs: dict[str, Any], request: ChatRequest, deadline: float) -> ChatResponse:- client = self.client()- with Stopwatch() as sw:- completion = client.chat.completions.create(- model=self.model, timeout=attempt_timeout_s(deadline, self.timeout_s), **kwargs)- if not completion.choices:- raise ModelError("OpenAI returned no choices", retryable=True)- text, finish = parse_choice(completion.choices[0])- usage = self._usage(completion.usage, sw.ms)- parsed: Any = None- if request.response_schema is not None:- try:- parsed = parse_json_lenient(text)- except JsonParseError as exc:- raise ModelError(- f"structured output is not valid JSON (finish_reason={finish}): {exc}",- retryable=finish != "length", usage=usage, # billed like a good reply- ) from exc- if not text:- raise ModelError(- f"OpenAI returned no content (finish_reason={finish})",- retryable=finish != "content_filter", usage=usage,- )- raw: dict[str, Any] = {- "finish_reason": finish,- "id": completion.id,- "model": completion.model,- }- return ChatResponse(text=text, parsed=parsed, finish_reason=finish, usage=usage, raw=raw)-- def _usage(self, u: Any, latency_ms: int) -> Usage:- ptd = getattr(u, "prompt_tokens_details", None)- ctd = getattr(u, "completion_tokens_details", None)- return openai_usage("openai", self.model, prompt=int(getattr(u, "prompt_tokens", 0) or 0),- cached=int(getattr(ptd, "cached_tokens", 0) or 0),- completion=int(getattr(u, "completion_tokens", 0) or 0),- reasoning=int(getattr(ctd, "reasoning_tokens", 0) or 0), latency_ms=latency_ms)diff --git a/flywheel/c3d_flywheel/models/parts.py b/flywheel/c3d_flywheel/models/parts.pydeleted file mode 100644index 142a4c8f..00000000--- a/flywheel/c3d_flywheel/models/parts.py+++ /dev/null@@ -1,196 +0,0 @@-"""Small helpers shared by the provider backends (images, timing, usage)."""--from __future__ import annotations--import base64-import mimetypes-import threading-import time-from collections.abc import Callable-from pathlib import Path-from typing import Any--from codeverse3d.config import get_settings-from codeverse3d.contracts.chat import ChatMessage, ImagePart, TextPart-from codeverse3d.contracts.common import Usage-from c3d_flywheel.models.base import ModelError---def image_bytes(part: ImagePart) -> tuple[bytes, str]:- """Return ``(raw_bytes, mime)`` for an ``ImagePart``.- Raises ``ModelError(retryable=False)`` when the part has no readable path."""- if part.path:- p = Path(part.path)- if not p.is_file():- raise ModelError(f"ImagePart.path does not exist: {part.path}")- mime = part.mime- if not mime or mime == "image/png" and p.suffix.lower() not in ("", ".png"):- mime = mimetypes.guess_type(p.name)[0] or "image/png"- return p.read_bytes(), mime- raise ModelError("ImagePart has no path")---def image_b64(part: ImagePart) -> tuple[str, str]:- """``(base64_string, mime)`` for providers that want base64 (Anthropic, OpenAI data URLs)."""- raw, mime = image_bytes(part)- return base64.b64encode(raw).decode("ascii"), mime---def message_blocks(msg: ChatMessage, *, text: Callable[[str], Any], image: Callable[[ImagePart], Any]) -> list[Any]:- """One message's parts as a provider's blocks: empty text is dropped, and a labelled image- is preceded by an ``[image: <label>]`` text block (the one place that text is written)."""- out: list[Any] = []- for p in msg.parts:- if isinstance(p, TextPart):- if p.text:- out.append(text(p.text))- elif isinstance(p, ImagePart):- if p.label:- out.append(text(f"[image: {p.label}]"))- out.append(image(p))- return out---def default_timeout_s() -> float:- """``Settings.model_timeout_s``."""- return float(get_settings().model_timeout_s)---class SdkModel:- """The shell the SDK adapters (anthropic, openai) share: the constructor, ``id`` and the- lazily built client. Each adapter owns ``_make_client`` and its own ``generate``."""-- provider = ""-- def __init__(- self,- model: str,- *,- timeout_s: float | None = None,- max_attempts: int = 6,- sleep: Callable[[float], None] = time.sleep,- client: Any | None = None,- ) -> None:- self.model = model- self.timeout_s = default_timeout_s() if timeout_s is None else timeout_s- self.max_attempts = max(1, max_attempts)- self._sleep = sleep- self._client = client- self._lock = threading.Lock()-- @property- def id(self) -> str:- return f"{self.provider}:{self.model}"-- def client(self) -> Any:- with self._lock:- if self._client is None:- self._client = self._make_client()- return self._client-- def _make_client(self) -> Any:- raise NotImplementedError---class Stopwatch:- """Context-manager timer: ``with Stopwatch() as sw: ...`` leaves the elapsed milliseconds in ``sw.ms``."""-- def __enter__(self) -> Stopwatch:- self._t0 = time.perf_counter()- self.ms = 0- return self-- def __exit__(self, *exc: object) -> None:- self.ms = int((time.perf_counter() - self._t0) * 1000)---def retryable_status(status: int | None) -> bool:- """HTTP statuses worth retrying: 408, 409, 429 and every 5xx."""- return status is not None and (status in (408, 409, 429) or status >= 500)---def classify_sdk_exception(exc: BaseException, sdk: Any, label: str) -> Any:- """The openai/anthropic SDK exception ladder → ``ModelError``.-- The message wording is load-bearing: eval/bench/_infra.py string-matches "request timed- out" and "connection error" to tell a provider outage from a model failure.- """- if isinstance(exc, ModelError):- return exc- if isinstance(exc, sdk.APIStatusError):- status = int(getattr(exc, "status_code", 0) or 0)- return ModelError(f"{label} API error {status}: {exc.message}", retryable=retryable_status(status), status=status)- if isinstance(exc, sdk.APITimeoutError):- return ModelError(f"{label} request timed out: {exc}", retryable=True, status=408)- if isinstance(exc, sdk.APIConnectionError):- return ModelError(f"{label} connection error: {exc}", retryable=True)- if isinstance(exc, sdk.APIError):- return ModelError(f"{label} API error: {exc}", retryable=False)- return ModelError(f"{label} unexpected error: {type(exc).__name__}: {exc}", retryable=False)---#: floor of the per-attempt timeout, so a call near its deadline still gets one real attempt-SDK_TIMEOUT_FLOOR_S = 20.0---def retry_budget_s(max_wait_s: float | None) -> float:- """The budget for ONE logical call: the caller clips ``RETRY_DEADLINE_S``, never extends it."""- from c3d_flywheel.models.retry import RETRY_DEADLINE_S-- return RETRY_DEADLINE_S if max_wait_s is None else min(RETRY_DEADLINE_S, float(max_wait_s))---def attempt_timeout_s(deadline: float, ceiling: float) -> float:- """Per-attempt SDK timeout: time left to ``deadline``, capped at ``ceiling``, floored at- :data:`SDK_TIMEOUT_FLOOR_S`. The retry loop checks the deadline only between attempts."""- return min(ceiling, max(SDK_TIMEOUT_FLOOR_S, deadline - time.monotonic()))---class FailedBills:- """What the failed attempts of ONE logical call were billed for (``ModelError.usage``): it joins- the call's single ledger row, on success and on a final failure alike."""-- def __init__(self) -> None:- self.usage = Usage()-- def track[T](self, attempt: Callable[..., T]) -> Callable[..., T]:- """``attempt`` that remembers the usage of every ``ModelError`` it raises."""- def run(*args: Any) -> T:- try:- return attempt(*args)- except ModelError as e:- if e.usage is not None:- self.usage = self.usage + e.usage- raise- return run-- @property- def billed(self) -> bool:- return bool(self.usage.input_tokens or self.usage.output_tokens)-- def join(self, resp: Any) -> Any:- """``resp`` with the failed attempts' usage added."""- return resp.model_copy(update={"usage": resp.usage + self.usage}) if self.billed else resp-- def charge(self, err: ModelError) -> None:- """A finally-raised error carries ALL the usage billed (its own attempt included)."""- if self.billed:- err.usage = self.usage---def retry_one_key(attempt: Any, *, label: str, classify: Any, attempts: int, sleep: Any,- max_wait_s: float | None = None) -> Any:- """The SDK adapters' retry loop: ``rotate_with_retries`` over a one-key pool, no cooldown or- storm patience, inside the request's ``max_wait_s`` deadline."""- from c3d_flywheel.models.retry import KeyPool, rotate_with_retries-- bills = FailedBills()- try:- resp = rotate_with_retries(KeyPool([label], cooldown_s=0.0), lambda _key: bills.track(attempt)(),- classify=classify, max_attempts=attempts,- max_total_s=retry_budget_s(max_wait_s), storm_attempts=0,- sleep=sleep, label=label)- except ModelError as e:- bills.charge(e)- raise- return bills.join(resp)diff --git a/flywheel/c3d_flywheel/models/registry.py b/flywheel/c3d_flywheel/models/registry.pydeleted file mode 100644index 3ef10c71..00000000--- a/flywheel/c3d_flywheel/models/registry.py+++ /dev/null@@ -1,49 +0,0 @@-"""Model id parsing + factory. Ids: ``<provider>:<model>``.--Every model handed out here is wrapped by ``c3d_flywheel.cost.instrument`` so each-call lands in the cost ledger (the run's ``telemetry/cost.jsonl`` when a run is-active, otherwise a per-process log) — always: the ledger is the only record of money."""--from __future__ import annotations--from functools import lru_cache-from typing import Any--from c3d_flywheel.models.base import ChatModel--PROVIDERS = ("gemini", "anthropic", "openai")---def parse_model_id(model_id: str) -> tuple[str, str]:- if ":" not in model_id:- raise ValueError(f"model id must be '<provider>:<model>', got {model_id!r}")- provider, model = model_id.split(":", 1)- if provider not in PROVIDERS:- raise ValueError(f"unknown provider {provider!r}; known: {PROVIDERS}")- if not model.strip():- raise ValueError(f"model id {model_id!r} names no model")- return provider, model---@lru_cache(maxsize=32)-def get_chat_model(model_id: str) -> ChatModel:- """The (metered) ChatModel for ``<provider>:<model>``."""- from c3d_flywheel.cost.instrument import metered_chat_model-- return metered_chat_model(build_chat_model(model_id))---def build_chat_model(model_id: str, **kw: Any) -> ChatModel:- """The bare (unmetered) model for ``<provider>:<model>``; ``kw`` goes to its constructor."""- provider, model = parse_model_id(model_id)- if provider == "gemini":- from c3d_flywheel.models.gemini import GeminiModel-- return GeminiModel(model, **kw)- if provider == "anthropic":- from c3d_flywheel.models.anthropic import AnthropicModel-- return AnthropicModel(model, **kw)- from c3d_flywheel.models.openai import OpenAIModel-- return OpenAIModel(model, **kw)diff --git a/flywheel/c3d_flywheel/models/retry.py b/flywheel/c3d_flywheel/models/retry.pydeleted file mode 100644index 45f991bd..00000000--- a/flywheel/c3d_flywheel/models/retry.py+++ /dev/null@@ -1,497 +0,0 @@-"""The scheduling machine under every model call.--``KeyPool``: multi-key pool with 429 cooldown, dead-key benching, health scores and a-machine-wide ``max_in_flight`` ceiling (``Slots``, flock'd lock files). ``rotate_with_retries``:-the one retry loop (pool, ``RETRY_DEADLINE_S``); SDK adapters run it over a one-key pool.-Every wait clips to ``MAX_WAIT_S``. ``report`` outcomes: ok | 429 | 5xx | error | dead | skip-(``skip`` = a content failure the key did not cause). Thread-safe; ``clock`` / ``sleep`` injectable.-"""--from __future__ import annotations--from codeverse3d.proc import Slots, SlotsBusy--import fcntl-import hashlib-import logging-import os-import random-import threading-import time-from collections.abc import Callable-from dataclasses import dataclass-from pathlib import Path-from typing import TYPE_CHECKING, Any, Literal--Outcome = Literal["ok", "429", "5xx", "error", "dead", "skip"]---class KeyPoolExhausted(SlotsBusy):- """Every key is cooling down (or every slot is busy) and the wait budget ran out."""---# message fragments of key-scoped failures; HTTP 401 / 403 are key-scoped regardless-_DEAD_KEY_MARKERS = (- "api_key_invalid",- "api key not valid",- "api key expired",- "permission_denied",- "unauthenticated",- "suspended",- "leaked",- "service_disabled",- "has not been used in project",-)---def is_dead_key_error(err: ModelError) -> bool:- """True when ``err`` indicts the API key rather than the request."""- if err.status in (401, 403):- return True- text = str(err).lower()- return err.status == 400 and any(m in text for m in _DEAD_KEY_MARKERS)---def failure_outcome(err: ModelError) -> Outcome:- """``KeyPool.report`` outcome for a failed call; status-less content failures are ``skip``."""- if err.status == 429:- return "429"- if (err.status or 0) >= 500:- return "5xx"- if is_dead_key_error(err):- return "dead"- return "error" if err.status else "skip"---#: Longest single retry / cooldown wait (seconds): the pool rotates keys rather than sitting out-#: a long backoff, so patience comes from the number of attempts, never from one sleep.-MAX_WAIT_S = 3.0--#: Default total wait budget of one :meth:`KeyPool.acquire` (seconds): the in-flight slot wait plus-#: the key / cooldown wait; ``None`` waits forever.-ACQUIRE_TIMEOUT_S = 120.0---@dataclass-class _KeyState:- key: str- health: float = 1.0- cooldown_until: float = 0.0- dead_until: float = 0.0- n_ok: int = 0- n_429: int = 0- n_5xx: int = 0- n_error: int = 0- health_ts: float = 0.0-- def recover(self, now: float, rate: float = 0.01) -> None:- """Passive health recovery: +``rate`` per idle second, capped at 1.0."""- if now > self.health_ts:- self.health = min(1.0, self.health + (now - self.health_ts) * rate)- self.health_ts = now---class KeyPool:- """Round-robin key pool with cooldowns and an in-flight ceiling."""-- def __init__(- self,- keys: list[str] | tuple[str, ...],- *,- max_in_flight: int = 0,- slots_dir: Path | str | None = None,- cooldown_s: float = MAX_WAIT_S,- dead_cooldown_s: float = 3600.0,- clock: Callable[[], float] = time.monotonic,- sleep: Callable[[float], None] = time.sleep,- start: int = 0,- ) -> None:- """``start``: the key the round-robin begins at (``shared_pool`` draws it at random)."""- uniq = [k for i, k in enumerate(keys) if k and k not in keys[:i]]- if not uniq:- raise ValueError("KeyPool needs at least one non-empty key")- self._clock = clock- self._sleep = sleep- self._cooldown_s = float(cooldown_s)- self._dead_cooldown_s = float(dead_cooldown_s)- self._lock = threading.Lock()- self._rr = start % len(uniq)- self._in_flight = 0- # machine-wide ceiling on concurrent model calls; 0 = unlimited- self._max_in_flight = int(max_in_flight or 0)- if self._max_in_flight and slots_dir is None:- raise ValueError("max_in_flight needs slots_dir: the slots are lock files every process shares")- #: the machine-wide slots (``None`` when uncapped)- self.slots = Slots(self._max_in_flight, slots_dir) if self._max_in_flight else None- self._held: list[int] = [] # slot descriptors of this process's in-flight calls- now = clock()- self._states = [_KeyState(key=k, health_ts=now) for k in uniq]- self._by_key = {s.key: s for s in self._states}-- # ------------------------------------------------------------------ public- def __len__(self) -> int:- return len(self._states)-- def acquire(- self,- *,- exclude: set[str] | frozenset[str] | None = None,- timeout_s: float | None = ACQUIRE_TIMEOUT_S,- ) -> str:- """Next usable key, blocking up to ``timeout_s`` (slot wait + key wait) while all keys cool.- ``exclude`` skips keys that already failed this call; raises ``KeyPoolExhausted`` at once- when no key can become available before the deadline."""- slot = None- if self.slots is not None:- t0 = self._clock()- slot = self.slots.take(timeout_s)- if timeout_s is not None:- # the slot wait spent part of the budget; the key wait gets the rest- timeout_s = max(0.0, timeout_s - (self._clock() - t0))- try:- key = self._acquire_key(exclude, timeout_s)- except BaseException:- if slot is not None:- Slots.give(slot)- raise- self._hold(slot)- return key-- def lease_session_key(self, leases: Path | str, *, timeout_s: float | None = ACQUIRE_TIMEOUT_S) -> tuple[str, int | None]:- """A key for a vendor-CLI session: rotation, cooldowns and benching like :meth:`acquire`- but no in-flight slot, and one live session per key machine-wide while keys suffice- (a ``flock`` on ``<leases>/<key hash>.lock``). Returns ``(key, lease)``; close the lease- with :meth:`end_lease`. When every ready key is in a session one is shared (lease ``None``)."""- root = Path(leases)- root.mkdir(parents=True, exist_ok=True)- busy: set[str] = set()- while True:- try:- key = self._acquire_key(busy, 0.0 if busy else timeout_s)- except KeyPoolExhausted:- if not busy:- raise- return self._acquire_key(None, timeout_s), None # every ready key is in a session- fd = os.open(root / f"{hashlib.sha256(key.encode()).hexdigest()[:16]}.lock", os.O_RDWR | os.O_CREAT, 0o666)- try:- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)- except BlockingIOError:- os.close(fd)- busy.add(key)- continue- return key, fd-- @staticmethod- def end_lease(lease: int | None) -> None:- if lease is not None:- os.close(lease)-- def _hold(self, slot: int | None) -> None:- with self._lock:- self._in_flight += 1- if slot is not None:- self._held.append(slot)-- def _acquire_key(self, exclude: set[str] | frozenset[str] | None, timeout_s: float | None) -> str:- deadline = None if timeout_s is None else self._clock() + timeout_s- while True:- with self._lock:- now = self._clock()- chosen = self._pick(now, exclude or ())- if chosen is not None:- return chosen.key- soonest = min(- (max(now, s.cooldown_until) for s in self._states if s.key not in (exclude or ())),- default=float("inf"),- )- if soonest == float("inf"):- raise KeyPoolExhausted("all keys excluded")- if deadline is not None and soonest > deadline:- raise KeyPoolExhausted(- f"all {len(self._states)} keys cooling or dead; the earliest becomes "- f"available in {soonest - now:.0f}s (> {timeout_s}s wait budget)"- )- wait = max(0.01, min(soonest - now, MAX_WAIT_S))- if deadline is not None and self._clock() + wait > deadline:- raise KeyPoolExhausted(f"all {len(self._states)} keys cooling; waited {timeout_s}s")- self._sleep(wait)-- def release(self) -> None:- """Mark one :meth:`acquire`-d call finished (call in a ``finally``); a stray release is a no-op."""- with self._lock:- if self._in_flight <= 0:- return- self._in_flight -= 1- slot = self._held.pop() if self._held else None- if slot is not None:- Slots.give(slot)-- def report(self, key: str, outcome: Outcome) -> None:- """Feed back a call's result."""- with self._lock:- st = self._by_key.get(key)- if st is None:- return- now = self._clock()- st.recover(now)- if outcome == "ok":- st.n_ok += 1- st.health = min(1.0, st.health + 0.2 * (1.0 - st.health))- elif outcome == "429":- st.n_429 += 1- st.health *= 0.5- # rotate to another key rather than sit out a long delay- cool = min(self._cooldown_s, MAX_WAIT_S)- st.cooldown_until = max(st.cooldown_until, now + cool)- elif outcome == "5xx":- st.n_5xx += 1- st.health *= 0.8- elif outcome == "dead":- # key-scoped failure: bench until the long cooldown passes- st.health = 0.0- st.dead_until = max(st.dead_until, now + self._dead_cooldown_s)- st.cooldown_until = max(st.cooldown_until, st.dead_until)- elif outcome != "skip":- st.n_error += 1- st.health *= 0.9-- def stats(self) -> dict[str, int]:- """The reported outcomes summed over the keys (what the concurrency probe reads)."""- with self._lock:- return {- "ok": sum(s.n_ok for s in self._states),- "429": sum(s.n_429 for s in self._states),- "5xx": sum(s.n_5xx for s in self._states),- "error": sum(s.n_error for s in self._states),- }-- # ----------------------------------------------------------------- private- def _pick(self, now: float, exclude) -> _KeyState | None:- """Round-robin: the first healthy ready key, else the healthiest ready key."""- n = len(self._states)- best: _KeyState | None = None- best_idx = -1- for step in range(n):- idx = (self._rr + step) % n- st = self._states[idx]- if st.key in exclude:- continue- st.recover(now)- if st.cooldown_until > now:- continue- if st.health >= MIN_HEALTH:- self._rr = (idx + 1) % n- return st- if best is None or st.health > best.health:- best, best_idx = st, idx- if best is not None:- self._rr = (best_idx + 1) % n- return best---# ===================================================================== retry-#: a key below this health is passed over while a healthier one is ready-MIN_HEALTH = 0.3-RETRY_DEADLINE_S = 1800.0 # ceiling of one logical call, retries included--if TYPE_CHECKING: # pragma: no cover- from c3d_flywheel.models.base import ModelError--log = logging.getLogger(__name__)---def cause_for(err: BaseException, exc: BaseException | None) -> BaseException | None:- """The cause to raise ``err`` from, ``None`` when it would be ``err`` itself (a- self-cause chain recurses forever in naive walkers)."""- return None if exc is err else exc--def backoff_delay(- attempt: int, *, base_delay: float, max_delay: float, jitter: bool = True-) -> float:- """Delay before retry number ``attempt`` (1-based): ``base * 2**(attempt-1)`` capped,- with full jitter (uniform in ``[delay/2, delay]``) when ``jitter`` is on."""- raw = min(max_delay, base_delay * (2 ** max(0, attempt - 1)))- if not jitter:- return raw- return random.uniform(raw / 2.0, raw)---def rotate_with_retries[T](- pool: KeyPool,- call: Callable[[str], T],- *,- classify: Callable[[BaseException], ModelError],- outcome_of: Callable[[ModelError], Outcome] = failure_outcome,- max_attempts: int = 6,- base_delay: float = 1.0,- max_delay: float = MAX_WAIT_S,- storm_attempts: int = 60,- storm_max_delay: float = MAX_WAIT_S,- max_total_s: float = RETRY_DEADLINE_S,- sleep: Callable[[float], None] = time.sleep,- monotonic: Callable[[], float] = time.monotonic,- on_free_retry: Callable[[ModelError], bool] | None = None,- label: str = "model",- stats: dict[str, Any] | None = None,-) -> T:- """Run ``call(key)`` against a rotating :class:`KeyPool` until it succeeds.-- * success: report ok, bench keys that looked dead, return.- * ``on_free_retry(err)`` true: retry at once, free (e.g. thinking-config downgrade).- * dead key (auth failure): rotate free; benched only once a sibling key succeeds.- * 429 / 503 / 529: the key joins the call's failed set; rotation to an untried key is- free. Only when every key has failed within this call is a 503 a capacity storm:- up to ``storm_attempts`` waits (capped at ``storm_max_delay``) outside ``max_attempts``.- * other retryable errors: exponential backoff + jitter up to ``max_attempts``;- non-retryable raise at once.- * everything is bounded by ``max_total_s``.-- ``stats`` receives the ``attempts`` and ``storm`` counts.- """- failed_keys: set[str] = set() # keys that 429'd / 503'd or looked dead during this call- dead_keys: set[str] = set()- counts = {"attempts": 0, "storm": 0}-- def bench() -> None:- # bench dead-looking keys once another key got past the auth layer- for k in dead_keys:- pool.report(k, "dead")- dead_keys.clear()-- def run_one(key: str) -> _Try:- """One round-trip on ``key``: report to the pool and release the slot."""- counts["attempts"] += 1- try:- try:- result = call(key)- except Exception as exc: # noqa: BLE001 - classification is delegated- err = classify(exc)- outcome = outcome_of(err)- # a dead-looking key is benched later, by bench()- pool.report(key, "error" if outcome == "dead" else outcome)- return _Try(key, None, err, exc, outcome)- pool.report(key, "ok")- return _Try(key, result, None, None, "ok")- finally:- pool.release()-- last_err: ModelError | None = None- last_exc: BaseException | None = None- attempt = 0- storm = 0- deadline = monotonic() + max_total_s if max_total_s > 0 else float("inf")-- def out_of_time() -> bool:- return monotonic() >= deadline-- def clip(delay: float) -> float:- """A sleep never outlives the budget."""- return delay if deadline == float("inf") else min(delay, max(0.0, deadline - monotonic()))-- def gave_up(err: ModelError) -> ModelError:- log.warning("%s giving up after %.0f s of retrying (%s)", label, max_total_s, err)- return err-- try:- while attempt < max_attempts:- if out_of_time() and last_err is not None:- # free rotations must not out-live the budget- raise gave_up(last_err) from cause_for(last_err, last_exc)- attempt += 1- # skip dead-looking keys; skip throttled ones while an untried key remains- exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())- budget_left = None if deadline == float("inf") else max(0.0, deadline - monotonic())- try:- key = pool.acquire(- exclude=exclude,- timeout_s=ACQUIRE_TIMEOUT_S if budget_left is None- else min(ACQUIRE_TIMEOUT_S, budget_left),- )- except KeyPoolExhausted as exc:- raise classify(exc) from exc- t = run_one(key)- if t.err is None:- bench()- return t.result- err, exc = t.err, t.exc- last_err = err- last_exc = exc- free = False- if on_free_retry is not None and on_free_retry(err):- free = True- elif t.outcome == "dead":- dead_keys.add(key)- failed_keys.add(key)- if free:- attempt -= 1- continue- if t.outcome == "dead":- if len(dead_keys) < len(pool):- if out_of_time():- raise gave_up(err) from cause_for(err, exc)- log.warning("%s key …%s looks dead (%s); rotating", label, key[-4:], err)- attempt -= 1- continue- raise err from cause_for(err, exc) # every key failed the same way: not the keys' fault- if err.retryable and err.status in (503, 529) and storm < storm_attempts and not out_of_time():- failed_keys.add(key)- if len(failed_keys) < len(pool):- # a 503 is per key: while an untried key remains rotation is free- attempt -= 1- log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried)",- label, key[-4:], err, len(failed_keys), len(pool))- continue- # capacity storm: every key said 503, so wait (own budget, not max_attempts)- storm += 1- counts["storm"] = storm- # jitter first, cap last, so the wait never exceeds the cap- raw = base_delay * (2 ** min(storm, 8)) * (0.75 + 0.5 * random.random())- delay = clip(min(storm_max_delay, raw))- log.warning("%s capacity storm %d/%d (%s); waiting %.0fs",- label, storm, storm_attempts, err, delay)- sleep(delay)- attempt -= 1- continue- if t.outcome == "429":- failed_keys.add(key)- if len(failed_keys) < len(pool):- if out_of_time():- raise gave_up(err) from cause_for(err, exc)- attempt -= 1- log.warning(- "%s key …%s throttled (%s); rotating to a fresh key", label, key[-4:], err- )- sleep(clip(0.5))- continue- if not err.retryable or attempt >= max_attempts:- bench()- raise err from cause_for(err, exc)- if out_of_time():- raise gave_up(err) from cause_for(err, exc)- delay = backoff_delay(attempt, base_delay=base_delay, max_delay=max_delay)- log.warning(- "%s attempt %d/%d failed (%s); retrying in %.1fs",- label, attempt, max_attempts, err, delay,- )- sleep(clip(delay))- assert last_err is not None- bench()- raise last_err- except BaseException as exc:- if hasattr(exc, "attempts"): # a ModelError: how hard the call tried- exc.attempts = counts["attempts"]- raise- finally:- if stats is not None:- stats.update(counts)---@dataclass(frozen=True)-class _Try:- """The outcome of one round-trip on one key (``rotate_with_retries.run_one``)."""-- key: str- result: Any- err: ModelError | None- exc: BaseException | None- outcome: strdiff --git a/flywheel/c3d_flywheel/models/schema_utils.py b/flywheel/c3d_flywheel/models/schema_utils.pydeleted file mode 100644index e6e6e14a..00000000--- a/flywheel/c3d_flywheel/models/schema_utils.py+++ /dev/null@@ -1,263 +0,0 @@-"""JSON-schema adapters for the three providers, a fence-tolerant JSON parser and-``ask_structured`` (one schema-validated call → object | None, usage, error).--Pydantic schemas are rewritten per provider: Gemini takes an OpenAPI-ish subset, OpenAI strict-needs closed objects with every property required, Anthropic takes inlined standard JSON schema.-All functions are pure and return new dicts.-"""--from __future__ import annotations--import copy-import json-import re-from collections.abc import Callable, Sequence-from typing import Any, Literal--from pydantic import BaseModel, ValidationError--from c3d_flywheel.models.base import ModelError-from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart-from codeverse3d.contracts.common import Usage-from codeverse3d.jsonparse import ( # noqa: F401- JsonParseError,- parse_json_lenient,- strip_control_chars,-)--# keys Gemini's Schema type rejects or ignores-_GEMINI_DROP = {- "title",- "default",- "examples",- "example",- "$schema",- "$id",- "additionalProperties",- "exclusiveMinimum",- "exclusiveMaximum",- "discriminator",- "readOnly",- "writeOnly",- "deprecated",- "uniqueItems",- "multipleOf",- "contentMediaType",- "contentEncoding",-}---# ----------------------------------------------------------------- $ref inline-def inline_refs(schema: dict[str, Any]) -> dict[str, Any]:- """Copy with every local ``$ref`` replaced by its definition; recursive schemas raise ``ValueError``."""- root = copy.deepcopy(schema)- defs: dict[str, Any] = {}- for table in ("$defs", "definitions"):- defs.update(root.pop(table, {}) or {})-- def resolve(node: Any, stack: tuple[str, ...]) -> Any:- if isinstance(node, list):- return [resolve(n, stack) for n in node]- if not isinstance(node, dict):- return node- if "$ref" in node:- ref = node["$ref"]- name = ref.rsplit("/", 1)[-1]- if name not in defs:- raise ValueError(f"unresolvable $ref {ref!r}")- if name in stack:- raise ValueError(f"recursive $ref {ref!r} cannot be inlined")- merged = dict(defs[name])- for k, v in node.items():- if k != "$ref":- merged[k] = v- return resolve(merged, stack + (name,))- return {k: resolve(v, stack) for k, v in node.items()}-- return resolve(root, ())---# ------------------------------------------------------------------- helpers-def _is_null(s: Any) -> bool:- return (- isinstance(s, dict) and s.get("type") == "null" and len([k for k in s if k != "type"]) == 0- )---def _merge_prefix_items(node: dict[str, Any]) -> dict[str, Any]:- """``prefixItems`` becomes ``items`` (union if heterogeneous)."""- items = node.pop("prefixItems")- if not isinstance(items, list) or not items:- return node- uniq: list[dict[str, Any]] = []- for it in items:- if it not in uniq:- uniq.append(it)- node["items"] = uniq[0] if len(uniq) == 1 else {"anyOf": uniq}- node.setdefault("minItems", len(items))- node.setdefault("maxItems", len(items))- return node---# --------------------------------------------------------------------- gemini-def to_gemini_schema(schema: dict[str, Any]) -> dict[str, Any]:- """Sanitise a pydantic JSON schema for Gemini ``response_schema`` / function ``parameters``."""- root = inline_refs(schema)-- def walk(node: Any) -> Any:- if isinstance(node, list):- return [walk(n) for n in node]- if not isinstance(node, dict):- return node- node = {k: v for k, v in node.items() if k not in _GEMINI_DROP}- if "const" in node:- node["enum"] = [node.pop("const")]- node.setdefault("type", _json_type(node["enum"][0]))- if isinstance(node.get("type"), list):- types = [t for t in node["type"] if t != "null"]- if len(types) != len(node["type"]):- node["nullable"] = True- node["type"] = types[0] if len(types) == 1 else types- for key in ("anyOf", "oneOf"):- if key in node:- variants = [v for v in node[key] if not _is_null(v)]- if len(variants) != len(node[key]):- node["nullable"] = True- if len(variants) == 1:- inner = walk(variants[0])- merged = {**inner, **{k: v for k, v in node.items() if k != key}}- node = merged- else:- node.pop(key)- node["anyOf"] = [walk(v) for v in variants]- if "allOf" in node and len(node["allOf"]) == 1:- inner = walk(node.pop("allOf")[0])- node = {**inner, **node}- if "prefixItems" in node:- node = _merge_prefix_items(node)- # Gemini enums are strings only- if "enum" in node:- node["enum"] = [str(v) if not isinstance(v, str) else v for v in node["enum"]]- node["type"] = "string"- node.pop("format", None)- if (- "format" in node- and node.get("type") == "string"- and node["format"] not in ("date-time", "enum")- ):- node.pop("format")- if "properties" in node:- props = {k: walk(v) for k, v in node["properties"].items()}- node["properties"] = props- node.setdefault("type", "object")- if props:- node["propertyOrdering"] = list(props.keys())- else:- # Gemini rejects empty OBJECT properties- node.pop("properties")- node.pop("propertyOrdering", None)- if "items" in node:- node["items"] = walk(node["items"])- return node-- return walk(root)---def _json_type(value: Any) -> str:- if isinstance(value, bool):- return "boolean"- if isinstance(value, int):- return "integer"- if isinstance(value, float):- return "number"- if isinstance(value, str):- return "string"- if isinstance(value, list):- return "array"- return "object"---# ------------------------------------------------------------ openai / anthropic-def _walk_schema(root: Any, on_object: Callable[[dict[str, Any], Callable[[Any], Any]], None]) -> Any:- """Walk shared by the openai-strict and anthropic schemas; ``on_object(node, walk)`` rewrites ``properties``."""-- def walk(node: Any) -> Any:- if isinstance(node, list):- return [walk(n) for n in node]- if not isinstance(node, dict):- return node- node = {- k: v for k, v in node.items() if k not in ("title", "default", "examples", "example")- }- if "prefixItems" in node:- node = _merge_prefix_items(node)- if "properties" in node:- on_object(node, walk)- for key in ("anyOf", "oneOf", "allOf"):- if key in node:- node[key] = [walk(v) for v in node[key]]- if "items" in node:- node["items"] = walk(node["items"])- return node-- return walk(root)---def to_openai_strict_schema(schema: dict[str, Any]) -> dict[str, Any]:- """OpenAI strict schema: closed objects, every property required. Null is allowed only- where the source allows it; a non-null ``default`` becomes a ``[default: …]`` hint."""-- def strict_object(node: dict[str, Any], walk: Callable[[Any], Any]) -> None:- props = {k: walk(v) for k, v in node["properties"].items()}- required = set(node.get("required", []))- for name, sub in props.items():- original = node["properties"][name]- if name in required or not isinstance(original, dict):- continue- if original.get("default") is not None:- props[name] = _with_default_hint(sub, original["default"])- node["properties"] = props- node["required"] = list(props.keys())- node["additionalProperties"] = False- node.setdefault("type", "object")-- return _walk_schema(inline_refs(schema), strict_object)---def _with_default_hint(sub: dict[str, Any], default: Any) -> dict[str, Any]:-- hint = f"[default: {json.dumps(default)}]"- desc = str(sub.get("description") or "").strip()- return {**sub, "description": f"{desc} {hint}".strip()}---def to_anthropic_schema(schema: dict[str, Any]) -> dict[str, Any]:- """Standard JSON schema, refs inlined, objects closed."""-- def closed_object(node: dict[str, Any], walk: Callable[[Any], Any]) -> None:- node["properties"] = {k: walk(v) for k, v in node["properties"].items()}- node.setdefault("type", "object")- node.setdefault("additionalProperties", False)-- return _walk_schema(inline_refs(schema), closed_object)---# ------------------------------------------------------------------ JSON parse---def ask_structured(model: Any, schema: type[BaseModel], *, system: str, text: str,- images: Sequence[ImagePart] = (), temperature: float,- label: str, thinking: Literal["off", "low", "medium", "high"] = "low") -> tuple[Any, Usage, str]:- """One structured call → ``(validated object | None, usage, error)``; failures are returned, not raised."""- req = ChatRequest(messages=[ChatMessage.user(text, images=list(images) or None)], system=system,- response_schema=schema.model_json_schema(), temperature=temperature,- thinking=thinking, max_output_tokens=65_536, max_wait_s=900.0, label=label)- try:- resp = model.generate(req)- except ModelError as e: # e.usage is what the provider ALREADY billed for the failure- return None, e.usage, f"call failed: {e}"- try:- payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)- return schema.model_validate(payload), resp.usage, ""- except (JsonParseError, ValidationError, TypeError) as e:- return None, resp.usage, f"answer unparsable: {e}"diff --git a/flywheel/c3d_flywheel/record/record.py b/flywheel/c3d_flywheel/record/record.pyindex b98f6299..e037a3c5 100644--- a/flywheel/c3d_flywheel/record/record.py+++ b/flywheel/c3d_flywheel/record/record.py@@ -1,30 +1,17 @@-"""Run records: finalise / load / iterate.--``finalize_record`` fills the environment, reads the run's total off its cost ledger, writes-``record.json`` (telemetry included) atomically. Telemetry is best-effort and never fails a run.-"""+"""Reading the recorded runs: ``load_record`` / ``iter_runs`` and the derived per-round views."""from __future__ import annotationsimport logging-import platform-import subprocessfrom collections.abc import Callable, Iterable, Iterator-from contextlib import contextmanager-from datetime import UTC, datetime-from functools import lru_cachefrom pathlib import Pathfrom typing import Any, NamedTuple-from codeverse3d import __version__-from codeverse3d.config import get_settingsfrom codeverse3d.contracts.artifacts import Judgment-from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord-from c3d_flywheel.cost.ledger import existing_ledger_path, ledger_usage, load_ledger-from codeverse3d.proc import read_json_or_none, version_line-from codeverse3d.spatial.node import node_modules_dir, puppeteer_cache_dirfrom codeverse3d.workspace import Workspace+from c3d_flywheel.legacy import RoundRecord, RunId, RunRecord+log = logging.getLogger(__name__)@@ -32,79 +19,6 @@ class RecordError(RuntimeError):"""A record.json is missing or invalid."""-# --------------------------------------------------------------------------- environment-def _cmd_first_line(cmd: list[str]) -> str:- rc, line = version_line(cmd)- return line or f"exit {rc}"---def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str] | None:- try:- return subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, timeout=10, check=False)- except (OSError, subprocess.TimeoutExpired):- return None---def _harness_git_sha(pkg_file: Path | None = None) -> str:- """Commit of the harness checkout this package runs from (``""`` outside a checkout).-- The file itself must be tracked (a wheel or venv copy must not borrow an unrelated repo's- sha); a modified tree is marked ``-dirty``.- """- here = (pkg_file or Path(__file__)).resolve()- tracked = _git(["ls-files", "--error-unmatch", here.name], here.parent)- if tracked is None or tracked.returncode != 0:- return ""- head = _git(["rev-parse", "HEAD"], here.parent)- if head is None or head.returncode != 0 or not head.stdout.strip():- return ""- sha = head.stdout.strip()- # dirty = any tracked file under the package tree (harness/) modified; untracked files ignored- pkg_root = here.parents[2] if here.parent.name == "record" else here.parent- status = _git(["status", "--porcelain", "--untracked-files=no", "--", str(pkg_root)], here.parent)- if status is not None and status.returncode == 0 and status.stdout.strip():- sha += "-dirty"- return sha---def _chrome_version() -> str:- """Chrome build from the puppeteer cache directory name (offline, no launch)."""- cache = puppeteer_cache_dir() / "chrome"- if not cache.is_dir():- return ""- builds = sorted(p.name for p in cache.iterdir() if p.is_dir())- return builds[-1] if builds else ""---def _node_pkg_version(name: str) -> str:- try:- pkg = node_modules_dir() / name / "package.json"- except RuntimeError: # no runtime_js: provenance only- return ""- return str((read_json_or_none(pkg) or {}).get("version", ""))---@lru_cache(maxsize=1)-def environment_versions() -> dict[str, str]:- """Tool versions + harness provenance. Cached per process (Blender is slow to ask)."""- s = get_settings()- env: dict[str, str] = {- "python": platform.python_version(),- "codeverse3d": __version__,- "platform": platform.platform(),- "host": platform.node(),- "harness_git_sha": _harness_git_sha(),- }- blender = s.resolve_blender()- env["blender"] = _cmd_first_line([blender, "--version"]) if blender else ""- env["blender_path"] = blender- env["node"] = _cmd_first_line([s.binaries.node, "--version"])- env["three"] = _node_pkg_version("three")- env["puppeteer"] = _node_pkg_version("puppeteer")- env["chrome"] = _chrome_version()- return env--# --------------------------------------------------------------------------- derived fieldsdef effective_judgment(r: RoundRecord):"""The round's judgment, or ``None`` when degraded (never a 0.0 score)."""@@ -119,11 +33,9 @@ def previous_verdict(rounds: Iterable[RoundRecord | None], index: int) -> Judgmedef rubric_of(record: RunRecord) -> str:- """The run's rubric: the stamped one, else its last judged round's, else ``judges.base.run_rubric``."""- from codeverse3d.judges.base import run_rubric-+ """The run's rubric: the stamped one, else its last judged round's, else ``""``."""judged = (r.judgment.rubric for r in reversed(record.rounds) if r.judgment is not None and r.judgment.rubric)- return str(record.extra.get("rubric") or "") or next(judged, "") or run_rubric(record.spec)+ return str(record.extra.get("rubric") or "") or next(judged, "")def stop_reason(record: RunRecord) -> str:@@ -169,90 +81,6 @@ def round_summary(r: RoundRecord) -> dict[str, Any]:}-def fill_derived(record: RunRecord) -> RunRecord:- """Fill finished_at/environment when absent and the complexity block."""- if record.finished_at is None:- record.finished_at = datetime.now(UTC)- env = dict(environment_versions())- env.update(record.environment) # the track's own entries win- record.environment = env- block = complexity_block(record)- if block is not None:- record.extra["complexity"] = block- return record---def complexity_block(record: RunRecord) -> dict[str, Any] | None:- """``record.extra["complexity"]``: the last measured round's vector, plus plan parts and the per-round trail."""- cx = next((c for c in (round_complexity(r) for r in reversed(record.rounds)) if c), None)- if cx is None:- return None- plan_parts = len(record.plan.parts) if record.plan is not None else 0- block = dict(cx)- block["plan_parts"] = plan_parts- block["parts_per_plan_part"] = (- round(int(cx.get("part_count") or 0) / plan_parts, 3) if plan_parts else None- )- trail = [c["index"] for c in (round_complexity(r) for r in record.rounds) if c]- block["by_round"] = trail- return block---# --------------------------------------------------------------------------- io-def package_run(ws: Workspace, record: RunRecord) -> None:- """Read the run's total off its ledger (``record.total_usage`` = its sum at list price).-- Best-effort: a failure is logged and the run still gets its ``record.json``."""- from c3d_flywheel.record.telemetry import build_telemetry-- if existing_ledger_path(ws.root) is not None:- record.total_usage = ledger_usage(load_ledger(ws.root))- try:- ws.ensure_layout()- except OSError as e: # read-only mount- log.warning("run layout not created in %s: %s", ws.root, e)- return- try:- record.telemetry = build_telemetry(ws, record)- except Exception as e: # noqa: BLE001 - never fail a finished run over accounting- log.warning("telemetry not written for %s: %s", ws.root, e)---def repackage(ws: Workspace) -> RunRecord | None:- """Re-package a finished run's ``record.json`` after its ledger grew; None when there is no ledger or record."""- if existing_ledger_path(ws.root) is None:- return None- try:- record = load_record(ws)- except RecordError:- return None- package_run(ws, record)- ws.write_json(ws.record_path, record)- return record---@contextmanager-def join_post_run(ws: Workspace) -> Iterator[None]:- """Work on a finished run that may spend money: rows join its ledger, and the record is- re-packaged on the way out (even on error) so ``total_usage`` stays the ledger's sum."""- from c3d_flywheel.cost.instrument import run_ledger-- try:- with run_ledger(ws.root, run=ws.root.name, create=False):- yield- finally:- repackage(ws)---def finalize_record(ws: Workspace, record: RunRecord) -> Path:- """Fill derived fields + environment, build the telemetry block and write ``record.json``."""- record.workspace = record.workspace or str(ws.root)- fill_derived(record)- package_run(ws, record)- ws.write_json(ws.record_path, record)- return ws.record_path--def load_record(ws: Workspace | Path | str) -> RunRecord:ws = ws if isinstance(ws, Workspace) else Workspace(ws)if not ws.record_path.is_file():diff --git a/flywheel/c3d_flywheel/record/telemetry.py b/flywheel/c3d_flywheel/record/telemetry.pyindex 5ca20915..7b9108e8 100644--- a/flywheel/c3d_flywheel/record/telemetry.py+++ b/flywheel/c3d_flywheel/record/telemetry.py@@ -1,102 +1,19 @@-"""Run telemetry, stored in ``record.telemetry``: how the run was configured (the settings snapshot)-and a summary of the ledger rows. Builders degrade to what could be read."""+"""The cost summary of one run, from its ledger (``record.telemetry`` when the record lacks one)."""from __future__ import annotations-import hashlib-import jsonimport logging-from codeverse3d.config import get_settings-from codeverse3d.contracts.run import (- CostSummary,- RoleSettings,- RunRecord,- RunTelemetry,- SettingsSnapshot,- StageCost,-)+from codeverse3d.workspace import Workspace+from c3d_flywheel.cost.ledger import load_ledger, summarisefrom c3d_flywheel.cost.types import STAGE_ORDER, CallCost-from c3d_flywheel.record.record import effective_score, rubric_of-from codeverse3d.workspace import Workspace+from c3d_flywheel.legacy import CostSummary, RunRecord, RunTelemetry, StageCost+from c3d_flywheel.record.record import effective_scorelog = logging.getLogger(__name__)-# --------------------------------------------------------------------------- settings snapshot-def price_table_version() -> str:- """Content hash of the USD price table."""- from c3d_flywheel.models.pricing import PRICES-- payload = json.dumps(- {f"{p}/{m}": [v.input, v.output, v.cached, v.cache_write, v.approximate] for (p, m), v in sorted(PRICES.items())},- sort_keys=True,- )- return hashlib.sha256(payload.encode()).hexdigest()[:12]---def backend_kind(model_id: str) -> str:- """``gemini:x`` → ``gemini``."""- return model_id.split(":", 1)[0] if ":" in model_id else model_id---def rubric_hash(name: str) -> str:- if not name:- return ""- try:- from codeverse3d.judges.rubrics import load_rubric-- return load_rubric(name).content_hash()- except Exception as e: # unknown rubric / package missing- log.debug("rubric hash unavailable for %s: %s", name, e)- return ""---def _role_settings(record: RunRecord) -> list[RoleSettings]:- """The model per role from the spec; sampling knobs from ``record.extra["sampling"]``, else left empty."""- b = record.spec.backends- sampling = record.extra.get("sampling") or {}- roles: list[RoleSettings] = []- for role, model_id in (("planner", b.planner), ("generator", b.generator), ("judge", b.judge),- ("captioner", b.captioner)):- knobs = sampling.get(role) or {}- temperature = knobs.get("temperature")- roles.append(RoleSettings(- role=role, model=model_id, backend=backend_kind(model_id), thinking=str(knobs.get("thinking") or ""),- temperature=float(temperature) if isinstance(temperature, (int, float)) else None,- n_samples=knobs.get("n_samples") if role == "judge" else None, source="run" if knobs else "spec",- ))- return roles---def settings_snapshot(record: RunRecord) -> SettingsSnapshot:- """The resolved configuration block (no secrets)."""- spec = record.spec- rubric = rubric_of(record)- s = get_settings()- render = s.render.model_dump(mode="json")- keys = len(s.gemini_api_keys)- env = dict(record.environment)- return SettingsSnapshot(- roles=_role_settings(record),- budget=spec.budget.model_dump(mode="json"),- rubric=rubric,- rubric_hash=rubric_hash(rubric),- prompt_hashes=dict(record.prompt_hashes),- tool_versions={k: v for k, v in env.items() if k in _TOOL_KEYS},- harness_version=env.get("codeverse3d", ""),- harness_git_sha=env.get("harness_git_sha", ""),- key_pool_size=keys,- price_table_version=price_table_version(),- render=render,- )---_TOOL_KEYS = frozenset({"python", "platform", "host", "blender", "blender_path", "node", "three",- "chrome", "puppeteer", "codeverse3d", "harness_git_sha"})--# --------------------------------------------------------------------------- the ledger (c3d_flywheel.cost)def ledger_rows(ws: Workspace) -> list[CallCost]:"""The run's priced per-call rows (none without a ledger)."""@@ -130,7 +47,6 @@ def cost_summary(record: RunRecord, rows: list[CallCost]) -> CostSummary:)-# --------------------------------------------------------------------------- builddef build_telemetry(ws: Workspace, record: RunRecord) -> RunTelemetry:- """Compute the settings snapshot and the cost summary of one run (``record.telemetry``)."""- return RunTelemetry(settings=settings_snapshot(record), cost=cost_summary(record, ledger_rows(ws)))+ """The cost summary of one run, computed from its ledger (no settings snapshot: that is only ever read)."""+ return RunTelemetry(cost=cost_summary(record, ledger_rows(ws)))diff --git a/flywheel/c3d_flywheel/skill_targets.py b/flywheel/c3d_flywheel/skill_targets.pydeleted file mode 100644index f6d2bfbe..00000000--- a/flywheel/c3d_flywheel/skill_targets.py+++ /dev/null@@ -1,392 +0,0 @@-"""What each bundle CLAIMS to move — one deterministic quantity per skill."""--from __future__ import annotations--import importlib-import re-import tomllib-from dataclasses import dataclass, field-from pathlib import Path-from typing import Any--from codeverse3d.skills.model import Skill--DOWN = "down"-UP = "up"--SRC_GATE = "gate_kinds"-SRC_BUILD = "build"-SRC_GLB = "glb"-SRC_FRAMES = "frames"-SRC_ARTIFACT = "artifact"---@dataclass(frozen=True)-class Target:-- skill: str- metric: str- direction: str- unit: str- source: str- why: str- kinds: tuple[str, ...] = ()- severities: tuple[str, ...] = ()- languages: tuple[str, ...] = field(default_factory=tuple)- measurable: bool = True- caveat: str = ""---OBJECT_LANGS = ("blender", "urdf_blender", "cadquery", "threejs")--TARGETS: tuple[Target, ...] = (- Target(- skill="c3d-part-contact",- metric="interpenetrating_pairs",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("connectivity/penetration",),- languages=OBJECT_LANGS,- why="the penetrating-pair count is the strongest single predictor of assembly_fit (r = -0.258, eval/docs/COMPLEXITY.md §6)",- ),- Target(- skill="c3d-bbox-contract",- metric="contract_findings",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("contract/part_bbox", "contract/overall_bbox", "contract/footprint_offset",- "contract/ground_gap"),- languages=OBJECT_LANGS,- why="the bundle is the contract gate's own tolerance arithmetic: a part that is there at the wrong size or place",- caveat="contract/instance_* is c3d-repeats-and-mirrors; contract/missing_part is a language-API failure, not a tolerance one.",- ),- Target(- skill="c3d-repeats-and-mirrors",- metric="contract_instance_findings",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("contract/instance_bbox", "contract/instance_count"),- languages=OBJECT_LANGS,- why="metadata.owns already names these two kinds; they are the whole claim",- ),- Target(- skill="c3d-blender-forms",- metric="blender_lint_findings",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("lint/*",),- languages=("blender", "urdf_blender"),- why="the bundle is written around the five bpy traps the static lint already counts",- ),- Target(- skill="c3d-urdf-joints",- metric="joint_sweep_errors",- direction=DOWN,- unit="ERROR findings per run",- source=SRC_GATE,- kinds=("joint_sweep/penetration", "joint_sweep/unattached"),- severities=("error",),- languages=("urdf_blender",),- why="653 of 691 joint_sweep findings in the corpus are moved-pose ERRORs, which the clearance rule targets",- ),- Target(- skill="c3d-cadquery-forms",- metric="build_failure_rate",- direction=DOWN,- unit="fraction of rounds whose build failed",- source=SRC_BUILD,- languages=("cadquery",),- why="an OCC kernel refusal is a build failure, not a gate finding",- caveat="no baseline exists: eval/bench/out holds zero graded cadquery runs.",- ),- Target(- skill="c3d-threejs-forms",- metric="missing_parts",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("contract/missing_part",),- languages=("threejs",),- why="the traps that make a threejs part silently ship nothing count as a missing part",- caveat="no baseline exists: eval/bench/out holds zero graded threejs runs.",- ),- Target(- skill="c3d-scene-composition",- metric="camera_placement_findings",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("scene_frames/camera_*",),- languages=("scene_threejs",),- why="cameras below ground or inside geometry are arithmetic failures the bundle derives from measured bounds",- caveat="frame coverage (content_small, hero_*) also routes the bundle (R15) but is not in this count.",- ),- Target(- skill="c3d-scene-lighting",- metric="dark_or_flat_frames",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("scene_frames/dark_frame", "scene_frames/flat_frame", "scene_frames/blown_frame"),- languages=("scene_threejs",),- why="the bundle is the exposure thresholds of frame_metrics.py with a recipe",- ),- Target(- skill="c3d-scene-motion",- metric="min_authored_changed_frac",- direction=UP,- unit="fraction of pixels changed on the WEAKEST authored camera, t=0 → t=1.5 s",- source=SRC_FRAMES,- languages=("scene_threejs",),- measurable=False,- why="reported as a regression guard only",- caveat="NOT a target: no deterministic instrument attributes motion to a named plan item; a drop below MOVING_FRAC is still a regression.",- ),- Target(- skill="c3d-threejs-shader-traps",- metric="shader_preflight_findings",- direction=DOWN,- unit="WARN+ERROR findings per run",- source=SRC_ARTIFACT,- kinds=("shader_preflight/*",),- languages=("scene_threejs", "threejs"),- why="every trap the bundle teaches lands in the one shader_preflight report",- caveat="zero headroom in today's corpus (all 4 recorded scene runs are clean): detects a regression only.",- ),- Target(- skill="c3d-glsl-craft",- metric="mean_edge_density",- direction=UP,- unit="mean edge density over the 5 sampled frames (frame_stats)",- source=SRC_FRAMES,- languages=("glsl_shader",),- why="the gl_frames gate fired zero findings on 9 graded glsl runs; edge density is the continuous quantity it thresholds",- ),- Target(- skill="c3d-opengl-pipeline",- metric="gl_frame_findings",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("gl_frames/*",),- languages=("opengl_python",),- why="every gl_frames finding in the graphics corpus came from opengl_python",- ),- # ------------------------------------------------------------- recipe port- Target(- skill="c3d-scene-atmosphere",- metric="env_bake_orientation_flags",- direction=DOWN,- unit="inverted bakes per run (prospective census)",- source=SRC_ARTIFACT,- languages=("scene_threejs",),- measurable=False,- why="the reference audit found the sky bake inverted in 4 of 9 delivered scenes",- caveat="ported claim: reference-harness evidence, our own A/B pending.",- ),- Target(- skill="c3d-scene-water",- metric="metallic_water_planes",- direction=DOWN,- unit="water materials with metalness above 0.3 (prospective census)",- source=SRC_ARTIFACT,- languages=("scene_threejs",),- measurable=False,- why="dielectric water against a live environment bake is the reference's most-shipped material fix",- caveat="NOT yet measurable here: judged quality only; a fresnel/metalness census is the candidate instrument.",- ),- Target(- skill="c3d-scene-night",- metric="overbright_shaft_findings",- direction=DOWN,- unit="solid-reading light shafts per run (prospective census)",- source=SRC_ARTIFACT,- languages=("scene_threejs",),- measurable=False,- why="night briefs are the dark_or_flat gate's richest source",- caveat="ported claim: reference-harness evidence, our own A/B pending.",- ),- Target(- skill="c3d-scene-materials",- metric="flat_albedo_surfaces",- direction=DOWN,- unit="large surfaces with zero value variance (prospective census)",- source=SRC_ARTIFACT,- languages=("scene_threejs",),- measurable=False,- why="flat albedo and identical twins are the reference audit's most-cited surface defects",- caveat="NOT yet measurable here: judged quality only; a per-material value-spread census is the candidate instrument.",- ),- # ------------------------------------------------------------- scene_blender- Target(- skill="c3d-bpy-scene-placement",- metric="bpy_placement_errors",- direction=DOWN,- unit="ERROR findings per run",- source=SRC_GATE,- kinds=("scene_placement/floating", "scene_placement/sunken", "scene_placement/unsupported",- "scene_placement/interpenetration"),- severities=("error",),- languages=("scene_blender",),- why="the bundle seats every placed row on height_at or its support; the floating+sunken+interpenetration ERROR count is the number to move",- caveat="unsupported, cantilevered and free_exempt route the bundle (R30) but are WARN-only or per-report, so they add nothing to the count.",- ),- Target(- skill="c3d-bpy-scene-lighting",- metric="bpy_exposure_findings",- direction=DOWN,- unit="findings per run",- source=SRC_GATE,- kinds=("scene_frames/dark_frame", "scene_frames/flat_frame", "scene_frames/blown_frame"),- languages=("scene_blender",),- why="the bundle is the exposure thresholds of frame_metrics.py with the Cycles/AgX bands",- caveat="the same three kinds as c3d-scene-lighting over a disjoint language; the uniqueness test keys on (kind, language).",- ),- Target(- skill="c3d-bpy-scene-materials",- metric="flat_principled_materials",- direction=DOWN,- unit="materials whose Base Color, Roughness and Metallic are all unlinked constants (prospective census)",- source=SRC_ARTIFACT,- languages=("scene_blender",),- measurable=False,- why="placeholder_material, flat_ground and monotonous_vegetation are judge defects with no deterministic twin",- caveat="NOT yet measurable here: judged quality only; a census of linked Principled sockets is the candidate instrument.",- ),-)--BY_SKILL: dict[str, Target] = {t.skill: t for t in TARGETS}---def target_for(skill: str) -> Target | None:- return BY_SKILL.get(skill)---# ===================================================================== claims-# A claim pins a number some text states to the code constant that produces it (``_claims/<name>.toml``,-# ``prompts/_claims.toml``); the text must stand as its own token and, with ``context``, share a line with that phrase.-CLAIMS_DIR = "_claims"-PROMPT_CLAIMS = "_claims.toml"---def claims_path(skill: Skill | str, root: Path | None = None) -> Path:- from codeverse3d.skills import skills_dir-- name = skill.name if isinstance(skill, Skill) else skill- base = Path(root) if root is not None else skills_dir()- if isinstance(skill, Skill):- own = skill.dir.parent / CLAIMS_DIR / f"{name}.toml"- if own.is_file():- return own- return base / CLAIMS_DIR / f"{name}.toml"---def _load_rows(p: Path) -> list[dict[str, Any]]:- if not p.is_file():- return []- rows = tomllib.loads(p.read_text()).get("claim") or []- if not isinstance(rows, list):- raise ValueError(f"{p}: [[claim]] must be an array of tables")- return [dict(r) for r in rows]---def load_claims(skill: Skill | str, root: Path | None = None) -> list[dict[str, Any]]:- return _load_rows(claims_path(skill, root))---def resolve(dotted: str) -> Any:- mod, _, attr = dotted.partition(":") if ":" in dotted else dotted.rpartition(".")- if not mod or not attr:- raise ValueError(f"claim python target {dotted!r} must be 'module:NAME'")- return getattr(importlib.import_module(mod), attr)---def resolve_js(target: str) -> float:- rel, _, name = target.rpartition(":")- if not rel or not name:- raise ValueError(f"claim js target {target!r} must be '<path>:NAME'")- head, _, tail = rel.partition("/")- if head == "runtime_js":- from codeverse3d.spatial.node import runtime_js_dir-- path = runtime_js_dir() / tail- else:- path = Path(__file__).resolve().parents[1] / rel- m = re.search(rf"^\s*(?:export\s+)?const\s+{re.escape(name)}\s*=\s*([-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)\s*[;,]",- path.read_text(), re.M)- if m is None:- raise ValueError(f"{path} states no numeric `const {name} = …`")- return float(m.group(1))---def _target(row: dict[str, Any]) -> tuple[str, Any]:- if row.get("js"):- return f"js:{row['js']}", resolve_js(str(row["js"]))- return str(row["python"]), resolve(str(row["python"]))---def render_claim(row: dict[str, Any]) -> str:- value = _target(row)[1]- scale = row.get("scale")- fmt = row.get("format")- if row.get("sep"):- items = [str(fmt or "{:g}").format(v * scale if scale is not None else v) for v in value]- head, last = items[:-1], items[-1:]- return str(row["sep"]).join(head) + (str(row.get("last_sep") or row["sep"]) if head else "") + "".join(last)- if scale is not None:- value = value * scale- return format(value, "") if not fmt else str(fmt).format(value)---def stated(text: str, doc: str, context: str = "") -> bool:- pat = re.compile(rf"(?<![\w.]){re.escape(text)}(?!\w|\.\d)")- return any(pat.search(line) and context in line for line in doc.splitlines())---def check_rows(rows: list[dict[str, Any]], read: Any) -> list[str]:- issues: list[str] = []- for i, row in enumerate(rows):- where = f"claim[{i}] {row.get('key', '?')}"- missing = [k for k in ("key", "text") if not row.get(k)]- if not row.get("python") and not row.get("js"):- missing.append("python` or `js")- if missing:- issues += [f"{where}: missing `{k}`" for k in missing]- continue- try:- live = render_claim(row)- except Exception as e: # noqa: BLE001- issues.append(f"{where}: cannot resolve {row.get('js') or row.get('python')!r}: {e}")- continue- try:- doc = read(row.get("file"))- except OSError as e:- issues.append(f"{where}: cannot read {row.get('file')!r}: {e}")- continue- if live != row["text"]:- issues.append(f"{where}: the constant now renders {live!r}, the text says {row['text']!r}")- elif not stated(row["text"], doc, str(row.get("context", ""))):- ctx = f" on a line with {row['context']!r}" if row.get("context") else ""- issues.append(f"{where}: {row['text']!r} no longer appears{ctx} in {row.get('file') or 'the body'}")- return issues---def check_claims(skill: Skill, root: Path | None = None) -> list[str]:- try:- rows = load_claims(skill, root)- except (ValueError, OSError) as e:- return [f"claims file unreadable: {e}"]- return check_rows(rows, lambda f: skill.body if not f else (skill.dir / f).read_text())---def check_prompt_claims(root: Path | None = None) -> list[str]:- base = Path(root) if root is not None else Path(__file__).resolve().parents[1] / "prompts"- try:- rows = _load_rows(base / PROMPT_CLAIMS)- except (ValueError, OSError) as e:- return [f"prompt claims file unreadable: {e}"]- return check_rows(rows, lambda f: (base / f).read_text() if f else "")diff --git a/flywheel/c3d_flywheel/vlm_judge.py b/flywheel/c3d_flywheel/vlm_judge.pydeleted file mode 100644index c8cf8e2d..00000000--- a/flywheel/c3d_flywheel/vlm_judge.py+++ /dev/null@@ -1,405 +0,0 @@-"""``VlmJudge``: rubric YAML + labelled renders + measurements → ``Judgment``.--Per sample, one ``ChatRequest`` with a per-rubric JSON schema; samples differ by montage order so the-n-sample std measures judge noise. Score, penalties, floors, caps and pass are computed in code-(``rubrics.aggregate_samples``). Retries stay inside one ``sample_budget_s``; if no sample succeeds a-*degraded* Judgment (``raw.status='degraded'``) is returned, which the orchestrator treats as a glitch.-"""--from __future__ import annotations--import contextlib-import logging-import shutil-import time-from dataclasses import dataclass, field-from pathlib import Path-from typing import Any, Literal--from codeverse3d.config import get_settings-from codeverse3d.contracts.artifacts import Judgment-from codeverse3d.contracts.chat import ChatRequest, ChatResponse-from codeverse3d.contracts.common import TRACK_INFO, Track, Usage-from codeverse3d.judges.base import SLICE_TRACKS, JudgeInput-from codeverse3d.judges.prompt_builder import (- _key,- _stat,- build_judge_messages,- connectivity_error_pairs,- default_cache_dir,- judge_prompt_hash,-)-from codeverse3d.judges.rubrics import (- JudgeOutput,- JudgeParseError,- Rubric,- aggregate_samples,- degraded_judgment,- load_rubric,- parse_judge_output,- wire_schema,-)-from c3d_flywheel.models import get_chat_model-from c3d_flywheel.models.base import ChatModel, ModelError-from codeverse3d.proc import fan_out, unique_tmp-from codeverse3d.reference import conflict_note, dimension_conflict-from codeverse3d.spatial.silhouette import (- CompareFn,- ReferenceMatch,- compare_silhouette,- measure_reference,-)--log = logging.getLogger(__name__)--#: retry budget for ONE judge sample, every attempt included-SAMPLE_BUDGET_S = 900.0-SAMPLE_MIN_WAIT_S = 20.0--SLICE_LABELS = {- "front_back": "cross-section front-back · interior view",- "left_right": "cross-section left-right · interior view",-}--THINKING_LEVELS: tuple[str, ...] = ("off", "low", "medium", "high")--#: per-track montage ceilings under the ``Settings.judge.montages`` dial-TRACK_MONTAGE_CAP: dict[str, int] = {"scene": 4, "graphics": 3}---@dataclass-class JudgeContext:-- measured_scores: dict[str, float] = field(default_factory=dict)- extra_text: str = ""- extra_images: list[tuple[str, str]] = field(default_factory=list)- #: what building the context cost; folded into the verdict so budgets see it- usage: Usage = field(default_factory=Usage)---class VlmJudge:-- name = "vlm"-- def __init__(- self,- rubric: str | Rubric = "static_object",- model_id: str | None = None,- n_samples: int = 1,- temperature: float = 0.2,- *,- thinking: Literal["off", "low", "medium", "high"] | None = None,- max_attempts: int = 3,- chat_model: ChatModel | None = None,- cache_dir: Path | None = None,- label: str = "judge",- sample_budget_s: float | None = None,- fixed_order: bool = False,- ):- self.rubric: Rubric = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)- # fixed_order sends the identical prompt n times (re-judge sigma) instead of shuffling view order- self.fixed_order = bool(fixed_order)- self.model_id = model_id or get_settings().default_judge- self.n_samples = max(1, int(n_samples))- if self.n_samples % 2 == 0:- log.warning("judge n_samples=%d is even: an exact vote tie on a defect reads as absent and a tie on an "- "acceptance item follows the representative sample (rubrics.aggregate_samples); an odd n "- "gives a true majority", self.n_samples)- self.temperature = temperature- jd = get_settings().judge- # ``thinking=None`` reads ``Settings.judge.thinking``- level = str(jd.thinking if thinking is None else thinking)- if level not in THINKING_LEVELS:- raise ValueError(f"judge thinking must be one of {' | '.join(THINKING_LEVELS)}, not {level!r}")- self.thinking: str = level- self.max_attempts = max(1, int(max_attempts))- self.sample_budget_s = SAMPLE_BUDGET_S if sample_budget_s is None else float(sample_budget_s)- self.max_montages = max(1, int(jd.montages))- self.detail_crops = max(0, int(jd.detail_crops))- self.max_px = int(jd.max_px)- self._model = chat_model- self.cache_dir = cache_dir- self.label = label-- def _montages_for(self, inp: JudgeInput) -> int:- """Per-track montage cap (``TRACK_MONTAGE_CAP``); the object rig takes the dial."""- cap = TRACK_MONTAGE_CAP.get(inp.spec.track.value)- if cap is None:- return self.max_montages- return min(self.max_montages, cap)-- @property- def prompt_hash(self) -> str:- return judge_prompt_hash(self.rubric)-- @property- def model(self) -> ChatModel:- if self._model is None:- self._model = get_chat_model(self.model_id)- return self._model-- def judge(self, inp: JudgeInput) -> Judgment:- """Judge one round; clay/normals views in ``inp.renders`` are routed to the geometry montage."""- geometry_views = inp.geometry_views- acceptance_ids = [a.id for a in inp.acceptance]- schema = wire_schema(self.rubric, acceptance_ids)- ctx = self.context(inp)- measured, extra_text, extra_images = ctx.measured_scores, ctx.extra_text, ctx.extra_images- usage = ctx.usage- slice_images, elicit = self.slice_payload(inp)- samples: list[JudgeOutput] = []- errors: list[str] = []- reqs: list[ChatRequest] = []- for k in range(self.n_samples):- seed = None if (self.fixed_order or (self.n_samples == 1 and k == 0)) else (inp.round_index * 1000 + k)- system, messages = build_judge_messages(- inp, self.rubric, shuffle_seed=seed, geometry_views=geometry_views, max_montages=self._montages_for(inp),- detail_crops=self.detail_crops, max_px=self.max_px, cache_dir=self.cache_dir,- extra_images=extra_images, extra_text=extra_text,- slice_images=slice_images, provenance_elicitation=elicit,- )- reqs.append(ChatRequest(- messages=messages, system=system, response_schema=schema, temperature=self.temperature,- thinking=self.thinking, # type: ignore[arg-type]- # fan_out copies contextvars into every sample thread; the label adds what only this call knows- label=f"{self.label}:{self.rubric.name}:r{inp.round_index:02d}:s{k}",- ))- if len(reqs) == 1:- outcomes = [self._sample(reqs[0], acceptance_ids, measured)]- else:- self.model # noqa: B018- results = fan_out(- reqs, lambda r: self._sample(r, acceptance_ids, measured),- max_workers=len(reqs), label=f"{self.label}:samples", item_name=lambda r: r.label,- )- outcomes = [- r if not isinstance(r, Exception) else (None, Usage(), f"{type(r).__name__}: {r}")- for r in results- ]- for out, u, err in outcomes:- usage = usage + u- if out is not None:- samples.append(out)- else:- errors.append(err)- if not samples:- return degraded_judgment(- self.rubric, " || ".join(errors)[:2000], usage=usage, judge_backend=self.model_id, n_requested=self.n_samples,- judge_prompt_hash=self.prompt_hash,- )- return aggregate_samples(- self.rubric, samples, gates=inp.gates, acceptance_items=inp.acceptance,- console_errors=inp.renders.console_errors, views=list(inp.renders.views), usage=usage,- judge_backend=self.model_id, n_requested=self.n_samples, sample_errors=errors,- judge_prompt_hash=self.prompt_hash,- )-- def context(self, inp: JudgeInput) -> JudgeContext:- """Per-call extras: measured-criterion scores, extra text and extra images (none in the base judge)."""- if self.rubric.measured_criteria():- raise ValueError(- f"rubric {self.rubric.name} has measured criteria "- f"{[c.id for c in self.rubric.measured_criteria()]} but VlmJudge computes none; use a specialised judge"- )- return JudgeContext()-- def slice_payload(self, inp: JudgeInput) -> tuple[list[tuple[str, str]], bool]:- """``([(label, png_path)…], provenance_elicitation)`` for the slice channel.-- Fires only when ``Settings.judge.slices == "on-error"``, the track is an object track, ``inp.glb_path``- exists and the connectivity gate has an ERROR; otherwise ``([], False)``. Never raises.- """- if get_settings().judge.slices != "on-error":- return [], False- if inp.spec.track.value not in SLICE_TRACKS or not inp.glb_path:- return [], False- glb = Path(inp.glb_path)- if not glb.is_file():- return [], False- if not any(g.gate == "connectivity" and g.errors for g in inp.gates):- return [], False- pairs = connectivity_error_pairs(inp.gates)- try:- out_dir, manifest = self._render_slices(glb, pairs)- except ImportError as e:- log.warning("judge slices disabled: %s (install the 'mesh' extra)", e)- return [], False- except Exception as e: # noqa: BLE001- log.warning("judge slice render failed for %s: %s", glb, e)- return [], True- return [(SLICE_LABELS[s.name], str(out_dir / s.png)) for s in manifest.rendered()], True-- def _render_slices(self, glb: Path, pairs: list[tuple[str, str]]):- """Render (or reuse) the slice set for ``glb``, cached by path + mtime + size + error pairs."""- from codeverse3d.spatial.sections import SliceManifest, judge_slices-- cache = Path(self.cache_dir) if self.cache_dir else default_cache_dir()- out_dir = cache / f"slices_{_key([_stat(str(glb)), str(pairs)])}"- mpath = out_dir / "manifest.json"- if not mpath.is_file():- tmp = unique_tmp(out_dir)- shutil.rmtree(tmp, ignore_errors=True)- try:- judge_slices(glb, pairs, tmp)- with contextlib.suppress(OSError):- tmp.rename(out_dir)- finally:- shutil.rmtree(tmp, ignore_errors=True)- return out_dir, SliceManifest.model_validate_json(mpath.read_text())-- def _sample(- self, req: ChatRequest, acceptance_ids: list[str], measured: dict[str, float]- ) -> tuple[JudgeOutput | None, Usage, str]:- usage = Usage()- last = ""- t_start = time.monotonic()- for attempt in range(1, self.max_attempts + 1):- remaining = self.sample_budget_s - (time.monotonic() - t_start)- if remaining <= 0:- last = (f"sample budget of {self.sample_budget_s:.0f}s spent after {attempt - 1} attempt(s)"- + (f"; last: {last}" if last else ""))- log.warning("judge %s: %s", req.label, last[:300])- break- req = req.model_copy(update={"max_wait_s": max(SAMPLE_MIN_WAIT_S, remaining)})- t0 = time.monotonic()- try:- resp: ChatResponse = self.model.generate(req)- except ModelError as e:- usage = usage + (getattr(e, "usage", None) or Usage())- last = f"ModelError(attempt {attempt}): {e}"- log.warning("judge %s: %s", req.label, last)- if not e.retryable:- break- continue- usage = usage + resp.usage- payload = resp.parsed if resp.parsed is not None else resp.text- try:- out = parse_judge_output(payload, self.rubric, acceptance_ids, measured_scores=measured)- log.debug("judge %s ok in %.1fs", req.label, time.monotonic() - t0)- return out, usage, ""- except JudgeParseError as e:- last = f"parse(attempt {attempt}): {e}"- log.warning("judge %s: %s", req.label, last[:300])- req = req.model_copy(update={"temperature": min(1.0, self.temperature + 0.1 * attempt)})- return None, usage, last or "unknown judge failure"----#: IoU → score: ≤ 0.25 → 0, ≥ 0.85 → 1, linear between-IOU_LOW, IOU_HIGH = 0.25, 0.85--#: score the measured criterion gets when the measurement cannot be trusted-NEUTRAL_SCORE = 0.5--def iou_to_score(iou: float) -> float:- return max(0.0, min(1.0, (float(iou) - IOU_LOW) / (IOU_HIGH - IOU_LOW)))---class ReferenceJudge(VlmJudge):-- name = "reference"-- def __init__(- self,- model_id: str | None = None,- n_samples: int = 1,- temperature: float = 0.2,- *,- rubric: str | Rubric = "reference",- silhouette_fn: CompareFn | None = None,- **kwargs: Any,- ):- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)- self.silhouette_fn = silhouette_fn or compare_silhouette-- def context(self, inp: JudgeInput) -> JudgeContext:- refs = [r for r in inp.spec.references if Path(r.path).is_file()]- images: list[tuple[str, str]] = []- for i, r in enumerate(refs[:3], 1):- note = f" — {r.note}" if r.note else ""- images.append((f"REFERENCE {i}/{min(len(refs), 3)} ({r.role}){note}", r.path))- blocks: list[str] = []- conflict = self.dimension_conflict(inp, refs)- note = conflict_note(conflict)- if note:- blocks.append(note)- measured_ids = [c.id for c in self.rubric.measured_criteria()]- measured_scores: dict[str, float] = {}- match = measure_reference(inp.renders.views, inp.spec, compare=self.silhouette_fn, conflict=conflict) if refs else None- if measured_ids:- if match is None:- raise ReferenceJudgeError(f"cannot score measured criteria {measured_ids}: "- + ("no render view compared" if refs else "spec has no readable reference images"))- score, why = self.silhouette_score(match)- measured_scores = {cid: score for cid in measured_ids}- extra = match.model_dump(exclude={"iou", "reference", "conflict", "per_view"}, exclude_none=True)- blocks.append(- f"MEASURED SILHOUETTE (harness): {match.view} render vs reference {Path(match.reference).name}: "- f"IoU {match.iou:.3f} → silhouette_match score {score:.2f}{why}. extra: {extra}")- return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)-- def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:- """Does the target reference's own outline contradict the brief's dimensions? If so the brief wins."""- targets = [r for r in refs if r.role == "target"] or refs- if not targets:- return {"conflict": False}- try:- return dimension_conflict(inp.spec, targets[0].path)- except Exception as e: # noqa: BLE001- log.warning("dimension-conflict check failed: %s", e)- return {"conflict": False, "error": str(e)}-- def silhouette_score(self, match: ReferenceMatch) -> tuple[float, str]:- """IoU → measured score, neutral when the reference contradicts the brief or the mask is unreliable."""- if match.conflict:- return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"- if not match.reliable:- return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"- return iou_to_score(match.iou), ""---LIKENESS_NOTE = (- "REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "- "composition to copy; they are what the thing the brief names ACTUALLY looks like. Score "- "brief fidelity, colour/light and richness against them: dominant colour and where the "- "secondary colours sit, how the structure folds / layers / thins out, where the brightness "- "concentrates and how much of the frame stays dark, what the real texture is at fine scale. A "- "frame that ticks every noun in the brief but would never be mistaken for the photographed "- "thing (a row of evenly spaced bars standing in for curtains, a flat band standing in for a "- "gradient of light, cartoon-saturated colour where the photo is subtle) is a 0.4-0.5 on brief "- "fidelity, not a 1.0. Foreground, framing and landscape in the photos are incidental unless "- "the brief asks for them."-)---class LikenessJudge(VlmJudge):- """``VlmJudge`` + reference photos, for tracks with no silhouette: the photos are attached beside the- frames with :data:`LIKENESS_NOTE` and the rubric is unchanged."""-- name = "likeness"-- def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,- rubric: str | Rubric = "shader", **kwargs: Any):- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)-- def context(self, inp: JudgeInput) -> JudgeContext:- if self.rubric.measured_criteria():- raise ValueError(f"rubric {self.rubric.name} has measured criteria; LikenessJudge measures nothing")- refs = [r for r in inp.spec.references if Path(r.path).is_file()][:3]- images = [(f"REAL-WORLD REFERENCE {i}/{len(refs)}" + (f" — {r.note}" if r.note else ""), r.path)- for i, r in enumerate(refs, 1)]- return JudgeContext(extra_text=LIKENESS_NOTE if refs else "", extra_images=images)---class ReferenceJudgeError(RuntimeError):- """The reference judge cannot compute its measured criteria."""---def judge_for(track: Track, rubric: str | Rubric, *, references: bool) -> type[VlmJudge]:- """The judge class that scores a round, shared by the live run and every replay: :class:`ReferenceJudge`- for measured criteria or object-track references, :class:`LikenessJudge` for graphics/scene references,- else :class:`VlmJudge`."""- rub = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)- if rub.measured_criteria() or (references and not TRACK_INFO[track].likeness_refs):- return ReferenceJudge- return LikenessJudge if references else VlmJudge