Skip to content

Official Codebase of 3DCodeVerse

Python

flywheel: a reader of recorded runs, not a writer — delete the model clients, live accounting, captioner, VLM judge, calibration

gaoypeng09f7d9157f44
diff --git a/flywheel/c3d_flywheel/calibration.py b/flywheel/c3d_flywheel/calibration.py
deleted file mode 100644
index f71a4759..00000000
--- a/flywheel/c3d_flywheel/calibration.py
+++ /dev/null
@@ -1,292 +0,0 @@
-"""Judge calibration smoke: re-judge finished runs and check the judge separates good from bad."""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import statistics
-import time
-from concurrent.futures import ThreadPoolExecutor
-from pathlib import Path
-from typing import Any
-
-from pydantic import BaseModel, Field
-
-from c3d_flywheel import select
-from c3d_flywheel.cli._judge import build_judge_input, make_judge, rubric_for
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.artifacts import Judgment, RenderSet
-from codeverse3d.contracts.run import RoundRecord, RunRecord
-from codeverse3d.contracts.spec import Spec
-from codeverse3d.judges.base import JudgeInput
-from c3d_flywheel.vlm_judge import VlmJudge
-from c3d_flywheel.record.record import RecordError, load_record
-from codeverse3d.workspace import Workspace
-
-log = logging.getLogger(__name__)
-
-
-
-def _run_label(run_dir: Path) -> str:
- parts = run_dir.resolve().parts
- tail = [q for q in parts[-5:] if q not in ("runs", "cells")]
- return "_".join(tail) if tail[-1] == "run" else tail[-1]
-
-
-# --------------------------------------------------------------------------- data
-class RoundCase(BaseModel):
-
- run: str
- round_index: int
- kind: str
- rubric: str
- inp: JudgeInput
- gate_errors: int
- gate_warnings: int
- stored: Judgment | None = None
- is_picked: bool = False
- glb: str | None = None
-
-
-def _stored_weighted(case: RoundCase) -> float | None:
- from codeverse3d.judges.rubrics import load_rubric
-
- if not case.stored or not case.stored.scores:
- return None
- try:
- return round(load_rubric(case.rubric).weighted_overall(case.stored.scores), 3)
- except Exception: # noqa: BLE001
- return None
-
-
-class CalibrationRow(BaseModel):
- run: str
- round: int
- kind: str
- gate_errors: int
- gate_warnings: int
- stored_overall: float | None = None
- stored_weighted: float | None = None
- stored_backend: str = ""
- mean: float
- std: float
- uncapped: float
- defect_penalty: float
- per_criterion_mean: dict[str, float] = Field(default_factory=dict)
- per_criterion_std: dict[str, float] = Field(default_factory=dict)
- defects_present: list[str] = Field(default_factory=list)
- caps: list[str] = Field(default_factory=list)
- passed: bool
- n_used: int
- geometry_views: bool = False
- cost_usd: float = 0.0
- duration_s: float = 0.0
- summary: str = ""
- error: str = ""
-
-
-class CalibrationTable(BaseModel):
- model_id: str
- n_samples: int
- fixed_order: bool = False
- rows: list[CalibrationRow]
- pearson_errors_vs_score: float | None = None
- spearman_errors_vs_score: float | None = None
- pearson_stored_vs_new: float | None = None
- mean_std: float = 0.0
- mean_criterion_std: float = 0.0
- total_cost_usd: float = 0.0
-
- def to_markdown(self) -> str:
- head = ("| run | round | kind | gate err/warn | stored | new mean ± std | uncapped | penalty | defects | caps | pass | geo | cost |\n"
- "|---|---|---|---|---|---|---|---|---|---|---|---|---|")
- lines = [head]
- for r in self.rows:
- stored = f"{r.stored_overall:.3f}" if r.stored_overall is not None else "-"
- if (r.stored_overall is not None and r.stored_weighted is not None
- and abs(r.stored_overall - r.stored_weighted) > 0.05):
- stored += f" (w {r.stored_weighted:.3f}!)"
- lines.append(
- f"| {r.run} | r{r.round:02d} | {r.kind} | {r.gate_errors}/{r.gate_warnings} | {stored} | "
- f"{r.mean:.3f} ± {r.std:.3f} | {r.uncapped:.3f} | -{r.defect_penalty:.2f} | "
- f"{', '.join(r.defects_present) or '-'} | {', '.join(r.caps) or '-'} | {'Y' if r.passed else 'n'} | "
- f"{'Y' if r.geometry_views else '-'} | ${r.cost_usd:.3f} |" + (f" ERROR {r.error}" if r.error else "")
- )
- crit = sorted({k for r in self.rows for k in r.per_criterion_mean})
- if crit:
- lines.append("")
- lines.append("| run/round | " + " | ".join(crit) + " |")
- lines.append("|---|" + "---|" * len(crit))
- for r in self.rows:
- cells = [f"{r.per_criterion_mean.get(c, float('nan')):.2f}±{r.per_criterion_std.get(c, 0.0):.2f}" if c in r.per_criterion_mean else "-" for c in crit]
- lines.append(f"| {r.run}/r{r.round:02d} | " + " | ".join(cells) + " |")
- fmt = lambda v: "n/a" if v is None else f"{v:+.3f}" # noqa: E731
- lines.append("")
- lines.append(
- f"model {self.model_id}, n_samples {self.n_samples}{' (fixed montage order: σ is the re-judge noise)' if self.fixed_order else ''}: "
- f"pearson(gate errors, new score) = {fmt(self.pearson_errors_vs_score)}; "
- f"spearman = {fmt(self.spearman_errors_vs_score)}; pearson(stored, new) = {fmt(self.pearson_stored_vs_new)}; "
- f"mean overall std {self.mean_std:.3f}; mean per-criterion std {self.mean_criterion_std:.3f}; cost ${self.total_cost_usd:.3f}"
- )
- return "\n".join(lines)
-
-
-# --------------------------------------------------------------------------- loading runs
-def load_run_cases(run_dir: Path, *, rounds: list[int] | None = None) -> list[RoundCase]:
- run_dir = Path(run_dir)
- ws = Workspace(run_dir)
- saved = [RoundRecord.model_validate_json(p.read_text()) for p in sorted((run_dir / "rounds").glob("r*.json"))]
- try:
- run = load_record(ws)
- picked = select.summarise(run_dir, record=run).round
- except RecordError:
- run = RunRecord(spec=Spec.model_validate_json(ws.spec_path.read_text()), workspace=str(run_dir))
- picked = select.fallback_round(saved)
- cases: list[RoundCase] = []
- for rec in saved:
- if rounds is not None and rec.index not in rounds:
- continue
- if rec.renders is None or not rec.renders.views:
- continue
- cases.append(RoundCase(
- run=_run_label(run_dir), round_index=rec.index, kind=rec.kind, rubric=rubric_for(run, rec, None),
- inp=build_judge_input(ws, run, rec),
- gate_errors=sum(len(g.errors) for g in rec.gates),
- gate_warnings=sum(1 for g in rec.gates for f in g.findings if f.severity.value == "warn"),
- stored=rec.judgment, is_picked=picked == rec.index,
- glb=str(glb) if (glb := select.round_file(ws, rec)) is not None else None,
- ))
- return cases
-
-
-def render_geometry_views(case: RoundCase, out_dir: Path, mode: str) -> RenderSet | None:
- if case.inp.spec.track.value == "scene":
- return None
- if not case.glb:
- log.warning("no geometry views for %s r%02d: the round kept no GLB of its own", case.run, case.round_index)
- return None
- from codeverse3d.conventions import OBJECT_CLAY_VIEWS
- from codeverse3d.spatial.render import render_glb
-
- views = list(OBJECT_CLAY_VIEWS)
- try:
- return render_glb(case.glb, out_dir / case.run / f"geometry_{mode}", views=views, mode=mode, sheet=False)
- except Exception as e: # noqa: BLE001
- log.warning("geometry render failed for %s: %s", case.run, e)
- return None
-
-
-# --------------------------------------------------------------------------- statistics
-def pearson(xs: list[float], ys: list[float], method: str = "linear") -> float | None:
- if len(xs) < 2 or len(set(xs)) < 2 or len(set(ys)) < 2:
- return None
- return round(statistics.correlation(xs, ys, method=method), 4)
-
-
-def spearman(xs: list[float], ys: list[float]) -> float | None:
- return pearson(xs, ys, method="ranked")
-
-
-# --------------------------------------------------------------------------- main entry
-def _judge_case(case: RoundCase, judge: VlmJudge, geometry: RenderSet | None, out: Path) -> CalibrationRow:
- t0 = time.time()
- inp = case.inp.model_copy(update={"geometry_views": geometry}) if geometry is not None else case.inp
- j = judge.judge(inp)
- raw = json.loads(j.raw) if j.raw else {}
- jdir = out / "judgments"
- jdir.mkdir(parents=True, exist_ok=True)
- (jdir / f"{case.run}_r{case.round_index:02d}_{judge.model_id.replace(':', '_')}.json").write_text(j.model_dump_json(indent=1))
- return CalibrationRow(
- run=case.run, round=case.round_index, kind=case.kind, gate_errors=case.gate_errors, gate_warnings=case.gate_warnings,
- stored_overall=case.stored.overall if case.stored else None,
- stored_weighted=_stored_weighted(case),
- stored_backend=case.stored.judge_backend if case.stored else "",
- mean=j.overall, std=j.score_std, uncapped=float(raw.get("overall_uncapped", j.overall)),
- defect_penalty=float(raw.get("defect_penalty", 0.0)),
- per_criterion_mean=dict(j.scores), per_criterion_std=dict(raw.get("per_criterion_std", {})),
- defects_present=[d for d, on in raw.get("defects", {}).items() if on],
- caps=[c["rule"] for c in raw.get("caps", {}).get("caps_applied", [])],
- passed=j.passed, n_used=j.n_samples, geometry_views=inp.geometry_views is not None,
- cost_usd=j.usage.cost_usd, duration_s=round(time.time() - t0, 1), summary=j.summary,
- error=j.summary if j.degraded else "",
- )
-
-
-def calibrate(
- run_dirs: list[Path | str],
- *,
- model_id: str | None = None,
- n_samples: int = 3,
- out_dir: Path | str | None = None,
- geometry_mode: str | None = "clay",
- rounds: list[int] | None = None,
- thinking: str | None = None,
- max_workers: int = 4,
- chat_model: Any = None,
- fixed_order: bool = False,
-) -> CalibrationTable:
- model_id = model_id or get_settings().default_judge
- thinking = thinking or get_settings().judge.thinking
- out = Path(out_dir) if out_dir else Path.cwd()
- out.mkdir(parents=True, exist_ok=True)
- cases: list[RoundCase] = []
- for rd in run_dirs:
- cases.extend(load_run_cases(Path(rd), rounds=rounds))
- if not cases:
- raise ValueError(f"no judgeable rounds found under {list(map(str, run_dirs))}")
- geometry: dict[int, RenderSet | None] = {}
- for i, c in enumerate(cases):
- want = geometry_mode and c.is_picked and c.inp.geometry_views is None
- geometry[i] = render_geometry_views(c, out, geometry_mode) if want else None
-
- def kind(c: RoundCase) -> tuple[str, str, bool]:
- return c.rubric, c.inp.spec.track.value, bool(c.inp.spec.references)
-
- judges = {kind(c): make_judge(c.inp.spec, c.rubric, model_id, n_samples, thinking=thinking,
- cache_dir=out / "cache", label="calibrate", chat_model=chat_model,
- fixed_order=fixed_order) for c in cases}
- with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:
- rows = list(pool.map(lambda ic: _judge_case(ic[1], judges[kind(ic[1])], geometry[ic[0]], out), enumerate(cases)))
- ok = [r for r in rows if not r.error]
- table = CalibrationTable(
- model_id=model_id, n_samples=n_samples, fixed_order=fixed_order, rows=rows,
- pearson_errors_vs_score=pearson([float(r.gate_errors) for r in ok], [r.mean for r in ok]),
- spearman_errors_vs_score=spearman([float(r.gate_errors) for r in ok], [r.mean for r in ok]),
- pearson_stored_vs_new=pearson([r.stored_overall for r in ok if r.stored_overall is not None],
- [r.mean for r in ok if r.stored_overall is not None]),
- mean_std=round(statistics.fmean([r.std for r in ok]), 4) if ok else 0.0,
- mean_criterion_std=round(statistics.fmean([v for r in ok for v in r.per_criterion_std.values()] or [0.0]), 4),
- total_cost_usd=round(sum(r.cost_usd for r in rows), 4),
- )
- stem = "calibration_" + model_id.replace(":", "_").replace("/", "_")
- (out / f"{stem}.json").write_text(json.dumps(table.model_dump(mode="json"), indent=1, ensure_ascii=False))
- (out / f"{stem}.md").write_text(table.to_markdown() + "\n")
- return table
-
-
-def main(argv: list[str] | None = None) -> int:
- ap = argparse.ArgumentParser(description="Re-judge finished runs to calibrate the VLM judge.")
- ap.add_argument("run_dirs", nargs="+")
- ap.add_argument("--model", default=None, help="chat model id, e.g. gemini:gemini-4-argon")
- ap.add_argument("--n", type=int, default=3, help="samples per round")
- ap.add_argument("--out", default=".", help="output directory (json + md)")
- ap.add_argument("--geometry", default="clay", help="clay | normals | none (geometry montage for the picked round)")
- ap.add_argument("--rounds", default="", help="comma-separated round indices (default all)")
- ap.add_argument("--thinking", default=None, choices=("off", "low", "medium", "high"),
- help="judge thinking level (default: Settings.judge.thinking / C3D_JUDGE__THINKING)")
- ap.add_argument("--workers", type=int, default=4)
- ap.add_argument("--fixed-order", action="store_true",
- help="same montage order for every sample: measures the model's re-judge σ, not order robustness")
- a = ap.parse_args(argv)
- logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s: %(message)s")
- rounds = [int(x) for x in a.rounds.split(",") if x.strip()] or None
- table = calibrate(a.run_dirs, model_id=a.model, n_samples=a.n, out_dir=a.out,
- geometry_mode=None if a.geometry == "none" else a.geometry, rounds=rounds,
- thinking=a.thinking, max_workers=a.workers, fixed_order=a.fixed_order)
- print(table.to_markdown())
- return 0
-
-
-if __name__ == "__main__": # pragma: no cover
- raise SystemExit(main())
diff --git a/flywheel/c3d_flywheel/cli/_judge.py b/flywheel/c3d_flywheel/cli/_judge.py
deleted file mode 100644
index 01750361..00000000
--- a/flywheel/c3d_flywheel/cli/_judge.py
+++ /dev/null
@@ -1,97 +0,0 @@
-"""``3dcodeverse judge`` implementation: re-judge a stored round with the inputs the in-run judge
-saw, built by ``judges.base.round_input`` from the run directory."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from pydantic import ValidationError
-
-from codeverse3d.cli import _common as C
-from codeverse3d.contracts.artifacts import BuildResult, RenderSet, RenderView
-from codeverse3d.contracts.plan import Plan
-from codeverse3d.contracts.run import RoundRecord, RunRecord
-from codeverse3d.contracts.spec import Spec
-from codeverse3d.judges.base import judged_subset, resolve_paths, round_input
-from codeverse3d.proc import read_json_or_none
-from c3d_flywheel.record.record import previous_verdict
-from codeverse3d.workspace import Workspace
-
-
-def load_round(ws: Workspace, rec: RunRecord, index: int) -> RoundRecord | None:
- p = ws.root / "rounds" / f"r{index:02d}.json"
- if p.is_file():
- try:
- return RoundRecord.model_validate_json(p.read_text())
- except Exception: # noqa: BLE001 — fall through to the record copy
- pass
- return next((r for r in rec.rounds if r.index == index), None)
-
-
-def rubric_for(rec: RunRecord, rnd: RoundRecord, override: str | None) -> str:
- from c3d_flywheel.record.record import rubric_of
-
- if override:
- return override
- if rnd.judgment is not None and rnd.judgment.rubric:
- return rnd.judgment.rubric
- return rubric_of(rec)
-
-
-def stored_plan(ws: Workspace, rec: RunRecord) -> Plan | None:
- """The run's typed plan: record.json's, else ``plan.json``."""
- if rec.plan is not None:
- return rec.plan
- from codeverse3d.tracks import get_track
-
- data = read_json_or_none(ws.plan_path)
- try:
- return get_track(rec.spec.track).plan_model.model_validate(data) if data else None
- except ValidationError:
- return None
-
-
-def extra_context_for(ws: Workspace, rec: RunRecord, plan: Plan | None, rnd: RoundRecord) -> str:
- from codeverse3d.tracks import get_track
-
- build = rnd.build if rnd.build is not None else BuildResult(ok=False, language=rec.spec.language.value)
- try:
- return get_track(rec.spec.track).make_pipeline().judge_context(ws, plan, rnd.index, build, list(rnd.gates))
- except Exception as e: # noqa: BLE001 — extra context is optional judge input
- return f"(judge context unavailable: {type(e).__name__}: {e})"
-
-
-def clay_geometry_views(ws: Workspace, index: int) -> RenderSet | None:
- d = ws.renders_dir(index) / "clay"
- if not d.is_dir():
- return None
- views = [RenderView(name=p.stem.removeprefix("view_"), path=str(p), mode="clay")
- for p in sorted(d.glob("view_*.png"))]
- return RenderSet(views=views, renderer="stored") if views else None
-
-
-def build_judge_input(ws: Workspace, rec: RunRecord, rnd: RoundRecord) -> Any:
- plan = stored_plan(ws, rec)
- return round_input(rec.spec, plan, rnd, renders=judged_subset(resolve_paths(ws, rnd.renders)), gates=rnd.gates,
- previous=previous_verdict([load_round(ws, rec, rnd.index - 1)], rnd.index), extra_context=extra_context_for(ws, rec, plan, rnd),
- geometry_views=clay_geometry_views(ws, rnd.index), glb_path=stored_glb_path(ws, rnd))
-
-
-def stored_glb_path(ws: Workspace, rnd: RoundRecord) -> str | None:
- """The round's own GLB (never the canonical one, which is the last build's)."""
- from c3d_flywheel.select import round_file
-
- glb = round_file(ws, rnd)
- return str(glb) if glb is not None else None
-
-
-def make_judge(spec: Spec, rubric_name: str, model_id: str, n: int, **options: Any) -> Any:
- from codeverse3d.judges import vlm_judge
- from codeverse3d.judges.rubrics import load_rubric
-
- try:
- rubric = load_rubric(rubric_name)
- except Exception as e: # unknown rubric name / bad yaml
- raise C.CliError(f"cannot load rubric {rubric_name!r}: {e}") from e
- cls = vlm_judge.judge_for(spec.track, rubric, references=bool(spec.references))
- return cls(model_id=model_id, n_samples=n, rubric=rubric_name, **options)
diff --git a/flywheel/c3d_flywheel/cli/inspect_cmd.py b/flywheel/c3d_flywheel/cli/inspect_cmd.py
deleted file mode 100644
index 63c098e0..00000000
--- a/flywheel/c3d_flywheel/cli/inspect_cmd.py
+++ /dev/null
@@ -1,88 +0,0 @@
-"""``3dcodeverse judge`` — re-judge a round of one existing run into ``artifacts/judge/rNN_cli.json``."""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Annotated
-
-import typer
-
-from codeverse3d.cli import _common as C
-from codeverse3d.cli._common import RunsDirOpt, console, kv_table
-from codeverse3d.config import get_settings
-
-
-def judge(
- slug: str,
- round_index: Annotated[int | None, typer.Option("--round")] = None,
- rubric: Annotated[
- str | None, typer.Option("--rubric", help="default: the rubric the round was judged with")
- ] = None,
- model: Annotated[str | None, typer.Option("--model", help="judge chat model id")] = None,
- n: Annotated[int | None, typer.Option("--n", min=1, help="samples (default: Settings.judge.samples)")] = None,
- thinking: Annotated[
- str | None, typer.Option("--thinking", help="off | low | medium | high (default: Settings.judge.thinking, "
- "C3D_JUDGE__THINKING; gemini-4-argon thinks only from medium)"),
- ] = None,
- runs_dir: RunsDirOpt = None,
-) -> None:
- """Re-judge a round with the same inputs as the in-run judge; writes artifacts/judge/rNN_cli.json."""
- from c3d_flywheel import select
- from codeverse3d.cli import _judge as J
- from c3d_flywheel.record.record import RecordError, join_post_run, load_record
-
- ws = C.open_workspace(slug, runs_dir)
- try:
- rec = load_record(ws)
- except RecordError as e:
- raise C.CliError(str(e), code=2) from e
- idx = round_index if round_index is not None else select.summarise(ws.root, record=rec).round
- if idx is None:
- raise C.CliError(f"no round of {ws.root.name} built with renders: nothing to judge (--round N names one)")
- rnd = J.load_round(ws, rec, idx)
- if rnd is None or rnd.renders is None or not rnd.renders.views:
- raise C.CliError(f"round {idx} has no renders (rounds/r{idx:02d}.json / record.json)")
- rubric_name = J.rubric_for(rec, rnd, rubric)
- inp = J.build_judge_input(ws, rec, rnd)
- missing = [v.path for v in inp.renders.views if not Path(v.path).is_file()]
- if missing:
- raise C.CliError(f"round {idx}: {len(missing)} of its {len(inp.renders.views)} judged renders are not on disk "
- f"(e.g. {missing[0]}): nothing to re-judge it from", code=2)
- try:
- judge_obj = J.make_judge(rec.spec, rubric_name, model or rec.spec.backends.judge,
- n or get_settings().judge.samples,
- **({"thinking": thinking} if thinking else {}))
- except ValueError as e:
- raise C.CliError(str(e), code=2) from e
- # writes into the run: one writer per run dir
- with C.mutating(ws, what=f"3dcodeverse judge {ws.root.name}", action="judge"):
- try:
- with join_post_run(ws):
- verdict = judge_obj.judge(inp)
- except ValueError as e:
- raise C.CliError(f"judge failed: {e}") from e
- except Exception as e:
- from c3d_flywheel.vlm_judge import ReferenceJudgeError
-
- if isinstance(e, ReferenceJudgeError):
- raise C.CliError(f"judge failed: {e}") from e
- raise
- ws.write_json(ws.judge_path(idx, "_cli"), verdict)
- console.print(
- kv_table(
- "judgment",
- {
- "rubric": verdict.rubric,
- "round": idx,
- "overall": f"{verdict.overall:.3f}",
- "passed": verdict.passed,
- "std": verdict.score_std,
- "thinking": getattr(judge_obj, "thinking", "-"),
- "stored score": f"{rnd.judgment.overall:.3f}" if rnd.judgment else "-",
- "summary": verdict.summary,
- "cost": f"${verdict.usage.cost_usd:.4f}",
- },
- )
- )
- for it in verdict.improvement_plan:
- console.print(f" [{it.priority}] {it.target}: {it.instruction}")
diff --git a/flywheel/c3d_flywheel/cost/__init__.py b/flywheel/c3d_flywheel/cost/__init__.py
index eaf26a17..0836d8b1 100644
--- a/flywheel/c3d_flywheel/cost/__init__.py
+++ b/flywheel/c3d_flywheel/cost/__init__.py
@@ -1,20 +1,10 @@
-"""Cost accounting: a typed ledger of every model call, and a price-provenance layer on
-top of ``models/pricing.py``. (The audit of FINISHED runs and
-its report are ``c3d_flywheel.costreport``.)
-
-This package re-exports only the names other packages actually import through it.
-Everything else lives in — and is imported from — its own module: ``cost.instrument``
-(metered_chat_model/MeteredAgent), ``cost.types`` (CallCost/CostBucket/Summary), ``cost.tally`` (the step clock).
-Reading the ledgers of finished runs back is ``c3d_flywheel.costreport``.
-
-Nothing here imports tracks/ or orchestrator/, so any layer may use it.
-"""
+"""Reading a run's cost ledger (``telemetry/cost.jsonl``) back: ``ledger`` (rows, re-pricing, buckets),
+``types`` (CallCost / CostBucket / Summary, the stage → role map), ``pricing`` (the USD table + provenance).
+The audit over many runs and its report are ``c3d_flywheel.costreport``."""
from __future__ import annotations
-from c3d_flywheel.cost.context import call_context
-from c3d_flywheel.cost.instrument import run_ledger
-from c3d_flywheel.cost.ledger import load_ledger, record_call
+from c3d_flywheel.cost.ledger import load_ledger
from c3d_flywheel.cost.types import Role, Stage
-__all__ = ["Role", "Stage", "call_context", "load_ledger", "record_call", "run_ledger"]
+__all__ = ["Role", "Stage", "load_ledger"]
diff --git a/flywheel/c3d_flywheel/cost/context.py b/flywheel/c3d_flywheel/cost/context.py
deleted file mode 100644
index e84d7164..00000000
--- a/flywheel/c3d_flywheel/cost/context.py
+++ /dev/null
@@ -1,169 +0,0 @@
-"""Where a call sits in a run: the attribution the ledger needs.
-
-Precedence: explicit arguments > a label naming a job of its own (:data:`SELF_DESCRIBING`)
-> the ambient context (:func:`call_context`) > the rest of the label. The run binding
-(:func:`bound_run`) is a plain ``ContextVar`` copied into fan-out workers, with no
-process-wide fallback.
-"""
-
-from __future__ import annotations
-
-from collections.abc import Iterator
-from contextlib import contextmanager
-from contextvars import ContextVar
-from dataclasses import dataclass, replace
-
-from c3d_flywheel.cost.types import Role, Stage, role_for_stage, stage_for_label
-
-
-@dataclass(frozen=True)
-class CallContext:
- """Attribution for one call (all fields optional; ``None`` = "not stated")."""
-
- run: str = ""
- round: int | None = None
- stage: Stage | None = None
- role: Role | None = None
- label: str = ""
-
- def merge(self, other: CallContext) -> CallContext:
- """``other`` wins field by field where it says something."""
- return CallContext(
- run=other.run or self.run,
- round=self.round if other.round is None else other.round,
- stage=other.stage or self.stage,
- role=other.role or self.role,
- label=other.label or self.label,
- )
-
- def resolved(self) -> CallContext:
- """Fill role from stage (and stage from label) so the row is never ``other`` needlessly."""
- stage = self.stage or (stage_for_label(self.label) if self.label else None)
- role = self.role or (role_for_stage(stage) if stage else None)
- return replace(self, stage=stage, role=role)
-
-
-_EMPTY = CallContext()
-_ctx: ContextVar[CallContext] = ContextVar("c3d_cost_ctx", default=_EMPTY)
-
-#: the run this execution context belongs to ("" = nothing bound here)
-_run_var: ContextVar[str] = ContextVar("c3d_cost_run", default="")
-
-
-@contextmanager
-def bound_run(run: str) -> Iterator[None]:
- """Name the run every row written inside the block belongs to (concurrent blocks unwind independently)."""
- token = _run_var.set(run or "")
- try:
- yield
- finally:
- _run_var.reset(token)
-
-
-def run_binding() -> CallContext:
- """The run bound in this execution context (``run=""`` when none is)."""
- return CallContext(run=_run_var.get())
-
-
-def current() -> CallContext:
- """Run binding merged with the ambient (thread-local) context."""
- return run_binding().merge(_ctx.get())
-
-
-@contextmanager
-def call_context(
- *,
- run: str = "",
- round: int | None = None, # noqa: A002 - matches the record/round vocabulary
- stage: Stage | str | None = None,
- role: Role | str | None = None,
- label: str = "",
-) -> Iterator[CallContext]:
- """Set the ambient attribution for the block; nesting merges."""
- ctx = _ctx.get().merge(CallContext(run=run, round=round, stage=_stage(stage), role=_role(role), label=label))
- token = _ctx.set(ctx)
- try:
- yield ctx
- finally:
- _ctx.reset(token)
-
-
-def _stage(value: Stage | str | None) -> Stage | None:
- if value is None or isinstance(value, Stage):
- return value
- try:
- return Stage(value)
- except ValueError:
- return stage_for_label(str(value))
-
-
-def _role(value: Role | str | None) -> Role | None:
- if value is None or isinstance(value, Role):
- return value
- try:
- return Role(value)
- except ValueError:
- return None
-
-
-# --------------------------------------------------------------------------- labels
-#: request-label prefix → (stage, role); longest prefix wins, the rest goes through ``stage_for_label``.
-_LABEL_HINTS: tuple[tuple[str, Stage, Role], ...] = (
- ("planner", Stage.PLAN, Role.PLANNER),
- ("plan", Stage.PLAN, Role.PLANNER),
- ("judge", Stage.JUDGE, Role.JUDGE),
- ("caption", Stage.CAPTION, Role.CAPTIONER),
- ("calibrate", Stage.JUDGE, Role.JUDGE),
-)
-
-
-def context_from_label(label: str) -> CallContext:
- """Best-effort attribution from a ``ChatRequest.label`` (head names the job, ``r<NN>`` the round)."""
- low = (label or "").strip()
- if not low:
- return _EMPTY
- round_index = _round_from_label(low)
- head = low.split(":", 1)[0].lower()
- best: tuple[int, Stage, Role] | None = None
- for prefix, stage, role in _LABEL_HINTS:
- if head.startswith(prefix) and (best is None or len(prefix) > best[0]):
- best = (len(prefix), stage, role)
- if best is not None:
- return CallContext(round=round_index, stage=best[1], role=best[2], label=low)
- return CallContext(round=round_index, stage=_stated(stage_for_label(head)), label=low)
-
-
-def _stated(stage: Stage | None) -> Stage | None:
- """``Stage.OTHER`` states nothing, so it must not outrank the ambient context."""
- return None if stage is Stage.OTHER else stage
-
-
-def _round_from_label(label: str) -> int | None:
- """``judge:static_object:r02:s1`` → 2 (``None`` when the label has no round tag)."""
- for token in label.replace("/", ":").split(":"):
- t = token.strip()
- if len(t) >= 2 and t[0] == "r" and t[1:].isdigit():
- return int(t[1:])
- return None
-
-
-#: Stages whose label names a job of its own: they outrank the ambient session context.
-SELF_DESCRIBING: frozenset[Stage] = frozenset(
- {Stage.PLAN, Stage.JUDGE, Stage.CAPTION})
-
-
-def attribute(explicit: CallContext | None = None, *, label: str = "") -> CallContext:
- """The final attribution for one call (precedence in the module docstring)."""
- ctx = current()
- if label:
- from_label = context_from_label(label)
- ctx = ctx.merge(from_label) if from_label.stage in SELF_DESCRIBING else from_label.merge(ctx)
- if explicit is not None:
- ctx = ctx.merge(explicit)
- if label and not (explicit is not None and explicit.label):
- ctx = replace(ctx, label=label)
- return ctx.resolved()
-
-
-__all__ = ["CallContext", "attribute", "bound_run", "call_context", "context_from_label", "current",
- "run_binding"]
diff --git a/flywheel/c3d_flywheel/cost/guard.py b/flywheel/c3d_flywheel/cost/guard.py
deleted file mode 100644
index 5bab3180..00000000
--- a/flywheel/c3d_flywheel/cost/guard.py
+++ /dev/null
@@ -1,14 +0,0 @@
-"""Prompt-size estimate (~4 chars/token, +-15%) for the skills' token accounting."""
-
-from __future__ import annotations
-
-from collections.abc import Iterable
-
-CHARS_PER_TOKEN = 4.0
-
-
-def text_tokens(text: str | Iterable[str]) -> int:
- """Estimated prompt tokens for one string or a bag of blocks."""
- if isinstance(text, str):
- return int(len(text) / CHARS_PER_TOKEN + 0.5)
- return sum(text_tokens(t) for t in text)
diff --git a/flywheel/c3d_flywheel/cost/instrument.py b/flywheel/c3d_flywheel/cost/instrument.py
deleted file mode 100644
index 06e9bc84..00000000
--- a/flywheel/c3d_flywheel/cost/instrument.py
+++ /dev/null
@@ -1,176 +0,0 @@
-"""Meter every model call and agent session into the ledger.
-
-:class:`MeteredChatModel` appends one ``CallCost`` row per ``generate``; :class:`MeteredAgent`
-sets the attribution for a session and records its one session row. Accounting never fails
-a run: ledger errors are logged and swallowed, and other attributes forward to the real object.
-"""
-
-from __future__ import annotations
-
-import logging
-import time
-from collections.abc import Iterator
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Any
-
-from codeverse3d.contracts.agent import AgentJob, AgentResult
-from codeverse3d.contracts.chat import ChatRequest, ChatResponse
-from codeverse3d.contracts.common import Usage
-from c3d_flywheel.cost.context import (
- bound_run,
- call_context,
-)
-from c3d_flywheel.cost.ledger import (
- CostLedger,
- bound_ledger,
- existing_ledger_path,
- open_run_ledger,
- record_call,
-)
-from c3d_flywheel.cost.tally import book_time
-from c3d_flywheel.cost.types import Role, Stage, stage_for_label
-
-log = logging.getLogger(__name__)
-
-
-class _Proxy:
-
- def __init__(self, inner: Any):
- self._inner = inner
-
- @property
- def model(self) -> str:
- return getattr(self._inner, "model", "")
-
- @property
- def id(self) -> str:
- return str(self._inner.id)
-
- def __getattr__(self, name: str) -> Any: # everything else is the real object's
- return getattr(self._inner, name)
-
- def __repr__(self) -> str: # pragma: no cover - debugging aid
- return f"{type(self).__name__}({self._inner!r})"
-
-
-class MeteredChatModel(_Proxy):
-
- @property
- def provider(self) -> str:
- return getattr(self._inner, "provider", "")
-
- # -- the meter --------------------------------------------------------
- def generate(self, request: ChatRequest) -> ChatResponse:
- t0 = time.perf_counter()
- try:
- resp = self._inner.generate(request)
- except BaseException as exc: # noqa: BLE001 - record the attempt, then re-raise
- # a failed call may still have been billed (ModelError.usage)
- ms = int((time.perf_counter() - t0) * 1000)
- usage = getattr(exc, "usage", None)
- usage = usage if isinstance(usage, Usage) else Usage(backend=self.provider, model=self.model)
- book_time(ms / 1000, _lost_s(ms, usage.latency_ms))
- self._record(usage, request, outcome=_outcome(exc), ms=ms, attempts=getattr(exc, "attempts", 0))
- raise
- ms = int((time.perf_counter() - t0) * 1000)
- book_time(ms / 1000, _lost_s(ms, resp.usage.latency_ms))
- self._record(resp.usage, request, outcome="ok", ms=ms,
- key=resp.raw.get("key"), attempts=resp.raw.get("attempts", 0))
- return resp
-
- def _record(self, usage: Usage, request: ChatRequest, *, outcome: str, ms: int,
- key: object = None, attempts: object = 0) -> None:
- try:
- record_call(usage, label=request.label, backend=usage.backend or self.provider,
- model=usage.model or self.model, outcome=outcome,
- latency_ms=usage.latency_ms or ms,
- key=_key_suffix(key), attempts=int(attempts or 0))
- except Exception as e: # pragma: no cover - accounting must never break a call
- log.debug("cost: could not record %s: %s", request.label, e)
-
-
-def _lost_s(wall_ms: int, answered_ms: int) -> float:
- """Seconds lost to provider errors: the call's wall time outside the answered round-trips."""
- return max(0, wall_ms - answered_ms) / 1000
-
-
-def _key_suffix(key: object) -> str:
- """The last 4 chars of an API key as ``…ab12`` — the ledger never holds more."""
- s = str(key or "").strip()
- return f"…{s[-4:]}" if s else ""
-
-
-def _outcome(exc: BaseException) -> str:
- name = type(exc).__name__.lower()
- if "timeout" in name:
- return "timeout"
- return "error"
-
-
-class MeteredAgent(_Proxy):
- """CodingAgent proxy: attribution for the session + its one ledger row."""
-
- @property
- def kind(self) -> str:
- return str(getattr(self._inner, "kind", ""))
-
- def run(self, job: AgentJob) -> AgentResult:
- stage = stage_for_label(job.kind or job.label)
- t0 = time.perf_counter()
- with call_context(round=job.round, stage=stage, role=Role.GENERATOR, label=job.label):
- result = self._inner.run(job)
- wall_s = time.perf_counter() - t0
- book_time(wall_s, float(result.provider_wait_s or 0.0))
- self._record_session(job, result, stage, int(wall_s * 1000))
- return result
-
- def _record_session(self, job: AgentJob, result: AgentResult, stage: Stage, ms: int) -> None:
- usage = result.usage
- if not (usage.input_tokens or usage.output_tokens or usage.cost_usd):
- return
- try:
- record_call(usage, round=job.round, stage=stage, role=Role.GENERATOR, label=job.label,
- backend=usage.backend or self.kind, model=usage.model or self.model,
- outcome="ok" if result.ok else (result.exit_reason or "error"),
- latency_ms=usage.latency_ms or ms,
- n_calls=max(1, int(result.tool_calls or 1)),
- source="estimate" if result.usage_estimated else "session",
- **({"effort": result.effort} if result.effort else {}))
- except Exception as e: # pragma: no cover
- log.debug("cost: could not record agent session %s: %s", job.label, e)
-
-
-# --------------------------------------------------------------------------- factories
-def metered_chat_model(model: Any) -> Any:
- """Wrap ``model`` unless it is already metered. Metering cannot be switched off."""
- if model is None or isinstance(model, MeteredChatModel):
- return model
- return MeteredChatModel(model)
-
-
-def metered_agent(agent: Any) -> Any:
- if agent is None or isinstance(agent, MeteredAgent):
- return agent
- return MeteredAgent(agent)
-
-
-# --------------------------------------------------------------------------- run activation
-@contextmanager
-def run_ledger(workspace: str | Path, *, run: str = "", create: bool = True) -> Iterator[CostLedger | None]:
- """Meter one run into ``<workspace>/telemetry/cost.jsonl``.
-
- ``create=False`` (work after a run finished) appends only to an existing ledger, else rows go
- to the per-process log, so a re-judge never masquerades as the run's whole cost.
- Nests and parallelises: bindings unwind by ``ContextVar`` token."""
- ws = Path(workspace)
- if not create and existing_ledger_path(ws) is None:
- with bound_run(run or ws.name):
- yield None
- return
- ledger = open_run_ledger(ws)
- with bound_run(run or ws.name), bound_ledger(ledger.path):
- yield ledger
-
-
-__all__ = ["MeteredAgent", "MeteredChatModel", "metered_agent", "metered_chat_model", "run_ledger"]
diff --git a/flywheel/c3d_flywheel/cost/ledger.py b/flywheel/c3d_flywheel/cost/ledger.py
index 4b5ee851..c5a100bd 100644
--- a/flywheel/c3d_flywheel/cost/ledger.py
+++ b/flywheel/c3d_flywheel/cost/ledger.py
@@ -1,40 +1,26 @@
-"""Append-only cost ledger: one JSONL row per model / agent call.
-
-``record_call`` prices and appends a row (never raises); ``load_ledger`` reads rows back
-tolerantly; ``summarise`` buckets them by dimension.
-"""
+"""The cost ledger of a finished run (``telemetry/cost.jsonl``): read tolerantly, re-price, bucket by dimension."""
from __future__ import annotations
-import json
import logging
-import os
-import threading
-import time
-from collections.abc import Iterable, Iterator, Sequence
-from contextlib import contextmanager
-from contextvars import ContextVar
+from collections.abc import Iterable, Sequence
from pathlib import Path
from typing import Any
from codeverse3d.contracts.common import Usage
-from c3d_flywheel.cost.context import CallContext, _stage, attribute
-from c3d_flywheel.cost.tally import book_usage
+from codeverse3d.proc import iter_jsonl_lines
+
+from c3d_flywheel.cost.pricing import (
+ estimate_cost,
+ price_provenance,
+ unit_prices,
+)
from c3d_flywheel.cost.types import (
CallCost,
CostBucket,
- Role,
- Stage,
Summary,
normalise_ids,
- role_for_stage,
-)
-from c3d_flywheel.models.pricing import (
- estimate_cost,
- price_provenance,
- unit_prices,
)
-from codeverse3d.proc import iter_jsonl_lines
log = logging.getLogger(__name__)
@@ -48,91 +34,12 @@ DIMENSIONS = ("run", "stage", "role", "backend", "model", "provider", "round", "
PROVIDER_PRICED_BACKENDS = frozenset({"claude-code"})
-class CostLedger:
- """Append-only JSONL ledger. Thread-safe; one line per call, flushed on write."""
-
- def __init__(self, path: str | Path):
- self.path = Path(path)
- self._lock = threading.Lock()
-
- def append(self, row: CallCost) -> CallCost:
- line = json.dumps(row.model_dump(mode="json"), ensure_ascii=False, default=str)
- try:
- self.path.parent.mkdir(parents=True, exist_ok=True)
- with self._lock, self.path.open("a", encoding="utf-8") as fh:
- fh.write(line + "\n")
- fh.flush()
- except OSError as e: # pragma: no cover - disk full / read-only run dir
- log.warning("cost ledger append failed (%s): %s", self.path, e)
- return row
-
-
-_fallback_lock = threading.Lock()
-_fallback: CostLedger | None = None
-
-#: sentinel: "this execution context did not state a ledger"
-_UNSET: Any = object()
-#: context-local ONLY override of the default ledger, so parallel runs never share a file
-_default_var: ContextVar[Any] = ContextVar("c3d_cost_default_ledger", default=_UNSET)
-
-
-@contextmanager
-def bound_ledger(path: str | Path) -> Iterator[CostLedger]:
- """Point :func:`record_call` at ``path`` for the block."""
- led = CostLedger(path)
- token = _default_var.set(led)
- try:
- yield led
- finally:
- _default_var.reset(token)
-
-
-def default_ledger() -> CostLedger:
- """The ledger :func:`record_call` writes to: the bound one, else the per-process fallback
- log, so a call outside any run is still accounted for. There is no off switch."""
- global _fallback
- led = _default_var.get()
- if led is not _UNSET:
- return led
- with _fallback_lock:
- if _fallback is None:
- _fallback = CostLedger(process_ledger_path())
- return _fallback
-
-
-def process_ledger_path() -> Path:
- """The fallback log for calls with no run context: one file per process under ``<cache_dir>/cost/``."""
- from codeverse3d.config import get_settings
-
- base = Path(get_settings().cache_dir)
- stamp = time.strftime("%Y%m%d", time.localtime())
- return base / "cost" / f"p{os.getpid()}-{stamp}.jsonl"
-
-
def existing_ledger_path(workspace: str | Path) -> Path | None:
"""The run's ``telemetry/cost.jsonl`` if it exists, else ``None``."""
p = Path(workspace) / TELEMETRY_LEDGER
return p if p.is_file() else None
-def open_run_ledger(workspace: str | Path) -> CostLedger:
- """The run's ledger (the file is created by its first row)."""
- return CostLedger(Path(workspace) / TELEMETRY_LEDGER)
-
-
-#: ``_as_role`` differs from ``context._role``: it returns Role.OTHER (truthy) where that returns None.
-_as_stage = _stage
-
-
-def _as_role(value: Role | str | None) -> Role | None:
- if value is None or isinstance(value, Role):
- return value
- try:
- return Role(value)
- except ValueError:
- return Role.OTHER
-
-
def price_call(
usage: Usage,
*,
@@ -173,70 +80,6 @@ def price_call(
return cost, fields
-def record_call(
- usage: Usage | None = None,
- *,
- run: str = "",
- stage: Stage | str | None = None,
- role: Role | str | None = None,
- round: int | None = None, # noqa: A002 - matches the record/round vocabulary
- label: str = "",
- backend: str = "",
- model: str = "",
- outcome: str = "ok",
- latency_ms: int | None = None,
- n_calls: int = 1,
- source: str = "live",
- ledger: CostLedger | str | Path | None = None,
- ts: float | None = None,
- reprice: bool = False,
- **extra: Any,
-) -> CallCost:
- """Append one priced call to the ledger, book it into the open tallies, return the row.
-
- Unstated ``run`` / ``round`` / ``stage`` / ``role`` resolve from the ambient context and label.
- Never raises."""
- u = usage or Usage()
- ctx = attribute(CallContext(run=run, round=round, stage=_as_stage(stage), role=_as_role(role),
- label=label), label=label)
- st = ctx.stage or Stage.OTHER
- rl = ctx.role or role_for_stage(st)
- run, round, label = ctx.run, ctx.round, ctx.label # noqa: A001 - see the signature
- try:
- cost, price_fields = price_call(u, backend=backend, model=model, reprice=reprice)
- except Exception as e: # pragma: no cover - pricing must never break a run
- log.warning("cost: pricing failed for %s/%s: %s", backend, model, e)
- cost, price_fields = float(u.cost_usd), {}
- row = CallCost(
- ts=ts if ts is not None else time.time(),
- run=run,
- round=round,
- stage=st,
- role=rl,
- label=label,
- input_tokens=int(u.input_tokens),
- cached_tokens=int(u.cached_tokens),
- output_tokens=int(u.output_tokens),
- thoughts_tokens=int(u.thoughts_tokens),
- tool_calls=int(u.tool_calls),
- cost_usd=cost,
- recorded_usd=float(u.cost_usd),
- latency_ms=int(u.latency_ms if latency_ms is None else latency_ms),
- cache_hit=int(u.cached_tokens) > 0,
- outcome=outcome,
- n_calls=max(1, int(n_calls)),
- source=source,
- **price_fields,
- **extra,
- )
- (ledger if isinstance(ledger, CostLedger) else CostLedger(ledger) if ledger else default_ledger()).append(row)
- book_usage(Usage(backend=row.backend, model=row.model, input_tokens=row.input_tokens,
- output_tokens=row.output_tokens, cached_tokens=row.cached_tokens,
- thoughts_tokens=row.thoughts_tokens, tool_calls=row.tool_calls,
- cost_usd=row.cost_usd, latency_ms=row.latency_ms))
- return row
-
-
def load_ledger(path: str | Path) -> list[CallCost]:
"""Read a ledger file, or a run directory holding one; bad lines are skipped."""
p = Path(path)
diff --git a/flywheel/c3d_flywheel/models/pricing.py b/flywheel/c3d_flywheel/cost/pricing.py
similarity index 99%
rename from flywheel/c3d_flywheel/models/pricing.py
rename to flywheel/c3d_flywheel/cost/pricing.py
index cbe60d19..c84466b0 100644
--- a/flywheel/c3d_flywheel/models/pricing.py
+++ b/flywheel/c3d_flywheel/cost/pricing.py
@@ -136,7 +136,7 @@ def lookup_price(provider: str, model: str) -> Price | None:
@dataclass(frozen=True)
class PriceRow:
- """A resolved price plus its audit trail — what ``3dcodeverse cost prices`` prints."""
+ """A resolved price plus its audit trail — what ``c3d-flywheel cost prices`` prints."""
provider: str
model: str
diff --git a/flywheel/c3d_flywheel/cost/tally.py b/flywheel/c3d_flywheel/cost/tally.py
deleted file mode 100644
index e6f6bb6b..00000000
--- a/flywheel/c3d_flywheel/cost/tally.py
+++ /dev/null
@@ -1,83 +0,0 @@
-"""What the meters saw inside one block of work: its money and its lost time.
-
-A :class:`Tally` opened by :func:`tally` nests; the ledger books priced rows and the
-``cost.instrument`` proxies book wall/lost seconds into every open tally. :func:`timed`
-turns a block into a ``StepTime`` row. The binding is a ``ContextVar`` that fan-out copies.
-"""
-
-from __future__ import annotations
-
-import threading
-import time
-from collections.abc import Iterator
-from contextlib import contextmanager
-from contextvars import ContextVar
-
-from codeverse3d.contracts.common import Usage
-from codeverse3d.contracts.run import StepTime
-
-
-class Tally:
-
- def __init__(self) -> None:
- self.usage = Usage()
- self._time: dict[int, list[float]] = {} # thread id → [wall_s, lost_s] of its metered calls
- self._lock = threading.Lock()
-
- def add_usage(self, usage: Usage) -> None:
- with self._lock:
- self.usage = self.usage + usage
-
- def add_time(self, wall_s: float, lost_s: float) -> None:
- with self._lock:
- t = self._time.setdefault(threading.get_ident(), [0.0, 0.0])
- t[0] += wall_s
- t[1] += min(lost_s, wall_s)
-
- def lost_s(self) -> float:
- """Seconds lost to provider errors: losses add within a thread; across parallel threads
- it is ``max(wall) - max(wall - lost)``."""
- with self._lock:
- if not self._time:
- return 0.0
- return max(w for w, _ in self._time.values()) - max(w - lost for w, lost in self._time.values())
-
-
-_open: ContextVar[tuple[Tally, ...]] = ContextVar("c3d_cost_tallies", default=())
-
-
-@contextmanager
-def tally() -> Iterator[Tally]:
- t = Tally()
- token = _open.set(_open.get() + (t,))
- try:
- yield t
- finally:
- _open.reset(token)
-
-
-def book_usage(usage: Usage) -> None:
- for t in _open.get():
- t.add_usage(usage)
-
-
-def book_time(wall_s: float, lost_s: float) -> None:
- for t in _open.get():
- t.add_time(max(0.0, wall_s), max(0.0, lost_s))
-
-
-@contextmanager
-def timed(step: str, into: list[StepTime], *, round_index: int | None = None) -> Iterator[Tally]:
- """Time ``step`` into ``into`` whether the block returns or raises; a ~0 s step leaves no row."""
- t0 = time.monotonic()
- with tally() as t:
- try:
- yield t
- finally:
- wall = time.monotonic() - t0
- if wall >= 0.01:
- into.append(StepTime(step=step, round=round_index, wall_s=round(wall, 2),
- lost_s=round(min(wall, t.lost_s()), 2)))
-
-
-__all__ = ["Tally", "book_time", "book_usage", "tally", "timed"]
diff --git a/flywheel/c3d_flywheel/dataset/captions.py b/flywheel/c3d_flywheel/dataset/captions.py
deleted file mode 100644
index f968e336..00000000
--- a/flywheel/c3d_flywheel/dataset/captions.py
+++ /dev/null
@@ -1,189 +0,0 @@
-"""Caption one finished run: ``{detailed, instruction, factory}``."""
-
-from __future__ import annotations
-
-import json
-import re
-from datetime import UTC, datetime
-from pathlib import Path
-
-from pydantic import BaseModel, Field
-
-from c3d_flywheel.dataset.sample import code_files_for_round, exported_round
-from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart
-from codeverse3d.contracts.common import ENTRY_FILE, Language, Track
-from codeverse3d.contracts.run import RunRecord
-from c3d_flywheel.models import get_chat_model
-from codeverse3d.proc import write_json_atomic
-from codeverse3d.prompts import prompt_hash
-from c3d_flywheel.record import git_history
-from codeverse3d.workspace import Workspace
-
-MAX_CODE_CHARS = 12_000
-N_VIEWS = 2
-MOTION_VIEW_HINTS = ("articulation", "pose_")
-
-LANGUAGE_PHRASE: dict[Language, tuple[str, str]] = {
- Language.BLENDER: ("a Blender Python script", "blender"),
- Language.CADQUERY: ("a CadQuery script", "cadquery"),
- Language.THREEJS: ("a Three.js module", "three.js"),
- Language.URDF_BLENDER: ("a URDF model with Blender-Python-built link meshes", "urdf"),
- Language.SCENE_THREEJS: ("a Three.js scene", "three.js"),
- Language.SCENE_BLENDER: ("a Blender Python scene", "blender"),
- Language.GLSL_SHADER: ("a GLSL fragment shader", "glsl"),
- Language.OPENGL_PYTHON: ("an OpenGL Python program", "opengl"),
-}
-#: leaked API / platform tokens banned from captions; ``THREE.`` is checked case-sensitively because "Three.js" is required.
-_FORBIDDEN = re.compile(
- r"\b(bpy|bmesh|mathutils|cq\.|cadquery\.|GLTFLoader|ShaderMaterial|MeshStandardMaterial|"
- r"Workplane|moderngl|Shadertoy|Sketchfab|Gemini|Claude|OpenAI|GPT|Antigravity)\b",
- re.IGNORECASE,
-)
-_FORBIDDEN_THREE_NS = re.compile(r"\bTHREE\.(?!js\b)")
-
-
-class Captions(BaseModel):
- detailed: str = Field(min_length=20, description="objective, image-grounded description (2-5 sentences)")
- instruction: str = Field(min_length=15, description="one natural user request asking to generate this as code")
- factory: str = Field(min_length=20, description="code-grounded build recipe: parts + construction order")
-
-
-class CaptionProvenance(BaseModel):
- captioner: str
- round_index: int | None
- images_used: list[str]
- code_chars: int
- prompt_hash: str
- created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
- cost_usd: float = 0.0
-
-
-class CaptionError(RuntimeError):
- pass
-
-
-def _system_prompt() -> str:
- return (
- "You label 3D assets for a code-generation dataset. You receive rendered views of an asset, "
- "the user's original request, and the raw source code that built it. Return STRICT JSON with "
- "exactly the keys detailed, instruction, factory (string values).\n"
- "- detailed: an objective description of what is VISIBLE in the renders: the object/scene, its "
- "main parts, proportions, materials/colours, and — if it articulates or animates — how its parts "
- "move. 2-5 sentences, grounded in the images.\n"
- "- instruction: ONE natural user request (1-2 sentences, imperative) asking an AI to generate this "
- "asset as {language_phrase}; mention the key parts/requirements a user would state.\n"
- "- factory: a technical build recipe (3-6 sentences) derived from the CODE: which parts are built, "
- "in what order, from which primitives/operations (extrusions, booleans, lofts, instancing, joints, "
- "shader effects...), how they are positioned and finished.\n"
- "Rules: be specific to THIS asset; describe shapes in plain words; do NOT mention API, module, "
- "class or function names, file names, model/vendor names or platforms; no markdown."
- )
-
-
-def _round_images(ws: Workspace, record: RunRecord) -> tuple[list[ImagePart], list[str]]:
- rnd = exported_round(ws, record)
- images: list[ImagePart] = []
- used: list[str] = []
- if rnd is None or rnd.renders is None:
- return images, used
-
- def _add(path: str | None, label: str) -> None:
- if not path or len(images) >= N_VIEWS + 1:
- return
- p = ws.rebase(path)
- if p.is_file():
- images.append(ImagePart(path=str(p), label=label))
- used.append(p.relative_to(ws.root).as_posix() if p.is_relative_to(ws.root) else str(p))
-
- _add(rnd.renders.contact_sheet, "contact sheet (all views, labelled)")
- views = list(rnd.renders.views)
- motion = [v for v in views if any(h in v.name for h in MOTION_VIEW_HINTS)]
- for v in (motion + [v for v in views if v not in motion])[:N_VIEWS]:
- _add(v.path, f"view: {v.name}")
- return images, used
-
-
-def _code_excerpt(ws: Workspace, record: RunRecord) -> str:
- rnd = exported_round(ws, record)
- files, _src = code_files_for_round(ws, rnd)
- entry = ENTRY_FILE[record.spec.language]
- ordered = {k: files[k] for k in sorted(files, key=lambda k: (k != entry, k))}
- text, _skipped = git_history.decode_text_files(ordered, max_total=MAX_CODE_CHARS)
- return "\n\n".join(f"### {p}\n{t}" for p, t in text.items())
-
-
-def _user_prompt(record: RunRecord, code: str) -> str:
- kind = {Track.STATIC_OBJECT: "static object", Track.ARTICULATED_OBJECT: "articulated object",
- Track.SCENE: "scene", Track.GRAPHICS: "animated procedural graphics (judged from sampled frames)"}[record.spec.track]
- return (
- f"Asset kind: {kind}.\nOriginal user request: {record.spec.prompt}\n\n"
- f"Source code (raw, may be truncated):\n{code}\n\nReturn the JSON now."
- )
-
-
-def validate_captions(caps: Captions, language: Language) -> list[str]:
- problems = []
- for field in ("detailed", "instruction", "factory"):
- m = _FORBIDDEN.search(getattr(caps, field)) or _FORBIDDEN_THREE_NS.search(getattr(caps, field))
- if m:
- problems.append(f"{field}: must not mention '{m.group(0)}' (API/platform name)")
- phrase, word = LANGUAGE_PHRASE[language]
- if word not in caps.instruction.lower():
- problems.append(f"instruction: must name the target language ({phrase})")
- return problems
-
-
-def caption_sample(
- ws: Workspace, record: RunRecord, model_id: str, *, model: object | None = None,
- out_dir: Path | str | None = None, slug: str | None = None
-) -> Captions:
- if model is None:
- model = get_chat_model(model_id)
- images, used = _round_images(ws, record)
- code = _code_excerpt(ws, record)
- if not code and not images:
- raise CaptionError(f"{ws.root}: no code and no renders to caption")
- system = _system_prompt().replace("{language_phrase}", LANGUAGE_PHRASE[record.spec.language][0])
- user = _user_prompt(record, code)
- messages = [ChatMessage.user(user, images=images)]
- schema = Captions.model_json_schema()
- cost = 0.0
- caps: Captions | None = None
- problems: list[str] = []
- for _attempt in range(2):
- resp = model.generate(ChatRequest(messages=messages, system=system, response_schema=schema,
- temperature=0.3, thinking="low", max_wait_s=900.0, label="captioner")) # type: ignore[attr-defined]
- cost += resp.usage.cost_usd
- data = resp.parsed
- try:
- caps = Captions.model_validate(data)
- except Exception as e:
- problems = [f"invalid JSON/schema: {e}"]
- else:
- problems = validate_captions(caps, record.spec.language)
- if not problems:
- break
- messages = messages + [ChatMessage.assistant(resp.text or json.dumps(data)),
- ChatMessage.user("Fix these problems and return the JSON again:\n- " + "\n- ".join(problems))]
- if caps is None or problems:
- raise CaptionError(f"{ws.root}: captions rejected after retry: {problems}")
- rnd = exported_round(ws, record)
- prov = CaptionProvenance(captioner=model_id, round_index=rnd.index if rnd else None,
- images_used=used, code_chars=len(code),
- prompt_hash=prompt_hash(system), cost_usd=cost)
- payload = {**caps.model_dump(), "provenance": prov.model_dump(mode="json")}
- record.extra["captions"] = payload
- if out_dir is not None:
- write_json_atomic(Path(out_dir) / f"{slug or ws.root.name}.json", payload)
- return caps
- from c3d_flywheel.record.deliverable import build_deliverable, load_deliverable
- from c3d_flywheel.record.record import package_run
-
- ws.write_json(ws.root / "captions.json", payload)
- package_run(ws, record)
- ws.write_json(ws.record_path, record)
- handed = load_deliverable(ws)
- if handed is not None and handed.round is not None:
- build_deliverable(ws, record, handed.round)
- return caps
-
diff --git a/flywheel/c3d_flywheel/legacy.py b/flywheel/c3d_flywheel/legacy.py
new file mode 100644
index 00000000..99c0f13c
--- /dev/null
+++ b/flywheel/c3d_flywheel/legacy.py
@@ -0,0 +1,273 @@
+"""The v0.0.x run record (``record.json``) and everything stored inside it, read leniently.
+
+The plugin no longer writes runs: these models exist to READ the recorded ones. Spec, plan and the
+artifact blocks are tolerant of what later versions dropped (``Budget``/``Backends`` were removed
+from the plugin's contracts; the plan is read as plain data)."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from enum import StrEnum
+from pathlib import PurePosixPath
+from typing import Any
+
+from codeverse3d.contracts.artifacts import (
+ BuildResult,
+ GateReport,
+ Judgment,
+ Measurement,
+ RenderSet,
+)
+from codeverse3d.contracts.common import Usage
+from codeverse3d.contracts.spec import Spec
+from pydantic import BaseModel, ConfigDict, Field
+
+
+# ===================================================================== spec + plan (as recorded)
+class Budget(BaseModel):
+ """The v0.0.x run ceilings (the plugin has no budget)."""
+
+ max_rounds: int = 4
+ max_minutes: float = 60.0
+ max_repair_attempts: int = 3
+
+
+class Backends(BaseModel):
+ """Who did which job in a v0.0.x run (``<kind>:<model>`` ids)."""
+
+ planner: str = "gemini:gemini-4-argon"
+ generator: str = "agy:gemini-4-argon"
+ judge: str = "gemini:gemini-4-argon"
+ captioner: str = "gemini:gemini-4-argon"
+
+
+class LegacySpec(Spec):
+ budget: Budget = Field(default_factory=Budget)
+ backends: Backends = Field(default_factory=Backends)
+ tags: list[str] = Field(default_factory=list)
+ created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+
+
+class AcceptanceItem(BaseModel):
+ model_config = ConfigDict(extra="ignore")
+
+ id: str
+ text: str = ""
+ how: str = "visual"
+ priority: str = "must"
+
+
+class LegacyPlan(BaseModel):
+ """The plan as the gallery and the dataset read it: a name, a part count, the acceptance checklist."""
+
+ model_config = ConfigDict(extra="ignore")
+
+ object_name: str = ""
+ title: str = ""
+ parts: list[Any] = Field(default_factory=list)
+ acceptance: list[AcceptanceItem] = Field(default_factory=list)
+
+
+# ===================================================================== run
+# --------------------------------------------------------------------------- identity
+#: battery-layout path segments that are pure plumbing, never part of a run's identity
+RUN_PATH_NOISE = frozenset({"arms", "cells", "runs"})
+
+
+class RunId(BaseModel):
+ """Where a run directory sits inside its scan root, and the one name to call it.
+
+ ``battery`` labels the scan root; ``rel`` is the run dir's posix path relative to it.
+ Only the scanner knows the root, so only it mints one (``record.record.run_id_for``)."""
+
+ model_config = ConfigDict(frozen=True)
+
+ battery: str = Field(description="label of the scan root this run was found under")
+ rel: str = Field(description="posix path of the run dir relative to the scan root")
+
+ def _parts(self) -> tuple[str, ...]:
+ return PurePosixPath(self.rel).parts
+
+ @property
+ def slug(self) -> str:
+ """The stable, filesystem-safe name of this run.
+
+ A flat ``rel`` keeps its basename; a nested one joins its meaningful segments with ``__``
+ (dropping :data:`RUN_PATH_NOISE` and a trailing ``run``)."""
+ parts = self._parts()
+ if len(parts) <= 1:
+ return parts[0] if parts else self.battery
+ meaningful = [p for p in parts if p not in RUN_PATH_NOISE]
+ if len(meaningful) > 1 and meaningful[-1] == "run":
+ meaningful.pop() # the literal `run` dir a battery cell wraps its harness run in
+ if not meaningful:
+ meaningful = [parts[-1]]
+ return "__".join(meaningful)
+
+
+class RunStatus(StrEnum):
+ """Where a run is, and once stopped, why it stopped. Never a verdict."""
+
+ PLANNING = "planning"
+ GENERATING = "generating"
+ REFINING = "refining"
+ MAX_ROUNDS = "max_rounds" # every round the budget asked for ran
+ BUDGET = "budget" # the wall clock (``max_minutes``) ran out
+ AGENT_QUOTA = "agent_quota" # the vendor's usage limit, not ours
+ NO_CHANGE = "no_change" # a refine round's sessions all failed / changed nothing
+ NO_REFINE_TASKS = "no_refine_tasks" # the last round left nothing to ask for
+ JUDGE_UNAVAILABLE = "judge_unavailable" # the last round has no verdict, even after a re-judge
+ FAILED = "failed"
+
+
+class StepTime(BaseModel):
+ """One timed step: its wall clock and the part lost to provider errors (503 / 429 / timeouts).
+
+ Parallel work is ONE step, so no second is counted twice."""
+
+ step: str = Field(description="plan | skeleton | assets+env+layouts | zones | assemble"
+ " | generate | build | gates | render | judge")
+ round: int | None = Field(default=None, description="the round the step belongs to; None = a run-level stage")
+ wall_s: float = 0.0
+ lost_s: float = Field(default=0.0, description="seconds of wall_s lost to provider errors")
+
+ @property
+ def active_s(self) -> float:
+ return max(0.0, self.wall_s - self.lost_s)
+
+
+class RoundRecord(BaseModel):
+ index: int
+ kind: str = Field(description="baseline | refine | repair | asset:<name> ...")
+ commit: str = Field(default="", description="git commit of src/ after this round")
+ agent_backend: str = ""
+ instructions: list[str] = Field(default_factory=list, description="what the agent was asked to change")
+ build: BuildResult | None = None
+ gates: list[GateReport] = Field(default_factory=list)
+ measurement: Measurement | None = None
+ renders: RenderSet | None = None
+ judgment: Judgment | None = None
+ usage: Usage = Field(default_factory=Usage)
+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+ duration_s: float = Field(default=0.0, description="the round's clock, provider errors included")
+ steps: list[StepTime] = Field(
+ default_factory=list, description="the round's timed steps (generate / build / gates / render / judge); "
+ "empty until timing is recorded")
+ notes: str = ""
+
+ @property
+ def score(self) -> float | None:
+ return self.judgment.overall if self.judgment else None
+
+ @property
+ def minutes(self) -> float:
+ """The round's timed steps, excluding provider wait."""
+ return sum(s.active_s for s in self.steps) / 60.0
+
+
+# --------------------------------------------------------------------------- telemetry (settings + cost)
+class RoleSettings(BaseModel):
+ """Resolved model settings for ONE role of a run.
+
+ ``source``: ``run`` (stamped as it ran) or ``spec`` (only the model is known)."""
+
+ role: str
+ model: str = ""
+ backend: str = Field(default="", description="gemini | anthropic | agy | claude-code | codex | …")
+ thinking: str = Field(default="", description="off | low | medium | high; '' = not recorded")
+ temperature: float | None = None
+ n_samples: int | None = Field(default=None, description="judge samples per verdict")
+ source: str = "spec"
+
+
+class SettingsSnapshot(BaseModel):
+ """Everything that decided how the run ran."""
+
+ roles: list[RoleSettings] = Field(default_factory=list)
+ budget: dict[str, Any] = Field(default_factory=dict, description="max_rounds / max_minutes / max_repair_attempts")
+ rubric: str = ""
+ rubric_hash: str = Field(default="", description="content hash of the judge rubric YAML")
+ prompt_hashes: dict[str, str] = Field(default_factory=dict, description="contract / generate / refine / repair / skill:<name> / judge …")
+ tool_versions: dict[str, str] = Field(default_factory=dict, description="python / blender / node / three / chrome / harness sha")
+ harness_version: str = ""
+ harness_git_sha: str = ""
+ key_pool_size: int = Field(default=0, description="number of API keys in the rotating pool")
+ price_table_version: str = Field(default="", description="content hash of the price table the run was billed with")
+ render: dict[str, Any] = Field(default_factory=dict, description="resolved render settings (size, gpu, sheet)")
+
+
+class StageCost(BaseModel):
+
+ stage: str
+ calls: int = 0
+ input_tokens: int = 0
+ output_tokens: int = 0
+ cached_tokens: int = 0
+ thoughts_tokens: int = 0
+ cost_usd: float = 0.0
+ seconds: float = 0.0
+
+
+class CostSummary(BaseModel):
+ """The run's ledger rows summarised (``record.telemetry.cost``)."""
+
+ total_usd: float = Field(default=0.0, description="the ledger's sum at list price = record.total_usage")
+ minutes: float | None = Field(default=None, description="RunRecord.minutes when the record was written")
+ max_minutes: float = 0.0
+ n_calls: int = 0
+ tokens: Usage = Field(default_factory=Usage, description="run totals (record.total_usage)")
+ by_stage: list[StageCost] = Field(default_factory=list)
+ by_role: dict[str, float] = Field(default_factory=dict, description="planner/generator/judge/… → USD")
+ by_model: dict[str, float] = Field(default_factory=dict, description="model → USD")
+ by_round: list[dict[str, Any]] = Field(default_factory=list, description="index / kind / cost_usd / minutes / score")
+
+
+class RunTelemetry(BaseModel):
+
+ settings: SettingsSnapshot | None = None
+ cost: CostSummary | None = None
+
+
+# --------------------------------------------------------------------------- deliverable
+class DeliverableFile(BaseModel):
+
+ path: str = Field(description="run-relative path (under deliverable/)")
+ role: str = Field(description="code | model | urdf | mesh | sheet | preview | frames | captions | bundle | manifest")
+ bytes: int = 0
+ sha256: str = ""
+
+
+class RunDeliverable(BaseModel):
+ """``deliverable/manifest.json``: the handed-over round as paths + hashes + sizes."""
+
+ round: int | None = Field(default=None, description="the packaged round")
+ commit: str = ""
+ entry: str = Field(default="", description="run-relative entry file of the code snapshot")
+ files: list[DeliverableFile] = Field(default_factory=list)
+
+
+class RunRecord(BaseModel):
+ """One run."""
+
+ spec: LegacySpec
+ plan: LegacyPlan | None = None
+ workspace: str
+ status: RunStatus = RunStatus.PLANNING
+ rounds: list[RoundRecord] = Field(default_factory=list)
+ total_usage: Usage = Field(default_factory=Usage)
+ environment: dict[str, str] = Field(default_factory=dict, description="tool versions, git sha, host")
+ prompt_hashes: dict[str, str] = Field(default_factory=dict)
+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+ finished_at: datetime | None = None
+ error: str = ""
+ telemetry: RunTelemetry | None = Field(
+ default=None, description="settings snapshot + cost summary (written by finalize_record)")
+ steps: list[StepTime] = Field(
+ default_factory=list, description="the run-level timed steps (plan, the pre-round stages, best-of-N, the part of "
+ "an aborted round that ran)")
+ extra: dict[str, Any] = Field(default_factory=dict)
+
+ @property
+ def minutes(self) -> float:
+ """The run's minutes: ``wall_s - lost_s`` over every step (provider errors excluded)."""
+ return sum(s.active_s for s in self.steps) / 60.0 + sum(r.minutes for r in self.rounds)
diff --git a/flywheel/c3d_flywheel/models/__init__.py b/flywheel/c3d_flywheel/models/__init__.py
deleted file mode 100644
index ddeee53a..00000000
--- a/flywheel/c3d_flywheel/models/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-"""ChatModel backends (API). Use ``get_chat_model("gemini:gemini-4-argon")``."""
-
-from c3d_flywheel.models.registry import get_chat_model
-
-__all__ = ["get_chat_model"]
diff --git a/flywheel/c3d_flywheel/models/anthropic.py b/flywheel/c3d_flywheel/models/anthropic.py
deleted file mode 100644
index 1b42fe5a..00000000
--- a/flywheel/c3d_flywheel/models/anthropic.py
+++ /dev/null
@@ -1,248 +0,0 @@
-"""ChatRequest ⇄ Anthropic Messages API dicts (messages, submit tool, thinking, parse).
-
-Adaptive-thinking models take ``thinking={"type":"adaptive"}`` + ``output_config.effort``;
-some always think (``disabled`` is a 400) and reject a forced ``tool_choice``, so "off" is
-effort ``low``. Older models take ``budget_tokens`` (< max_tokens) and need default temperature.
-"""
-
-from __future__ import annotations
-
-import json
-import time
-from typing import Any
-
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.chat import (
- ChatMessage,
- ChatRequest,
- ChatResponse,
- ImagePart,
-)
-from codeverse3d.contracts.common import Usage
-from c3d_flywheel.models.base import ModelError
-from c3d_flywheel.models.parts import (
- SdkModel,
- Stopwatch,
- attempt_timeout_s,
- classify_sdk_exception,
- image_b64,
- message_blocks,
- retry_budget_s,
- retry_one_key,
-)
-from c3d_flywheel.models.pricing import cache_write_surcharge, estimate_cost
-from c3d_flywheel.models.schema_utils import (
- JsonParseError,
- parse_json_lenient,
- strip_control_chars,
- to_anthropic_schema,
-)
-
-SUBMIT_TOOL = "submit"
-THINKING_BUDGET: dict[str, int] = {"off": 0, "low": 1024, "medium": 4096, "high": 16000}
-EFFORT: dict[str, str] = {"off": "low", "low": "low", "medium": "medium", "high": "high"}
-
-_ADAPTIVE_MARKERS = (
- "opus-5",
- "sonnet-5",
- "haiku-5",
- "fable-5",
- "mythos",
- "opus-4-6",
- "opus-4-7",
- "opus-4-8",
- "sonnet-4-6",
-)
-_ALWAYS_THINKING = ("fable-5", "mythos", "opus-5-5", "sonnet-5-5")
-_NO_SAMPLING = ("opus-5", "sonnet-5", "haiku-5", "fable-5", "mythos", "opus-4-7", "opus-4-8")
-
-
-def uses_adaptive_thinking(model: str) -> bool:
- m = model.lower()
- return any(tok in m for tok in _ADAPTIVE_MARKERS)
-
-
-def thinking_always_on(model: str) -> bool:
- m = model.lower()
- return any(tok in m for tok in _ALWAYS_THINKING)
-
-
-def sampling_allowed(model: str) -> bool:
- m = model.lower()
- return not any(tok in m for tok in _NO_SAMPLING)
-
-
-# ------------------------------------------------------------------ messages
-def _image_block(p: ImagePart) -> dict[str, Any]:
- data, mime = image_b64(p)
- return {"type": "image", "source": {"type": "base64", "media_type": mime, "data": data}}
-
-
-def to_messages(messages: list[ChatMessage]) -> list[dict[str, Any]]:
- """ChatMessages → Anthropic ``messages``; consecutive same-role turns are merged."""
- out: list[dict[str, Any]] = []
- for msg in messages:
- role = "assistant" if msg.role == "assistant" else "user"
- blocks = message_blocks(msg, text=lambda t: {"type": "text", "text": t}, image=_image_block)
- if not blocks:
- continue
- if out and out[-1]["role"] == role:
- out[-1]["content"].extend(blocks)
- else:
- out.append({"role": role, "content": blocks})
- if not out:
- raise ModelError("ChatRequest has no content to send")
- if out[0]["role"] != "user":
- out.insert(0, {"role": "user", "content": [{"type": "text", "text": "(continue)"}]})
- return out
-
-
-# --------------------------------------------------------------------- tools
-def submit_tool(schema: dict[str, Any]) -> dict[str, Any]:
- return {
- "name": SUBMIT_TOOL,
- "description": "Submit the final answer. Call exactly once with the complete JSON object.",
- "input_schema": to_anthropic_schema(schema),
- }
-
-
-# ------------------------------------------------------------------- request
-def build_kwargs(request: ChatRequest, model: str) -> dict[str, Any]:
- """Build ``client.messages.create(**kwargs)`` (without ``model``/``timeout``); a
- response schema becomes a forced ``submit`` tool."""
- kw: dict[str, Any] = {
- "messages": to_messages(request.messages),
- "max_tokens": request.max_output_tokens,
- }
- system = request.system
- thinking_on = request.thinking != "off" or thinking_always_on(model)
-
- if uses_adaptive_thinking(model):
- if request.thinking != "off":
- kw["thinking"] = {"type": "adaptive"}
- kw["output_config"] = {"effort": EFFORT[request.thinking]}
- elif not thinking_always_on(model):
- kw["thinking"] = {"type": "disabled"}
- else:
- kw["output_config"] = {"effort": "low"}
- elif request.thinking != "off":
- budget = THINKING_BUDGET[request.thinking]
- kw["thinking"] = {"type": "enabled", "budget_tokens": budget}
- kw["max_tokens"] = max(request.max_output_tokens, budget + 2048)
-
- if sampling_allowed(model) and not thinking_on:
- kw["temperature"] = request.temperature
-
- if request.response_schema is not None:
- kw["tools"] = [submit_tool(request.response_schema)]
- if not thinking_on:
- kw["tool_choice"] = {"type": "tool", "name": SUBMIT_TOOL}
- else:
- # forced tool_choice is incompatible with thinking
- system = (
- (system + "\n\n" if system else "")
- + f"When you have the final answer, call the `{SUBMIT_TOOL}` tool exactly once with the complete JSON object."
- )
- if system:
- kw["system"] = system
- return kw
-
-
-# ------------------------------------------------------------------ response
-def parse_content(content: list[Any]) -> tuple[str, Any]:
- """→ ``(text, submit_input)`` from response blocks."""
- texts: list[str] = []
- submit: Any = None
- for block in content:
- btype = getattr(block, "type", None)
- if btype == "text":
- texts.append(block.text)
- elif btype == "tool_use" and block.name == SUBMIT_TOOL:
- submit = (
- block.input
- if isinstance(block.input, dict)
- else json.loads(json.dumps(block.input))
- )
- return "".join(texts), submit
-
-
-# ===================================================================== anthropic
-def classify_exception(exc: BaseException) -> ModelError:
- """Map anthropic SDK exceptions onto ``ModelError``."""
- import anthropic
-
- return classify_sdk_exception(exc, anthropic, "Anthropic")
-
-
-class AnthropicModel(SdkModel):
- """ChatModel for ``anthropic:<model>``."""
-
- provider = "anthropic"
-
- def _make_client(self) -> Any:
- import anthropic
-
-
- key = get_settings().anthropic_api_key
- if not key:
- raise ModelError("ANTHROPIC_API_KEY is not configured")
- return anthropic.Anthropic(api_key=key, max_retries=0, timeout=self.timeout_s)
-
- # -------------------------------------------------------------- generate
- def generate(self, request: ChatRequest) -> ChatResponse:
- kwargs = build_kwargs(request, self.model)
- deadline = time.monotonic() + retry_budget_s(request.max_wait_s)
- return retry_one_key(lambda: self._once(kwargs, request, deadline), label=f"anthropic {self.model}",
- classify=classify_exception, attempts=self.max_attempts, sleep=self._sleep,
- max_wait_s=request.max_wait_s)
-
- def _once(self, kwargs: dict[str, Any], request: ChatRequest, deadline: float) -> ChatResponse:
- client = self.client()
- with Stopwatch() as sw:
- msg = client.messages.create(
- model=self.model, timeout=attempt_timeout_s(deadline, self.timeout_s), **kwargs)
- text, submit = parse_content(msg.content)
- stop = str(msg.stop_reason or "")
- usage = self._usage(msg.usage, sw.ms)
- if stop == "refusal":
- raise ModelError(
- f"Anthropic refused the request: {getattr(msg, 'stop_details', None)}",
- retryable=False, usage=usage, # a refusal is billed like any other reply
- )
- parsed: Any = None
- if request.response_schema is not None:
- if submit is not None:
- # tool input skips parse_json_lenient; strip NULs here or they reach subprocess.Popen
- parsed = strip_control_chars(submit)
- if not text:
- text = json.dumps(parsed)
- else:
- try:
- parsed = parse_json_lenient(text)
- except JsonParseError as exc:
- raise ModelError(
- f"structured output missing (stop_reason={stop}): {exc}",
- retryable=stop != "max_tokens", usage=usage,
- ) from exc
- if not text and parsed is None:
- raise ModelError(f"Anthropic returned no content (stop_reason={stop})",
- retryable=True, usage=usage)
- raw: dict[str, Any] = {"stop_reason": stop, "id": msg.id, "model": msg.model}
- return ChatResponse(text=text, parsed=parsed, finish_reason=stop, usage=usage, raw=raw)
-
- def _usage(self, u: Any, latency_ms: int) -> Usage:
- cache_read = int(getattr(u, "cache_read_input_tokens", 0) or 0)
- cache_write = int(getattr(u, "cache_creation_input_tokens", 0) or 0)
- usage = Usage(
- backend="anthropic",
- model=self.model,
- input_tokens=int(u.input_tokens or 0) + cache_read + cache_write,
- output_tokens=int(u.output_tokens or 0),
- cached_tokens=cache_read,
- thoughts_tokens=0, # thinking is billed inside output_tokens
- latency_ms=latency_ms,
- )
- usage.cost_usd = estimate_cost("anthropic", self.model, usage) + cache_write_surcharge(
- "anthropic", self.model, cache_write
- )
- return usage
diff --git a/flywheel/c3d_flywheel/models/base.py b/flywheel/c3d_flywheel/models/base.py
deleted file mode 100644
index 26554009..00000000
--- a/flywheel/c3d_flywheel/models/base.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""The ChatModel protocol every API backend implements."""
-
-from __future__ import annotations
-
-from typing import Protocol, runtime_checkable
-
-from codeverse3d.contracts.chat import ChatRequest, ChatResponse
-from codeverse3d.contracts.common import Usage
-
-
-@runtime_checkable
-class ChatModel(Protocol):
- """One provider+model. Implementations must be thread-safe (the
- orchestrator fans out judge/caption calls across threads)."""
-
- provider: str # gemini | anthropic | openai
- model: str
-
- @property
- def id(self) -> str: ...
-
- def generate(self, request: ChatRequest) -> ChatResponse:
- """Blocking call. Must:
- * honour ``request.response_schema`` (return ``parsed`` as a dict/list),
- * attach images (``ImagePart``) inline in order,
- * fill ``usage`` including ``cost_usd`` (see ``pricing.py``),
- * raise ``ModelError`` (retryable=True/False) rather than provider exceptions.
- """
- ...
-
-
-class ModelError(RuntimeError):
- """A model call failed. ``attempts`` = round-trips issued (0 = unknown); ``usage`` =
- what the provider already billed for the failed call (empty when nothing was)."""
-
- def __init__(
- self,
- message: str,
- *,
- retryable: bool = False,
- status: int | None = None,
- attempts: int = 0,
- usage: Usage | None = None,
- ):
- super().__init__(message)
- self.retryable = retryable
- self.status = status
- self.attempts = attempts
- self.usage = usage if usage is not None else Usage()
diff --git a/flywheel/c3d_flywheel/models/gemini.py b/flywheel/c3d_flywheel/models/gemini.py
deleted file mode 100644
index 7fd4d545..00000000
--- a/flywheel/c3d_flywheel/models/gemini.py
+++ /dev/null
@@ -1,429 +0,0 @@
-"""Gemini backend: ``ChatRequest`` ⇄ google-genai conversion, and ``GeminiModel`` (shared
-``KeyPool``, ``rotate_with_retries``, streaming with stall detection)."""
-
-from __future__ import annotations
-
-import logging
-import random
-import threading
-import time
-from collections.abc import Callable, Iterable
-from typing import Any
-
-import httpx
-from google import genai
-from google.genai import errors as genai_errors
-from google.genai import types
-
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.chat import (
- ChatMessage,
- ChatRequest,
- ChatResponse,
- ImagePart,
-)
-from codeverse3d.contracts.common import Usage
-from c3d_flywheel.models.base import ModelError
-from c3d_flywheel.models.parts import (
- FailedBills,
- Stopwatch,
- attempt_timeout_s,
- default_timeout_s,
- image_bytes,
- message_blocks,
- retry_budget_s,
- retryable_status,
-)
-from c3d_flywheel.models.pricing import estimate_cost
-from c3d_flywheel.models.retry import KeyPool, KeyPoolExhausted, rotate_with_retries
-from c3d_flywheel.models.schema_utils import JsonParseError, parse_json_lenient, to_gemini_schema
-
-#: thinking level → token budget
-THINKING_BUDGET: dict[str, int] = {"off": 0, "low": 1024, "medium": 4096, "high": 16384}
-
-#: finish reasons worth one more attempt
-RETRYABLE_FINISH = {"RECITATION", "MALFORMED_FUNCTION_CALL", "OTHER", "UNEXPECTED_TOOL_CALL"}
-#: finish reasons retrying cannot fix
-FATAL_FINISH = {"SAFETY", "PROHIBITED_CONTENT", "BLOCKLIST", "SPII", "IMAGE_SAFETY"}
-
-
-# ------------------------------------------------------------------ contents
-def _image_part(p: ImagePart) -> types.Part:
- raw, mime = image_bytes(p)
- return types.Part.from_bytes(data=raw, mime_type=mime)
-
-
-def to_contents(messages: list[ChatMessage]) -> list[types.Content]:
- """ChatMessages → Gemini ``Content`` list."""
- out: list[types.Content] = []
- for msg in messages:
- role = "model" if msg.role == "assistant" else "user"
- parts: list[types.Part] = message_blocks(msg, text=lambda t: types.Part.from_text(text=t), image=_image_part)
- if not parts:
- continue
- # Gemini wants strict role alternation
- if out and out[-1].role == role:
- out[-1].parts.extend(parts) # type: ignore[union-attr]
- else:
- out.append(types.Content(role=role, parts=parts))
- if not out:
- raise ModelError("ChatRequest has no content to send")
- return out
-
-
-# -------------------------------------------------------------------- config
-def build_config(
- request: ChatRequest,
- *,
- timeout_ms: int,
- use_thinking: bool = True,
-) -> types.GenerateContentConfig:
- """Map a ChatRequest onto ``GenerateContentConfig``."""
- cfg: dict[str, Any] = {
- "temperature": request.temperature,
- "max_output_tokens": request.max_output_tokens,
- "http_options": types.HttpOptions(timeout=timeout_ms),
- }
- if request.system:
- cfg["system_instruction"] = request.system
- if use_thinking:
- cfg["thinking_config"] = types.ThinkingConfig(
- thinking_budget=THINKING_BUDGET[request.thinking], include_thoughts=False
- )
- if request.response_schema is not None:
- cfg["response_mime_type"] = "application/json"
- cfg["response_schema"] = to_gemini_schema(request.response_schema)
- return types.GenerateContentConfig(**cfg)
-
-
-def clip_timeout(
- config: types.GenerateContentConfig, timeout_ms: int
-) -> types.GenerateContentConfig:
- """``config`` with its HTTP timeout set to ``timeout_ms`` (same object if unchanged)."""
- current = config.http_options.timeout if config.http_options is not None else None
- if current == timeout_ms:
- return config
- return config.model_copy(update={"http_options": types.HttpOptions(timeout=timeout_ms)})
-
-
-# ------------------------------------------------------------------ response
-def parse_usage(resp: types.GenerateContentResponse, model: str) -> Usage:
- um = resp.usage_metadata
- if um is None:
- return Usage(backend="gemini", model=model)
- return Usage(
- backend="gemini",
- model=resp.model_version or model,
- input_tokens=(um.prompt_token_count or 0) + (um.tool_use_prompt_token_count or 0),
- output_tokens=um.candidates_token_count or 0,
- cached_tokens=um.cached_content_token_count or 0,
- thoughts_tokens=um.thoughts_token_count or 0,
- )
-
-
-def _guard_candidates(resp: types.GenerateContentResponse, usage: Usage | None, what: str) -> Any:
- """The first candidate, or a blocked-prompt (fatal) / no-candidates (retryable) error."""
- pf = resp.prompt_feedback
- if pf is not None and pf.block_reason:
- raise ModelError(
- f"{what} prompt blocked: {pf.block_reason} {pf.block_reason_message or ''}".strip(),
- retryable=False, usage=usage,
- )
- if not resp.candidates:
- raise ModelError(f"{what} returned no candidates", retryable=True, usage=usage)
- return resp.candidates[0]
-
-
-def extract_candidate(
- resp: types.GenerateContentResponse, usage: Usage | None = None
-) -> tuple[str, str]:
- """→ ``(text, finish_reason)``; raises ``ModelError`` carrying ``usage`` (these failures are billed)."""
- cand = _guard_candidates(resp, usage, "Gemini")
- finish = str(cand.finish_reason.value if cand.finish_reason else "") or "UNKNOWN"
- texts: list[str] = []
- for part in cand.content.parts if cand.content and cand.content.parts else []:
- if part.text and not part.thought:
- texts.append(part.text)
- text = "".join(texts)
- if not text:
- if finish in FATAL_FINISH:
- raise ModelError(
- f"Gemini produced no content (finish_reason={finish})", retryable=False, usage=usage
- )
- if finish == "MAX_TOKENS":
- # Gemini 3 always thinks; a small max_output_tokens is eaten by thoughts. The
- # "finish_reason=MAX_TOKENS" text is matched by the planner's grow-the-budget retry.
- thoughts = resp.usage_metadata.thoughts_token_count if resp.usage_metadata else None
- raise ModelError(
- f"Gemini produced no content (finish_reason=MAX_TOKENS): max_output_tokens exhausted by thinking "
- f"(thoughts_tokens={thoughts}); raise max_output_tokens",
- retryable=False, usage=usage,
- )
- raise ModelError(f"Gemini produced no content (finish_reason={finish})",
- retryable=True, usage=usage)
- return text, finish
-
-
-# ===================================================================== gemini
-log = logging.getLogger(__name__)
-
-#: ceiling of the inter-chunk read timeout under streaming: a hung socket dies within it,
-#: while a long healthy stream is bounded only by its attempt budget
-STREAM_STALL_S = 300.0
-
-_pools: dict[tuple[str, ...], KeyPool] = {}
-_clients: dict[tuple[str, int], genai.Client] = {}
-_registry_lock = threading.Lock()
-
-
-def shared_pool(keys: list[str], *, max_in_flight: int | None = None) -> KeyPool:
- """One ``KeyPool`` per (key list, in-flight cap) per process; slots are machine-wide
- (``<cache_dir>/slots/gemini/NN.lock``)."""
- cap = _rate().max_in_flight if max_in_flight is None else max_in_flight
- sig = (*keys, f"|{cap}")
- with _registry_lock:
- pool = _pools.get(sig)
- if pool is None:
- slots = get_settings().cache_dir / "slots" / "gemini" if cap else None
- # random first key, so parallel processes do not all start on key 0
- pool = KeyPool(keys, max_in_flight=cap or 0, slots_dir=slots, start=random.randrange(len(keys)))
- _pools[sig] = pool
- return pool
-
-
-def _rate() -> Any:
- return get_settings().rate
-
-
-def _default_keys() -> list[str]:
- return list(get_settings().gemini_api_keys)
-
-
-def _keys_or_raise(keys: list[str] | None, pool: KeyPool | None) -> list[str]:
- keys = list(keys) if keys is not None else _default_keys()
- if pool is None and not keys:
- raise ModelError("no Gemini API keys configured (GEMINI_API_KEYS / ~/.config/astra3d/gemini_keys.env)")
- return keys
-
-
-def _client_for(key: str, timeout_s: float, factory: Callable[[str], Any] | None) -> Any:
- """One ``genai.Client`` per (key, timeout), process-wide; ``factory`` bypasses the cache."""
- if factory is not None:
- return factory(key)
- ck = (key, int(timeout_s * 1000))
- with _registry_lock:
- client = _clients.get(ck)
- if client is None:
- client = genai.Client(api_key=key, http_options=types.HttpOptions(
- timeout=ck[1],
- # the WSL2 IPv6 path silently hangs reads: bind the transport to IPv4
- client_args={"transport": httpx.HTTPTransport(local_address="0.0.0.0")}))
- _clients[ck] = client
- return client
-
-
-def classify_exception(exc: BaseException) -> ModelError:
- """Map provider / transport exceptions onto ``ModelError``."""
- if isinstance(exc, ModelError):
- return exc
- if isinstance(exc, genai_errors.APIError):
- code = int(exc.code or 0)
- msg = f"Gemini API error {code}: {exc.message}"
- return ModelError(msg, retryable=retryable_status(code), status=code)
- if isinstance(exc, (httpx.TimeoutException, TimeoutError)):
- return ModelError(f"Gemini request timed out: {exc}", retryable=True, status=408)
- if isinstance(exc, (httpx.TransportError, ConnectionError)):
- return ModelError(f"Gemini transport error: {exc}", retryable=True)
- if isinstance(exc, KeyPoolExhausted):
- return ModelError(f"Gemini key pool exhausted: {exc}", retryable=False, status=429)
- return ModelError(f"Gemini unexpected error: {type(exc).__name__}: {exc}", retryable=False)
-
-
-def _drain_stream(
- it: Iterable[types.GenerateContentResponse],
- deadline: float,
- clock: Callable[[], float] = time.monotonic,
-) -> list[types.GenerateContentResponse]:
- """Collect every chunk, enforcing the attempt ``deadline`` between chunks (the wire
- timeout only bounds silence between chunks)."""
- chunks: list[types.GenerateContentResponse] = []
- for ch in it:
- chunks.append(ch)
- if clock() > deadline:
- raise ModelError(
- f"Gemini stream exceeded its attempt budget after {len(chunks)} chunks",
- retryable=True,
- )
- return chunks
-
-
-def _merge_stream_chunks(
- chunks: list[types.GenerateContentResponse],
-) -> types.GenerateContentResponse:
- """One response from a chunk sequence: parts concatenate; finish_reason, usage_metadata
- and prompt_feedback come from the last chunk carrying each."""
- if not chunks:
- raise ModelError("Gemini stream yielded no chunks", retryable=True)
- parts: list[types.Part] = []
- finish = None
- usage_md = None
- feedback = None
- base = None
- for ch in chunks:
- if ch.usage_metadata is not None:
- usage_md = ch.usage_metadata
- if ch.prompt_feedback is not None and ch.prompt_feedback.block_reason:
- feedback = ch.prompt_feedback
- if ch.candidates:
- base = ch
- cand = ch.candidates[0]
- if cand.content and cand.content.parts:
- parts.extend(cand.content.parts)
- if cand.finish_reason is not None:
- finish = cand.finish_reason
- if base is None: # no candidates: extract_candidate raises
- base = chunks[-1]
- else:
- base.candidates[0].content = types.Content(role="model", parts=parts)
- base.candidates[0].finish_reason = finish
- base.usage_metadata = usage_md
- if feedback is not None:
- base.prompt_feedback = feedback
- return base
-
-
-class GeminiModel:
- """ChatModel for ``gemini:<model>``."""
-
- provider = "gemini"
-
- def __init__(
- self,
- model: str,
- *,
- keys: list[str] | None = None,
- pool: KeyPool | None = None,
- timeout_s: float | None = None,
- max_attempts: int = 6,
- storm_attempts: int | None = None,
- sleep: Callable[[float], None] = time.sleep,
- client_factory: Callable[[str], Any] | None = None,
- ) -> None:
- self.model = model
- #: None = rotate_with_retries' default; 0 = a 503 is final (the storm branch ignores max_attempts)
- self.storm_attempts = storm_attempts
- keys = _keys_or_raise(keys, pool)
- self.pool = pool or shared_pool(keys)
- self.timeout_s = default_timeout_s() if timeout_s is None else timeout_s
- self.max_attempts = max(1, max_attempts)
- self._sleep = sleep
- self._client_factory = client_factory
- self._thinking_ok = True # flipped when the model rejects ThinkingConfig
- self._lock = threading.Lock()
-
- # ------------------------------------------------------------------ proto
- @property
- def id(self) -> str:
- return f"gemini:{self.model}"
-
- # --------------------------------------------------------------- generate
- def generate(self, request: ChatRequest) -> ChatResponse:
- contents = to_contents(request.messages)
- state = {"config": self._config(request)}
-
- def downgrade_thinking(err: ModelError) -> bool:
- """Model rejects ThinkingConfig → retry without it, once."""
- if not (self._thinking_ok and err.status == 400 and "thinking" in str(err).lower()):
- return False
- with self._lock:
- self._thinking_ok = False
- log.warning("%s rejected thinking config; retrying without it: %s", self.model, err)
- state["config"] = self._config(request)
- return True
-
- budget = retry_budget_s(request.max_wait_s)
- deadline = time.monotonic() + budget
- stats: dict[str, Any] = {}
- bills = FailedBills()
- try:
- resp = rotate_with_retries(
- self.pool,
- bills.track(lambda key: self._once(
- key, contents, self._attempt_config(state["config"], deadline),
- request,
- )),
- classify=classify_exception,
- max_attempts=self.max_attempts,
- max_total_s=budget,
- sleep=self._sleep,
- on_free_retry=downgrade_thinking,
- label=f"gemini {self.model}",
- stats=stats,
- **({} if self.storm_attempts is None else {"storm_attempts": self.storm_attempts}),
- )
- except ModelError as e:
- bills.charge(e)
- raise
- resp = bills.join(resp)
- resp.raw["attempts"] = int(stats.get("attempts", 0))
- return resp
-
- def _config(self, request: ChatRequest) -> types.GenerateContentConfig:
- return build_config(
- request,
- timeout_ms=int(self.timeout_s * 1000),
- use_thinking=self._thinking_ok,
- )
-
- def _attempt_config(
- self, config: types.GenerateContentConfig, deadline: float
- ) -> types.GenerateContentConfig:
- """Config whose HTTP timeout is the remaining retry budget (see ``attempt_timeout_s``)."""
- return clip_timeout(config, int(attempt_timeout_s(deadline, self.timeout_s) * 1000))
-
- def _once(
- self,
- key: str,
- contents: list[types.Content],
- config: types.GenerateContentConfig,
- request: ChatRequest,
- ) -> ChatResponse:
- client = _client_for(key, self.timeout_s, self._client_factory)
-
- # the attempt budget becomes an in-loop deadline; the wire timeout drops to the stall ceiling
- cfg_ms = config.http_options.timeout if config.http_options is not None else None
- attempt_s = (float(cfg_ms) / 1000.0) if cfg_ms else self.timeout_s
- wire = clip_timeout(config, int(min(attempt_s, STREAM_STALL_S) * 1000))
- with Stopwatch() as sw:
- chunks = _drain_stream(
- client.models.generate_content_stream(
- model=self.model, contents=contents, config=wire
- ),
- time.monotonic() + attempt_s,
- )
- resp = _merge_stream_chunks(chunks)
- usage = parse_usage(resp, self.model) # before extract_candidate, whose raises carry it
- usage.latency_ms = sw.ms
- usage.cost_usd = estimate_cost("gemini", self.model, usage)
- text, finish = extract_candidate(resp, usage)
- parsed: Any = None
- if request.response_schema is not None:
- try:
- parsed = parse_json_lenient(text)
- except JsonParseError as exc:
- if finish in FATAL_FINISH or finish == "MAX_TOKENS":
- raise ModelError(
- f"structured output unavailable (finish_reason={finish}; "
- f"raise max_output_tokens if truncated): {exc}",
- retryable=False,
- usage=usage,
- ) from exc
- raise ModelError(
- f"structured output is not valid JSON (finish_reason={finish}): {exc}",
- retryable=True,
- usage=usage,
- ) from exc
- elif finish in RETRYABLE_FINISH and not text.strip():
- raise ModelError(f"Gemini finish_reason={finish}", retryable=True, usage=usage)
- return ChatResponse(text=text, parsed=parsed, finish_reason=finish, usage=usage,
- raw={"key": f"…{key[-4:]}"})
diff --git a/flywheel/c3d_flywheel/models/health.py b/flywheel/c3d_flywheel/models/health.py
deleted file mode 100644
index c3ad6d7d..00000000
--- a/flywheel/c3d_flywheel/models/health.py
+++ /dev/null
@@ -1,109 +0,0 @@
-"""Is a model actually serving right now? ``probe`` asks once per key, in parallel, with a
-short timeout and no retries; callers use it as a preflight gate before spending real money.
-The probe prompt is workload-sized: a tiny prompt can pass while large calls still fail.
-"""
-
-from __future__ import annotations
-
-import concurrent.futures as cf
-import logging
-import random
-from dataclasses import dataclass
-from typing import Any
-
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.common import Backends
-from c3d_flywheel.models.registry import build_chat_model
-
-log = logging.getLogger(__name__)
-
-#: pool keys sampled per probe
-DEFAULT_SAMPLE = 6
-DEFAULT_TIMEOUT_S = 30.0
-#: probe prompt size in tokens, sized like a real planner/generator call
-PROBE_TOKENS = 8000
-#: below this share of answering calls the model counts as down (a battery needs many calls to land)
-DEFAULT_MIN_OK = 0.75
-
-
-@dataclass(frozen=True)
-class Health:
- model: str
- n_ok: int
- n_tried: int
- reasons: tuple[str, ...] = ()
-
- @property
- def ok(self) -> bool:
- return self.n_tried > 0 and self.n_ok / self.n_tried >= DEFAULT_MIN_OK
-
- def __str__(self) -> str:
- head = f"{self.model}: {self.n_ok}/{self.n_tried} keys answering"
- return (
- head if self.ok else f"{head} — {'; '.join(self.reasons[:2]) or 'no reason reported'}"
- )
-
-
-def probe(
- model: str = Backends().planner,
- *,
- sample: int = DEFAULT_SAMPLE,
- timeout_s: float = DEFAULT_TIMEOUT_S,
-) -> Health:
- """One workload-sized call (``PROBE_TOKENS``) per sampled key, in parallel, no retries. Never raises."""
- from codeverse3d.contracts.chat import ChatMessage, ChatRequest
-
- prompt = _probe_prompt()
-
- def one(key: str | None) -> str | None:
- try:
- m = _bare_model(model, timeout_s, key)
- m.generate(
- ChatRequest(
- messages=[ChatMessage.user(prompt)],
- temperature=0.0,
- max_output_tokens=256,
- thinking="off",
- label="health",
- max_wait_s=timeout_s,
- )
- )
- return None
- except Exception as e: # noqa: BLE001 — a probe reports, it does not propagate
- return f"{type(e).__name__}: {str(e)[:100]}"
-
- keys = _sampled_keys(model, max(1, sample))
- n = len(keys)
- with cf.ThreadPoolExecutor(max_workers=n) as pool:
- out = list(pool.map(one, keys))
- reasons = tuple(dict.fromkeys(r for r in out if r)) # de-duplicated, order kept
- return Health(model=model, n_ok=sum(r is None for r in out), n_tried=n, reasons=reasons)
-
-
-def _probe_prompt(tokens: int = PROBE_TOKENS) -> str:
- """A workload-sized prompt whose answer is still one word."""
- filler = "The quick brown fox jumps over the lazy dog. " * max(1, tokens // 9)
- return f"{filler}\n\nIgnore the text above. Reply with the single word: pong"
-
-
-def _sampled_keys(model_id: str, n: int) -> list[str | None]:
- """One pool key per probe for a gemini model (distinct while the pool allows); ``[None] * n`` otherwise."""
- if ":" in model_id and not model_id.startswith("gemini:"):
- return [None] * n
- keys = list(get_settings().gemini_api_keys)
- if not keys:
- return [None] * n
- chosen = random.sample(keys, min(n, len(keys)))
- return [chosen[i % len(chosen)] for i in range(n)]
-
-
-def _bare_model(model_id: str, timeout_s: float, key: str | None = None):
- """The model with every retry off, on ONE key when given; a bare name is a gemini model."""
- if ":" not in model_id:
- model_id = f"gemini:{model_id}"
- extra: dict[str, Any] = {}
- if model_id.startswith("gemini:"):
- extra = {"storm_attempts": 0}
- if key is not None:
- extra["keys"] = [key]
- return build_chat_model(model_id, timeout_s=timeout_s, max_attempts=1, **extra)
diff --git a/flywheel/c3d_flywheel/models/openai.py b/flywheel/c3d_flywheel/models/openai.py
deleted file mode 100644
index 4e36778f..00000000
--- a/flywheel/c3d_flywheel/models/openai.py
+++ /dev/null
@@ -1,227 +0,0 @@
-"""ChatRequest ⇄ OpenAI Chat Completions dicts (messages, response_format).
-
-Chat Completions is used because every OpenAI-compatible endpoint (``OPENAI_BASE_URL``) speaks it.
-"""
-
-from __future__ import annotations
-
-import logging
-import time
-from typing import Any
-
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.chat import (
- ChatMessage,
- ChatRequest,
- ChatResponse,
- ImagePart,
- TextPart,
-)
-from codeverse3d.contracts.common import Usage
-from c3d_flywheel.models.base import ModelError
-from c3d_flywheel.models.parts import (
- SdkModel,
- Stopwatch,
- attempt_timeout_s,
- classify_sdk_exception,
- image_b64,
- message_blocks,
- retry_budget_s,
- retry_one_key,
-)
-from c3d_flywheel.models.pricing import openai_usage
-from c3d_flywheel.models.retry import cause_for
-from c3d_flywheel.models.schema_utils import (
- JsonParseError,
- inline_refs,
- parse_json_lenient,
- to_openai_strict_schema,
-)
-
-SCHEMA_NAME = "response"
-
-
-# ------------------------------------------------------------- model families
-def is_reasoning_model(model: str) -> bool:
- return model.lower().startswith("gpt-6")
-
-
-def reasoning_effort(model: str, thinking: str) -> str | None:
- """Map ``thinking`` onto ``reasoning_effort`` (None for non-reasoning models). gpt-6 has no
- off switch, so "off" is ``low``."""
- if not is_reasoning_model(model):
- return None
- return "low" if thinking == "off" else thinking
-
-
-# ------------------------------------------------------------------ messages
-def _image_block(p: ImagePart) -> dict[str, Any]:
- data, mime = image_b64(p)
- return {
- "type": "image_url",
- "image_url": {"url": f"data:{mime};base64,{data}", "detail": "auto"},
- }
-
-
-def to_messages(messages: list[ChatMessage], system: str) -> list[dict[str, Any]]:
- """ChatMessages → Chat Completions messages."""
- out: list[dict[str, Any]] = []
- if system:
- out.append({"role": "system", "content": system})
- for msg in messages:
- if msg.role == "assistant":
- out.append(_assistant_message(msg))
- continue
- content = message_blocks(msg, text=lambda t: {"type": "text", "text": t}, image=_image_block)
- if content:
- out.append({"role": "user", "content": content})
- if not out or all(m["role"] == "system" for m in out):
- raise ModelError("ChatRequest has no content to send")
- return out
-
-
-def _assistant_message(msg: ChatMessage) -> dict[str, Any]:
- text = "".join(p.text for p in msg.parts if isinstance(p, TextPart))
- return {"role": "assistant", "content": text or None}
-
-
-def response_format(schema: dict[str, Any], *, strict: bool) -> dict[str, Any]:
- body = to_openai_strict_schema(schema) if strict else inline_refs(schema)
- return {
- "type": "json_schema",
- "json_schema": {"name": SCHEMA_NAME, "schema": body, "strict": strict},
- }
-
-
-# ------------------------------------------------------------------- request
-def build_kwargs(request: ChatRequest, model: str, *, strict_schema: bool) -> dict[str, Any]:
- """``client.chat.completions.create(**kwargs)`` minus ``model``."""
- kw: dict[str, Any] = {
- "messages": to_messages(request.messages, request.system),
- "max_completion_tokens": request.max_output_tokens,
- }
- effort = reasoning_effort(model, request.thinking)
- if effort is not None:
- kw["reasoning_effort"] = effort
- else:
- kw["temperature"] = request.temperature
- if request.response_schema is not None:
- kw["response_format"] = response_format(request.response_schema, strict=strict_schema)
- return kw
-
-
-# ------------------------------------------------------------------ response
-def parse_choice(choice: Any) -> tuple[str, str]:
- """→ ``(text, finish_reason)``."""
- msg = choice.message
- text = msg.content or ""
- if getattr(msg, "refusal", None):
- raise ModelError(f"OpenAI refusal: {msg.refusal}", retryable=False)
- return text, str(choice.finish_reason or "")
-
-
-# ===================================================================== openai
-log = logging.getLogger(__name__)
-
-
-def classify_exception(exc: BaseException) -> ModelError:
- """Map openai SDK exceptions onto ``ModelError``."""
- import openai
-
- return classify_sdk_exception(exc, openai, "OpenAI")
-
-
-def _is_schema_rejection(err: ModelError) -> bool:
- s = str(err).lower()
- return err.status == 400 and ("schema" in s or "response_format" in s or "strict" in s)
-
-
-class OpenAIModel(SdkModel):
- """ChatModel for ``openai:<model>`` (and OpenAI-compatible endpoints)."""
-
- provider = "openai"
- _strict_ok = True # flipped (per instance) when the endpoint rejects strict schemas
-
- def _make_client(self) -> Any:
- import openai
-
-
- s = get_settings()
- key = s.openai_api_key
- base_url = s.openai_base_url or None
- if not key and not base_url:
- raise ModelError("OPENAI_API_KEY is not configured")
- return openai.OpenAI(
- api_key=key or "sk-local",
- base_url=base_url,
- max_retries=0,
- timeout=self.timeout_s,
- )
-
- # -------------------------------------------------------------- generate
- def generate(self, request: ChatRequest) -> ChatResponse:
- deadline = time.monotonic() + retry_budget_s(request.max_wait_s)
-
- def attempt() -> ChatResponse:
- strict = self._strict_ok and request.response_schema is not None
- kwargs = build_kwargs(request, self.model, strict_schema=strict)
- try:
- return self._once(kwargs, request, deadline)
- except Exception as exc: # noqa: BLE001 - classified
- err = classify_exception(exc)
- if strict and _is_schema_rejection(err):
- log.warning(
- "openai %s rejected strict json_schema (%s); falling back to strict=false",
- self.model,
- err,
- )
- with self._lock:
- self._strict_ok = False
- kwargs = build_kwargs(request, self.model, strict_schema=False)
- try:
- return self._once(kwargs, request, deadline)
- except Exception as exc2: # noqa: BLE001
- err2 = classify_exception(exc2)
- raise err2 from cause_for(err2, exc2)
- raise err from cause_for(err, exc)
-
- return retry_one_key(attempt, label=f"openai {self.model}", classify=classify_exception,
- attempts=self.max_attempts, sleep=self._sleep, max_wait_s=request.max_wait_s)
-
- def _once(self, kwargs: dict[str, Any], request: ChatRequest, deadline: float) -> ChatResponse:
- client = self.client()
- with Stopwatch() as sw:
- completion = client.chat.completions.create(
- model=self.model, timeout=attempt_timeout_s(deadline, self.timeout_s), **kwargs)
- if not completion.choices:
- raise ModelError("OpenAI returned no choices", retryable=True)
- text, finish = parse_choice(completion.choices[0])
- usage = self._usage(completion.usage, sw.ms)
- parsed: Any = None
- if request.response_schema is not None:
- try:
- parsed = parse_json_lenient(text)
- except JsonParseError as exc:
- raise ModelError(
- f"structured output is not valid JSON (finish_reason={finish}): {exc}",
- retryable=finish != "length", usage=usage, # billed like a good reply
- ) from exc
- if not text:
- raise ModelError(
- f"OpenAI returned no content (finish_reason={finish})",
- retryable=finish != "content_filter", usage=usage,
- )
- raw: dict[str, Any] = {
- "finish_reason": finish,
- "id": completion.id,
- "model": completion.model,
- }
- return ChatResponse(text=text, parsed=parsed, finish_reason=finish, usage=usage, raw=raw)
-
- def _usage(self, u: Any, latency_ms: int) -> Usage:
- ptd = getattr(u, "prompt_tokens_details", None)
- ctd = getattr(u, "completion_tokens_details", None)
- return openai_usage("openai", self.model, prompt=int(getattr(u, "prompt_tokens", 0) or 0),
- cached=int(getattr(ptd, "cached_tokens", 0) or 0),
- completion=int(getattr(u, "completion_tokens", 0) or 0),
- reasoning=int(getattr(ctd, "reasoning_tokens", 0) or 0), latency_ms=latency_ms)
diff --git a/flywheel/c3d_flywheel/models/parts.py b/flywheel/c3d_flywheel/models/parts.py
deleted file mode 100644
index 142a4c8f..00000000
--- a/flywheel/c3d_flywheel/models/parts.py
+++ /dev/null
@@ -1,196 +0,0 @@
-"""Small helpers shared by the provider backends (images, timing, usage)."""
-
-from __future__ import annotations
-
-import base64
-import mimetypes
-import threading
-import time
-from collections.abc import Callable
-from pathlib import Path
-from typing import Any
-
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.chat import ChatMessage, ImagePart, TextPart
-from codeverse3d.contracts.common import Usage
-from c3d_flywheel.models.base import ModelError
-
-
-def image_bytes(part: ImagePart) -> tuple[bytes, str]:
- """Return ``(raw_bytes, mime)`` for an ``ImagePart``.
- Raises ``ModelError(retryable=False)`` when the part has no readable path."""
- if part.path:
- p = Path(part.path)
- if not p.is_file():
- raise ModelError(f"ImagePart.path does not exist: {part.path}")
- mime = part.mime
- if not mime or mime == "image/png" and p.suffix.lower() not in ("", ".png"):
- mime = mimetypes.guess_type(p.name)[0] or "image/png"
- return p.read_bytes(), mime
- raise ModelError("ImagePart has no path")
-
-
-def image_b64(part: ImagePart) -> tuple[str, str]:
- """``(base64_string, mime)`` for providers that want base64 (Anthropic, OpenAI data URLs)."""
- raw, mime = image_bytes(part)
- return base64.b64encode(raw).decode("ascii"), mime
-
-
-def message_blocks(msg: ChatMessage, *, text: Callable[[str], Any], image: Callable[[ImagePart], Any]) -> list[Any]:
- """One message's parts as a provider's blocks: empty text is dropped, and a labelled image
- is preceded by an ``[image: <label>]`` text block (the one place that text is written)."""
- out: list[Any] = []
- for p in msg.parts:
- if isinstance(p, TextPart):
- if p.text:
- out.append(text(p.text))
- elif isinstance(p, ImagePart):
- if p.label:
- out.append(text(f"[image: {p.label}]"))
- out.append(image(p))
- return out
-
-
-def default_timeout_s() -> float:
- """``Settings.model_timeout_s``."""
- return float(get_settings().model_timeout_s)
-
-
-class SdkModel:
- """The shell the SDK adapters (anthropic, openai) share: the constructor, ``id`` and the
- lazily built client. Each adapter owns ``_make_client`` and its own ``generate``."""
-
- provider = ""
-
- def __init__(
- self,
- model: str,
- *,
- timeout_s: float | None = None,
- max_attempts: int = 6,
- sleep: Callable[[float], None] = time.sleep,
- client: Any | None = None,
- ) -> None:
- self.model = model
- self.timeout_s = default_timeout_s() if timeout_s is None else timeout_s
- self.max_attempts = max(1, max_attempts)
- self._sleep = sleep
- self._client = client
- self._lock = threading.Lock()
-
- @property
- def id(self) -> str:
- return f"{self.provider}:{self.model}"
-
- def client(self) -> Any:
- with self._lock:
- if self._client is None:
- self._client = self._make_client()
- return self._client
-
- def _make_client(self) -> Any:
- raise NotImplementedError
-
-
-class Stopwatch:
- """Context-manager timer: ``with Stopwatch() as sw: ...`` leaves the elapsed milliseconds in ``sw.ms``."""
-
- def __enter__(self) -> Stopwatch:
- self._t0 = time.perf_counter()
- self.ms = 0
- return self
-
- def __exit__(self, *exc: object) -> None:
- self.ms = int((time.perf_counter() - self._t0) * 1000)
-
-
-def retryable_status(status: int | None) -> bool:
- """HTTP statuses worth retrying: 408, 409, 429 and every 5xx."""
- return status is not None and (status in (408, 409, 429) or status >= 500)
-
-
-def classify_sdk_exception(exc: BaseException, sdk: Any, label: str) -> Any:
- """The openai/anthropic SDK exception ladder → ``ModelError``.
-
- The message wording is load-bearing: eval/bench/_infra.py string-matches "request timed
- out" and "connection error" to tell a provider outage from a model failure.
- """
- if isinstance(exc, ModelError):
- return exc
- if isinstance(exc, sdk.APIStatusError):
- status = int(getattr(exc, "status_code", 0) or 0)
- return ModelError(f"{label} API error {status}: {exc.message}", retryable=retryable_status(status), status=status)
- if isinstance(exc, sdk.APITimeoutError):
- return ModelError(f"{label} request timed out: {exc}", retryable=True, status=408)
- if isinstance(exc, sdk.APIConnectionError):
- return ModelError(f"{label} connection error: {exc}", retryable=True)
- if isinstance(exc, sdk.APIError):
- return ModelError(f"{label} API error: {exc}", retryable=False)
- return ModelError(f"{label} unexpected error: {type(exc).__name__}: {exc}", retryable=False)
-
-
-#: floor of the per-attempt timeout, so a call near its deadline still gets one real attempt
-SDK_TIMEOUT_FLOOR_S = 20.0
-
-
-def retry_budget_s(max_wait_s: float | None) -> float:
- """The budget for ONE logical call: the caller clips ``RETRY_DEADLINE_S``, never extends it."""
- from c3d_flywheel.models.retry import RETRY_DEADLINE_S
-
- return RETRY_DEADLINE_S if max_wait_s is None else min(RETRY_DEADLINE_S, float(max_wait_s))
-
-
-def attempt_timeout_s(deadline: float, ceiling: float) -> float:
- """Per-attempt SDK timeout: time left to ``deadline``, capped at ``ceiling``, floored at
- :data:`SDK_TIMEOUT_FLOOR_S`. The retry loop checks the deadline only between attempts."""
- return min(ceiling, max(SDK_TIMEOUT_FLOOR_S, deadline - time.monotonic()))
-
-
-class FailedBills:
- """What the failed attempts of ONE logical call were billed for (``ModelError.usage``): it joins
- the call's single ledger row, on success and on a final failure alike."""
-
- def __init__(self) -> None:
- self.usage = Usage()
-
- def track[T](self, attempt: Callable[..., T]) -> Callable[..., T]:
- """``attempt`` that remembers the usage of every ``ModelError`` it raises."""
- def run(*args: Any) -> T:
- try:
- return attempt(*args)
- except ModelError as e:
- if e.usage is not None:
- self.usage = self.usage + e.usage
- raise
- return run
-
- @property
- def billed(self) -> bool:
- return bool(self.usage.input_tokens or self.usage.output_tokens)
-
- def join(self, resp: Any) -> Any:
- """``resp`` with the failed attempts' usage added."""
- return resp.model_copy(update={"usage": resp.usage + self.usage}) if self.billed else resp
-
- def charge(self, err: ModelError) -> None:
- """A finally-raised error carries ALL the usage billed (its own attempt included)."""
- if self.billed:
- err.usage = self.usage
-
-
-def retry_one_key(attempt: Any, *, label: str, classify: Any, attempts: int, sleep: Any,
- max_wait_s: float | None = None) -> Any:
- """The SDK adapters' retry loop: ``rotate_with_retries`` over a one-key pool, no cooldown or
- storm patience, inside the request's ``max_wait_s`` deadline."""
- from c3d_flywheel.models.retry import KeyPool, rotate_with_retries
-
- bills = FailedBills()
- try:
- resp = rotate_with_retries(KeyPool([label], cooldown_s=0.0), lambda _key: bills.track(attempt)(),
- classify=classify, max_attempts=attempts,
- max_total_s=retry_budget_s(max_wait_s), storm_attempts=0,
- sleep=sleep, label=label)
- except ModelError as e:
- bills.charge(e)
- raise
- return bills.join(resp)
diff --git a/flywheel/c3d_flywheel/models/registry.py b/flywheel/c3d_flywheel/models/registry.py
deleted file mode 100644
index 3ef10c71..00000000
--- a/flywheel/c3d_flywheel/models/registry.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""Model id parsing + factory. Ids: ``<provider>:<model>``.
-
-Every model handed out here is wrapped by ``c3d_flywheel.cost.instrument`` so each
-call lands in the cost ledger (the run's ``telemetry/cost.jsonl`` when a run is
-active, otherwise a per-process log) — always: the ledger is the only record of money."""
-
-from __future__ import annotations
-
-from functools import lru_cache
-from typing import Any
-
-from c3d_flywheel.models.base import ChatModel
-
-PROVIDERS = ("gemini", "anthropic", "openai")
-
-
-def parse_model_id(model_id: str) -> tuple[str, str]:
- if ":" not in model_id:
- raise ValueError(f"model id must be '<provider>:<model>', got {model_id!r}")
- provider, model = model_id.split(":", 1)
- if provider not in PROVIDERS:
- raise ValueError(f"unknown provider {provider!r}; known: {PROVIDERS}")
- if not model.strip():
- raise ValueError(f"model id {model_id!r} names no model")
- return provider, model
-
-
-@lru_cache(maxsize=32)
-def get_chat_model(model_id: str) -> ChatModel:
- """The (metered) ChatModel for ``<provider>:<model>``."""
- from c3d_flywheel.cost.instrument import metered_chat_model
-
- return metered_chat_model(build_chat_model(model_id))
-
-
-def build_chat_model(model_id: str, **kw: Any) -> ChatModel:
- """The bare (unmetered) model for ``<provider>:<model>``; ``kw`` goes to its constructor."""
- provider, model = parse_model_id(model_id)
- if provider == "gemini":
- from c3d_flywheel.models.gemini import GeminiModel
-
- return GeminiModel(model, **kw)
- if provider == "anthropic":
- from c3d_flywheel.models.anthropic import AnthropicModel
-
- return AnthropicModel(model, **kw)
- from c3d_flywheel.models.openai import OpenAIModel
-
- return OpenAIModel(model, **kw)
diff --git a/flywheel/c3d_flywheel/models/retry.py b/flywheel/c3d_flywheel/models/retry.py
deleted file mode 100644
index 45f991bd..00000000
--- a/flywheel/c3d_flywheel/models/retry.py
+++ /dev/null
@@ -1,497 +0,0 @@
-"""The scheduling machine under every model call.
-
-``KeyPool``: multi-key pool with 429 cooldown, dead-key benching, health scores and a
-machine-wide ``max_in_flight`` ceiling (``Slots``, flock'd lock files). ``rotate_with_retries``:
-the one retry loop (pool, ``RETRY_DEADLINE_S``); SDK adapters run it over a one-key pool.
-Every wait clips to ``MAX_WAIT_S``. ``report`` outcomes: ok | 429 | 5xx | error | dead | skip
-(``skip`` = a content failure the key did not cause). Thread-safe; ``clock`` / ``sleep`` injectable.
-"""
-
-from __future__ import annotations
-
-from codeverse3d.proc import Slots, SlotsBusy
-
-import fcntl
-import hashlib
-import logging
-import os
-import random
-import threading
-import time
-from collections.abc import Callable
-from dataclasses import dataclass
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
-
-Outcome = Literal["ok", "429", "5xx", "error", "dead", "skip"]
-
-
-class KeyPoolExhausted(SlotsBusy):
- """Every key is cooling down (or every slot is busy) and the wait budget ran out."""
-
-
-# message fragments of key-scoped failures; HTTP 401 / 403 are key-scoped regardless
-_DEAD_KEY_MARKERS = (
- "api_key_invalid",
- "api key not valid",
- "api key expired",
- "permission_denied",
- "unauthenticated",
- "suspended",
- "leaked",
- "service_disabled",
- "has not been used in project",
-)
-
-
-def is_dead_key_error(err: ModelError) -> bool:
- """True when ``err`` indicts the API key rather than the request."""
- if err.status in (401, 403):
- return True
- text = str(err).lower()
- return err.status == 400 and any(m in text for m in _DEAD_KEY_MARKERS)
-
-
-def failure_outcome(err: ModelError) -> Outcome:
- """``KeyPool.report`` outcome for a failed call; status-less content failures are ``skip``."""
- if err.status == 429:
- return "429"
- if (err.status or 0) >= 500:
- return "5xx"
- if is_dead_key_error(err):
- return "dead"
- return "error" if err.status else "skip"
-
-
-#: Longest single retry / cooldown wait (seconds): the pool rotates keys rather than sitting out
-#: a long backoff, so patience comes from the number of attempts, never from one sleep.
-MAX_WAIT_S = 3.0
-
-#: Default total wait budget of one :meth:`KeyPool.acquire` (seconds): the in-flight slot wait plus
-#: the key / cooldown wait; ``None`` waits forever.
-ACQUIRE_TIMEOUT_S = 120.0
-
-
-@dataclass
-class _KeyState:
- key: str
- health: float = 1.0
- cooldown_until: float = 0.0
- dead_until: float = 0.0
- n_ok: int = 0
- n_429: int = 0
- n_5xx: int = 0
- n_error: int = 0
- health_ts: float = 0.0
-
- def recover(self, now: float, rate: float = 0.01) -> None:
- """Passive health recovery: +``rate`` per idle second, capped at 1.0."""
- if now > self.health_ts:
- self.health = min(1.0, self.health + (now - self.health_ts) * rate)
- self.health_ts = now
-
-
-class KeyPool:
- """Round-robin key pool with cooldowns and an in-flight ceiling."""
-
- def __init__(
- self,
- keys: list[str] | tuple[str, ...],
- *,
- max_in_flight: int = 0,
- slots_dir: Path | str | None = None,
- cooldown_s: float = MAX_WAIT_S,
- dead_cooldown_s: float = 3600.0,
- clock: Callable[[], float] = time.monotonic,
- sleep: Callable[[float], None] = time.sleep,
- start: int = 0,
- ) -> None:
- """``start``: the key the round-robin begins at (``shared_pool`` draws it at random)."""
- uniq = [k for i, k in enumerate(keys) if k and k not in keys[:i]]
- if not uniq:
- raise ValueError("KeyPool needs at least one non-empty key")
- self._clock = clock
- self._sleep = sleep
- self._cooldown_s = float(cooldown_s)
- self._dead_cooldown_s = float(dead_cooldown_s)
- self._lock = threading.Lock()
- self._rr = start % len(uniq)
- self._in_flight = 0
- # machine-wide ceiling on concurrent model calls; 0 = unlimited
- self._max_in_flight = int(max_in_flight or 0)
- if self._max_in_flight and slots_dir is None:
- raise ValueError("max_in_flight needs slots_dir: the slots are lock files every process shares")
- #: the machine-wide slots (``None`` when uncapped)
- self.slots = Slots(self._max_in_flight, slots_dir) if self._max_in_flight else None
- self._held: list[int] = [] # slot descriptors of this process's in-flight calls
- now = clock()
- self._states = [_KeyState(key=k, health_ts=now) for k in uniq]
- self._by_key = {s.key: s for s in self._states}
-
- # ------------------------------------------------------------------ public
- def __len__(self) -> int:
- return len(self._states)
-
- def acquire(
- self,
- *,
- exclude: set[str] | frozenset[str] | None = None,
- timeout_s: float | None = ACQUIRE_TIMEOUT_S,
- ) -> str:
- """Next usable key, blocking up to ``timeout_s`` (slot wait + key wait) while all keys cool.
- ``exclude`` skips keys that already failed this call; raises ``KeyPoolExhausted`` at once
- when no key can become available before the deadline."""
- slot = None
- if self.slots is not None:
- t0 = self._clock()
- slot = self.slots.take(timeout_s)
- if timeout_s is not None:
- # the slot wait spent part of the budget; the key wait gets the rest
- timeout_s = max(0.0, timeout_s - (self._clock() - t0))
- try:
- key = self._acquire_key(exclude, timeout_s)
- except BaseException:
- if slot is not None:
- Slots.give(slot)
- raise
- self._hold(slot)
- return key
-
- def lease_session_key(self, leases: Path | str, *, timeout_s: float | None = ACQUIRE_TIMEOUT_S) -> tuple[str, int | None]:
- """A key for a vendor-CLI session: rotation, cooldowns and benching like :meth:`acquire`
- but no in-flight slot, and one live session per key machine-wide while keys suffice
- (a ``flock`` on ``<leases>/<key hash>.lock``). Returns ``(key, lease)``; close the lease
- with :meth:`end_lease`. When every ready key is in a session one is shared (lease ``None``)."""
- root = Path(leases)
- root.mkdir(parents=True, exist_ok=True)
- busy: set[str] = set()
- while True:
- try:
- key = self._acquire_key(busy, 0.0 if busy else timeout_s)
- except KeyPoolExhausted:
- if not busy:
- raise
- return self._acquire_key(None, timeout_s), None # every ready key is in a session
- fd = os.open(root / f"{hashlib.sha256(key.encode()).hexdigest()[:16]}.lock", os.O_RDWR | os.O_CREAT, 0o666)
- try:
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- except BlockingIOError:
- os.close(fd)
- busy.add(key)
- continue
- return key, fd
-
- @staticmethod
- def end_lease(lease: int | None) -> None:
- if lease is not None:
- os.close(lease)
-
- def _hold(self, slot: int | None) -> None:
- with self._lock:
- self._in_flight += 1
- if slot is not None:
- self._held.append(slot)
-
- def _acquire_key(self, exclude: set[str] | frozenset[str] | None, timeout_s: float | None) -> str:
- deadline = None if timeout_s is None else self._clock() + timeout_s
- while True:
- with self._lock:
- now = self._clock()
- chosen = self._pick(now, exclude or ())
- if chosen is not None:
- return chosen.key
- soonest = min(
- (max(now, s.cooldown_until) for s in self._states if s.key not in (exclude or ())),
- default=float("inf"),
- )
- if soonest == float("inf"):
- raise KeyPoolExhausted("all keys excluded")
- if deadline is not None and soonest > deadline:
- raise KeyPoolExhausted(
- f"all {len(self._states)} keys cooling or dead; the earliest becomes "
- f"available in {soonest - now:.0f}s (> {timeout_s}s wait budget)"
- )
- wait = max(0.01, min(soonest - now, MAX_WAIT_S))
- if deadline is not None and self._clock() + wait > deadline:
- raise KeyPoolExhausted(f"all {len(self._states)} keys cooling; waited {timeout_s}s")
- self._sleep(wait)
-
- def release(self) -> None:
- """Mark one :meth:`acquire`-d call finished (call in a ``finally``); a stray release is a no-op."""
- with self._lock:
- if self._in_flight <= 0:
- return
- self._in_flight -= 1
- slot = self._held.pop() if self._held else None
- if slot is not None:
- Slots.give(slot)
-
- def report(self, key: str, outcome: Outcome) -> None:
- """Feed back a call's result."""
- with self._lock:
- st = self._by_key.get(key)
- if st is None:
- return
- now = self._clock()
- st.recover(now)
- if outcome == "ok":
- st.n_ok += 1
- st.health = min(1.0, st.health + 0.2 * (1.0 - st.health))
- elif outcome == "429":
- st.n_429 += 1
- st.health *= 0.5
- # rotate to another key rather than sit out a long delay
- cool = min(self._cooldown_s, MAX_WAIT_S)
- st.cooldown_until = max(st.cooldown_until, now + cool)
- elif outcome == "5xx":
- st.n_5xx += 1
- st.health *= 0.8
- elif outcome == "dead":
- # key-scoped failure: bench until the long cooldown passes
- st.health = 0.0
- st.dead_until = max(st.dead_until, now + self._dead_cooldown_s)
- st.cooldown_until = max(st.cooldown_until, st.dead_until)
- elif outcome != "skip":
- st.n_error += 1
- st.health *= 0.9
-
- def stats(self) -> dict[str, int]:
- """The reported outcomes summed over the keys (what the concurrency probe reads)."""
- with self._lock:
- return {
- "ok": sum(s.n_ok for s in self._states),
- "429": sum(s.n_429 for s in self._states),
- "5xx": sum(s.n_5xx for s in self._states),
- "error": sum(s.n_error for s in self._states),
- }
-
- # ----------------------------------------------------------------- private
- def _pick(self, now: float, exclude) -> _KeyState | None:
- """Round-robin: the first healthy ready key, else the healthiest ready key."""
- n = len(self._states)
- best: _KeyState | None = None
- best_idx = -1
- for step in range(n):
- idx = (self._rr + step) % n
- st = self._states[idx]
- if st.key in exclude:
- continue
- st.recover(now)
- if st.cooldown_until > now:
- continue
- if st.health >= MIN_HEALTH:
- self._rr = (idx + 1) % n
- return st
- if best is None or st.health > best.health:
- best, best_idx = st, idx
- if best is not None:
- self._rr = (best_idx + 1) % n
- return best
-
-
-# ===================================================================== retry
-#: a key below this health is passed over while a healthier one is ready
-MIN_HEALTH = 0.3
-RETRY_DEADLINE_S = 1800.0 # ceiling of one logical call, retries included
-
-if TYPE_CHECKING: # pragma: no cover
- from c3d_flywheel.models.base import ModelError
-
-log = logging.getLogger(__name__)
-
-
-def cause_for(err: BaseException, exc: BaseException | None) -> BaseException | None:
- """The cause to raise ``err`` from, ``None`` when it would be ``err`` itself (a
- self-cause chain recurses forever in naive walkers)."""
- return None if exc is err else exc
-
-def backoff_delay(
- attempt: int, *, base_delay: float, max_delay: float, jitter: bool = True
-) -> float:
- """Delay before retry number ``attempt`` (1-based): ``base * 2**(attempt-1)`` capped,
- with full jitter (uniform in ``[delay/2, delay]``) when ``jitter`` is on."""
- raw = min(max_delay, base_delay * (2 ** max(0, attempt - 1)))
- if not jitter:
- return raw
- return random.uniform(raw / 2.0, raw)
-
-
-def rotate_with_retries[T](
- pool: KeyPool,
- call: Callable[[str], T],
- *,
- classify: Callable[[BaseException], ModelError],
- outcome_of: Callable[[ModelError], Outcome] = failure_outcome,
- max_attempts: int = 6,
- base_delay: float = 1.0,
- max_delay: float = MAX_WAIT_S,
- storm_attempts: int = 60,
- storm_max_delay: float = MAX_WAIT_S,
- max_total_s: float = RETRY_DEADLINE_S,
- sleep: Callable[[float], None] = time.sleep,
- monotonic: Callable[[], float] = time.monotonic,
- on_free_retry: Callable[[ModelError], bool] | None = None,
- label: str = "model",
- stats: dict[str, Any] | None = None,
-) -> T:
- """Run ``call(key)`` against a rotating :class:`KeyPool` until it succeeds.
-
- * success: report ok, bench keys that looked dead, return.
- * ``on_free_retry(err)`` true: retry at once, free (e.g. thinking-config downgrade).
- * dead key (auth failure): rotate free; benched only once a sibling key succeeds.
- * 429 / 503 / 529: the key joins the call's failed set; rotation to an untried key is
- free. Only when every key has failed within this call is a 503 a capacity storm:
- up to ``storm_attempts`` waits (capped at ``storm_max_delay``) outside ``max_attempts``.
- * other retryable errors: exponential backoff + jitter up to ``max_attempts``;
- non-retryable raise at once.
- * everything is bounded by ``max_total_s``.
-
- ``stats`` receives the ``attempts`` and ``storm`` counts.
- """
- failed_keys: set[str] = set() # keys that 429'd / 503'd or looked dead during this call
- dead_keys: set[str] = set()
- counts = {"attempts": 0, "storm": 0}
-
- def bench() -> None:
- # bench dead-looking keys once another key got past the auth layer
- for k in dead_keys:
- pool.report(k, "dead")
- dead_keys.clear()
-
- def run_one(key: str) -> _Try:
- """One round-trip on ``key``: report to the pool and release the slot."""
- counts["attempts"] += 1
- try:
- try:
- result = call(key)
- except Exception as exc: # noqa: BLE001 - classification is delegated
- err = classify(exc)
- outcome = outcome_of(err)
- # a dead-looking key is benched later, by bench()
- pool.report(key, "error" if outcome == "dead" else outcome)
- return _Try(key, None, err, exc, outcome)
- pool.report(key, "ok")
- return _Try(key, result, None, None, "ok")
- finally:
- pool.release()
-
- last_err: ModelError | None = None
- last_exc: BaseException | None = None
- attempt = 0
- storm = 0
- deadline = monotonic() + max_total_s if max_total_s > 0 else float("inf")
-
- def out_of_time() -> bool:
- return monotonic() >= deadline
-
- def clip(delay: float) -> float:
- """A sleep never outlives the budget."""
- return delay if deadline == float("inf") else min(delay, max(0.0, deadline - monotonic()))
-
- def gave_up(err: ModelError) -> ModelError:
- log.warning("%s giving up after %.0f s of retrying (%s)", label, max_total_s, err)
- return err
-
- try:
- while attempt < max_attempts:
- if out_of_time() and last_err is not None:
- # free rotations must not out-live the budget
- raise gave_up(last_err) from cause_for(last_err, last_exc)
- attempt += 1
- # skip dead-looking keys; skip throttled ones while an untried key remains
- exclude = dead_keys | (failed_keys if len(failed_keys) < len(pool) else set())
- budget_left = None if deadline == float("inf") else max(0.0, deadline - monotonic())
- try:
- key = pool.acquire(
- exclude=exclude,
- timeout_s=ACQUIRE_TIMEOUT_S if budget_left is None
- else min(ACQUIRE_TIMEOUT_S, budget_left),
- )
- except KeyPoolExhausted as exc:
- raise classify(exc) from exc
- t = run_one(key)
- if t.err is None:
- bench()
- return t.result
- err, exc = t.err, t.exc
- last_err = err
- last_exc = exc
- free = False
- if on_free_retry is not None and on_free_retry(err):
- free = True
- elif t.outcome == "dead":
- dead_keys.add(key)
- failed_keys.add(key)
- if free:
- attempt -= 1
- continue
- if t.outcome == "dead":
- if len(dead_keys) < len(pool):
- if out_of_time():
- raise gave_up(err) from cause_for(err, exc)
- log.warning("%s key …%s looks dead (%s); rotating", label, key[-4:], err)
- attempt -= 1
- continue
- raise err from cause_for(err, exc) # every key failed the same way: not the keys' fault
- if err.retryable and err.status in (503, 529) and storm < storm_attempts and not out_of_time():
- failed_keys.add(key)
- if len(failed_keys) < len(pool):
- # a 503 is per key: while an untried key remains rotation is free
- attempt -= 1
- log.warning("%s key …%s 503 (%s); rotating to a fresh key (%d/%d tried)",
- label, key[-4:], err, len(failed_keys), len(pool))
- continue
- # capacity storm: every key said 503, so wait (own budget, not max_attempts)
- storm += 1
- counts["storm"] = storm
- # jitter first, cap last, so the wait never exceeds the cap
- raw = base_delay * (2 ** min(storm, 8)) * (0.75 + 0.5 * random.random())
- delay = clip(min(storm_max_delay, raw))
- log.warning("%s capacity storm %d/%d (%s); waiting %.0fs",
- label, storm, storm_attempts, err, delay)
- sleep(delay)
- attempt -= 1
- continue
- if t.outcome == "429":
- failed_keys.add(key)
- if len(failed_keys) < len(pool):
- if out_of_time():
- raise gave_up(err) from cause_for(err, exc)
- attempt -= 1
- log.warning(
- "%s key …%s throttled (%s); rotating to a fresh key", label, key[-4:], err
- )
- sleep(clip(0.5))
- continue
- if not err.retryable or attempt >= max_attempts:
- bench()
- raise err from cause_for(err, exc)
- if out_of_time():
- raise gave_up(err) from cause_for(err, exc)
- delay = backoff_delay(attempt, base_delay=base_delay, max_delay=max_delay)
- log.warning(
- "%s attempt %d/%d failed (%s); retrying in %.1fs",
- label, attempt, max_attempts, err, delay,
- )
- sleep(clip(delay))
- assert last_err is not None
- bench()
- raise last_err
- except BaseException as exc:
- if hasattr(exc, "attempts"): # a ModelError: how hard the call tried
- exc.attempts = counts["attempts"]
- raise
- finally:
- if stats is not None:
- stats.update(counts)
-
-
-@dataclass(frozen=True)
-class _Try:
- """The outcome of one round-trip on one key (``rotate_with_retries.run_one``)."""
-
- key: str
- result: Any
- err: ModelError | None
- exc: BaseException | None
- outcome: str
diff --git a/flywheel/c3d_flywheel/models/schema_utils.py b/flywheel/c3d_flywheel/models/schema_utils.py
deleted file mode 100644
index e6e6e14a..00000000
--- a/flywheel/c3d_flywheel/models/schema_utils.py
+++ /dev/null
@@ -1,263 +0,0 @@
-"""JSON-schema adapters for the three providers, a fence-tolerant JSON parser and
-``ask_structured`` (one schema-validated call → object | None, usage, error).
-
-Pydantic schemas are rewritten per provider: Gemini takes an OpenAPI-ish subset, OpenAI strict
-needs closed objects with every property required, Anthropic takes inlined standard JSON schema.
-All functions are pure and return new dicts.
-"""
-
-from __future__ import annotations
-
-import copy
-import json
-import re
-from collections.abc import Callable, Sequence
-from typing import Any, Literal
-
-from pydantic import BaseModel, ValidationError
-
-from c3d_flywheel.models.base import ModelError
-from codeverse3d.contracts.chat import ChatMessage, ChatRequest, ImagePart
-from codeverse3d.contracts.common import Usage
-from codeverse3d.jsonparse import ( # noqa: F401
- JsonParseError,
- parse_json_lenient,
- strip_control_chars,
-)
-
-# keys Gemini's Schema type rejects or ignores
-_GEMINI_DROP = {
- "title",
- "default",
- "examples",
- "example",
- "$schema",
- "$id",
- "additionalProperties",
- "exclusiveMinimum",
- "exclusiveMaximum",
- "discriminator",
- "readOnly",
- "writeOnly",
- "deprecated",
- "uniqueItems",
- "multipleOf",
- "contentMediaType",
- "contentEncoding",
-}
-
-
-# ----------------------------------------------------------------- $ref inline
-def inline_refs(schema: dict[str, Any]) -> dict[str, Any]:
- """Copy with every local ``$ref`` replaced by its definition; recursive schemas raise ``ValueError``."""
- root = copy.deepcopy(schema)
- defs: dict[str, Any] = {}
- for table in ("$defs", "definitions"):
- defs.update(root.pop(table, {}) or {})
-
- def resolve(node: Any, stack: tuple[str, ...]) -> Any:
- if isinstance(node, list):
- return [resolve(n, stack) for n in node]
- if not isinstance(node, dict):
- return node
- if "$ref" in node:
- ref = node["$ref"]
- name = ref.rsplit("/", 1)[-1]
- if name not in defs:
- raise ValueError(f"unresolvable $ref {ref!r}")
- if name in stack:
- raise ValueError(f"recursive $ref {ref!r} cannot be inlined")
- merged = dict(defs[name])
- for k, v in node.items():
- if k != "$ref":
- merged[k] = v
- return resolve(merged, stack + (name,))
- return {k: resolve(v, stack) for k, v in node.items()}
-
- return resolve(root, ())
-
-
-# ------------------------------------------------------------------- helpers
-def _is_null(s: Any) -> bool:
- return (
- isinstance(s, dict) and s.get("type") == "null" and len([k for k in s if k != "type"]) == 0
- )
-
-
-def _merge_prefix_items(node: dict[str, Any]) -> dict[str, Any]:
- """``prefixItems`` becomes ``items`` (union if heterogeneous)."""
- items = node.pop("prefixItems")
- if not isinstance(items, list) or not items:
- return node
- uniq: list[dict[str, Any]] = []
- for it in items:
- if it not in uniq:
- uniq.append(it)
- node["items"] = uniq[0] if len(uniq) == 1 else {"anyOf": uniq}
- node.setdefault("minItems", len(items))
- node.setdefault("maxItems", len(items))
- return node
-
-
-# --------------------------------------------------------------------- gemini
-def to_gemini_schema(schema: dict[str, Any]) -> dict[str, Any]:
- """Sanitise a pydantic JSON schema for Gemini ``response_schema`` / function ``parameters``."""
- root = inline_refs(schema)
-
- def walk(node: Any) -> Any:
- if isinstance(node, list):
- return [walk(n) for n in node]
- if not isinstance(node, dict):
- return node
- node = {k: v for k, v in node.items() if k not in _GEMINI_DROP}
- if "const" in node:
- node["enum"] = [node.pop("const")]
- node.setdefault("type", _json_type(node["enum"][0]))
- if isinstance(node.get("type"), list):
- types = [t for t in node["type"] if t != "null"]
- if len(types) != len(node["type"]):
- node["nullable"] = True
- node["type"] = types[0] if len(types) == 1 else types
- for key in ("anyOf", "oneOf"):
- if key in node:
- variants = [v for v in node[key] if not _is_null(v)]
- if len(variants) != len(node[key]):
- node["nullable"] = True
- if len(variants) == 1:
- inner = walk(variants[0])
- merged = {**inner, **{k: v for k, v in node.items() if k != key}}
- node = merged
- else:
- node.pop(key)
- node["anyOf"] = [walk(v) for v in variants]
- if "allOf" in node and len(node["allOf"]) == 1:
- inner = walk(node.pop("allOf")[0])
- node = {**inner, **node}
- if "prefixItems" in node:
- node = _merge_prefix_items(node)
- # Gemini enums are strings only
- if "enum" in node:
- node["enum"] = [str(v) if not isinstance(v, str) else v for v in node["enum"]]
- node["type"] = "string"
- node.pop("format", None)
- if (
- "format" in node
- and node.get("type") == "string"
- and node["format"] not in ("date-time", "enum")
- ):
- node.pop("format")
- if "properties" in node:
- props = {k: walk(v) for k, v in node["properties"].items()}
- node["properties"] = props
- node.setdefault("type", "object")
- if props:
- node["propertyOrdering"] = list(props.keys())
- else:
- # Gemini rejects empty OBJECT properties
- node.pop("properties")
- node.pop("propertyOrdering", None)
- if "items" in node:
- node["items"] = walk(node["items"])
- return node
-
- return walk(root)
-
-
-def _json_type(value: Any) -> str:
- if isinstance(value, bool):
- return "boolean"
- if isinstance(value, int):
- return "integer"
- if isinstance(value, float):
- return "number"
- if isinstance(value, str):
- return "string"
- if isinstance(value, list):
- return "array"
- return "object"
-
-
-# ------------------------------------------------------------ openai / anthropic
-def _walk_schema(root: Any, on_object: Callable[[dict[str, Any], Callable[[Any], Any]], None]) -> Any:
- """Walk shared by the openai-strict and anthropic schemas; ``on_object(node, walk)`` rewrites ``properties``."""
-
- def walk(node: Any) -> Any:
- if isinstance(node, list):
- return [walk(n) for n in node]
- if not isinstance(node, dict):
- return node
- node = {
- k: v for k, v in node.items() if k not in ("title", "default", "examples", "example")
- }
- if "prefixItems" in node:
- node = _merge_prefix_items(node)
- if "properties" in node:
- on_object(node, walk)
- for key in ("anyOf", "oneOf", "allOf"):
- if key in node:
- node[key] = [walk(v) for v in node[key]]
- if "items" in node:
- node["items"] = walk(node["items"])
- return node
-
- return walk(root)
-
-
-def to_openai_strict_schema(schema: dict[str, Any]) -> dict[str, Any]:
- """OpenAI strict schema: closed objects, every property required. Null is allowed only
- where the source allows it; a non-null ``default`` becomes a ``[default: …]`` hint."""
-
- def strict_object(node: dict[str, Any], walk: Callable[[Any], Any]) -> None:
- props = {k: walk(v) for k, v in node["properties"].items()}
- required = set(node.get("required", []))
- for name, sub in props.items():
- original = node["properties"][name]
- if name in required or not isinstance(original, dict):
- continue
- if original.get("default") is not None:
- props[name] = _with_default_hint(sub, original["default"])
- node["properties"] = props
- node["required"] = list(props.keys())
- node["additionalProperties"] = False
- node.setdefault("type", "object")
-
- return _walk_schema(inline_refs(schema), strict_object)
-
-
-def _with_default_hint(sub: dict[str, Any], default: Any) -> dict[str, Any]:
-
- hint = f"[default: {json.dumps(default)}]"
- desc = str(sub.get("description") or "").strip()
- return {**sub, "description": f"{desc} {hint}".strip()}
-
-
-def to_anthropic_schema(schema: dict[str, Any]) -> dict[str, Any]:
- """Standard JSON schema, refs inlined, objects closed."""
-
- def closed_object(node: dict[str, Any], walk: Callable[[Any], Any]) -> None:
- node["properties"] = {k: walk(v) for k, v in node["properties"].items()}
- node.setdefault("type", "object")
- node.setdefault("additionalProperties", False)
-
- return _walk_schema(inline_refs(schema), closed_object)
-
-
-# ------------------------------------------------------------------ JSON parse
-
-
-def ask_structured(model: Any, schema: type[BaseModel], *, system: str, text: str,
- images: Sequence[ImagePart] = (), temperature: float,
- label: str, thinking: Literal["off", "low", "medium", "high"] = "low") -> tuple[Any, Usage, str]:
- """One structured call → ``(validated object | None, usage, error)``; failures are returned, not raised."""
- req = ChatRequest(messages=[ChatMessage.user(text, images=list(images) or None)], system=system,
- response_schema=schema.model_json_schema(), temperature=temperature,
- thinking=thinking, max_output_tokens=65_536, max_wait_s=900.0, label=label)
- try:
- resp = model.generate(req)
- except ModelError as e: # e.usage is what the provider ALREADY billed for the failure
- return None, e.usage, f"call failed: {e}"
- try:
- payload = resp.parsed if resp.parsed is not None else parse_json_lenient(resp.text)
- return schema.model_validate(payload), resp.usage, ""
- except (JsonParseError, ValidationError, TypeError) as e:
- return None, resp.usage, f"answer unparsable: {e}"
diff --git a/flywheel/c3d_flywheel/record/record.py b/flywheel/c3d_flywheel/record/record.py
index b98f6299..e037a3c5 100644
--- a/flywheel/c3d_flywheel/record/record.py
+++ b/flywheel/c3d_flywheel/record/record.py
@@ -1,30 +1,17 @@
-"""Run records: finalise / load / iterate.
-
-``finalize_record`` fills the environment, reads the run's total off its cost ledger, writes
-``record.json`` (telemetry included) atomically. Telemetry is best-effort and never fails a run.
-"""
+"""Reading the recorded runs: ``load_record`` / ``iter_runs`` and the derived per-round views."""
from __future__ import annotations
import logging
-import platform
-import subprocess
from collections.abc import Callable, Iterable, Iterator
-from contextlib import contextmanager
-from datetime import UTC, datetime
-from functools import lru_cache
from pathlib import Path
from typing import Any, NamedTuple
-from codeverse3d import __version__
-from codeverse3d.config import get_settings
from codeverse3d.contracts.artifacts import Judgment
-from codeverse3d.contracts.run import RoundRecord, RunId, RunRecord
-from c3d_flywheel.cost.ledger import existing_ledger_path, ledger_usage, load_ledger
-from codeverse3d.proc import read_json_or_none, version_line
-from codeverse3d.spatial.node import node_modules_dir, puppeteer_cache_dir
from codeverse3d.workspace import Workspace
+from c3d_flywheel.legacy import RoundRecord, RunId, RunRecord
+
log = logging.getLogger(__name__)
@@ -32,79 +19,6 @@ class RecordError(RuntimeError):
"""A record.json is missing or invalid."""
-# --------------------------------------------------------------------------- environment
-def _cmd_first_line(cmd: list[str]) -> str:
- rc, line = version_line(cmd)
- return line or f"exit {rc}"
-
-
-def _git(args: list[str], cwd: Path) -> subprocess.CompletedProcess[str] | None:
- try:
- return subprocess.run(["git", "-C", str(cwd), *args], capture_output=True, text=True, timeout=10, check=False)
- except (OSError, subprocess.TimeoutExpired):
- return None
-
-
-def _harness_git_sha(pkg_file: Path | None = None) -> str:
- """Commit of the harness checkout this package runs from (``""`` outside a checkout).
-
- The file itself must be tracked (a wheel or venv copy must not borrow an unrelated repo's
- sha); a modified tree is marked ``-dirty``.
- """
- here = (pkg_file or Path(__file__)).resolve()
- tracked = _git(["ls-files", "--error-unmatch", here.name], here.parent)
- if tracked is None or tracked.returncode != 0:
- return ""
- head = _git(["rev-parse", "HEAD"], here.parent)
- if head is None or head.returncode != 0 or not head.stdout.strip():
- return ""
- sha = head.stdout.strip()
- # dirty = any tracked file under the package tree (harness/) modified; untracked files ignored
- pkg_root = here.parents[2] if here.parent.name == "record" else here.parent
- status = _git(["status", "--porcelain", "--untracked-files=no", "--", str(pkg_root)], here.parent)
- if status is not None and status.returncode == 0 and status.stdout.strip():
- sha += "-dirty"
- return sha
-
-
-def _chrome_version() -> str:
- """Chrome build from the puppeteer cache directory name (offline, no launch)."""
- cache = puppeteer_cache_dir() / "chrome"
- if not cache.is_dir():
- return ""
- builds = sorted(p.name for p in cache.iterdir() if p.is_dir())
- return builds[-1] if builds else ""
-
-
-def _node_pkg_version(name: str) -> str:
- try:
- pkg = node_modules_dir() / name / "package.json"
- except RuntimeError: # no runtime_js: provenance only
- return ""
- return str((read_json_or_none(pkg) or {}).get("version", ""))
-
-
-@lru_cache(maxsize=1)
-def environment_versions() -> dict[str, str]:
- """Tool versions + harness provenance. Cached per process (Blender is slow to ask)."""
- s = get_settings()
- env: dict[str, str] = {
- "python": platform.python_version(),
- "codeverse3d": __version__,
- "platform": platform.platform(),
- "host": platform.node(),
- "harness_git_sha": _harness_git_sha(),
- }
- blender = s.resolve_blender()
- env["blender"] = _cmd_first_line([blender, "--version"]) if blender else ""
- env["blender_path"] = blender
- env["node"] = _cmd_first_line([s.binaries.node, "--version"])
- env["three"] = _node_pkg_version("three")
- env["puppeteer"] = _node_pkg_version("puppeteer")
- env["chrome"] = _chrome_version()
- return env
-
-
# --------------------------------------------------------------------------- derived fields
def effective_judgment(r: RoundRecord):
"""The round's judgment, or ``None`` when degraded (never a 0.0 score)."""
@@ -119,11 +33,9 @@ def previous_verdict(rounds: Iterable[RoundRecord | None], index: int) -> Judgme
def rubric_of(record: RunRecord) -> str:
- """The run's rubric: the stamped one, else its last judged round's, else ``judges.base.run_rubric``."""
- from codeverse3d.judges.base import run_rubric
-
+ """The run's rubric: the stamped one, else its last judged round's, else ``""``."""
judged = (r.judgment.rubric for r in reversed(record.rounds) if r.judgment is not None and r.judgment.rubric)
- return str(record.extra.get("rubric") or "") or next(judged, "") or run_rubric(record.spec)
+ return str(record.extra.get("rubric") or "") or next(judged, "")
def stop_reason(record: RunRecord) -> str:
@@ -169,90 +81,6 @@ def round_summary(r: RoundRecord) -> dict[str, Any]:
}
-def fill_derived(record: RunRecord) -> RunRecord:
- """Fill finished_at/environment when absent and the complexity block."""
- if record.finished_at is None:
- record.finished_at = datetime.now(UTC)
- env = dict(environment_versions())
- env.update(record.environment) # the track's own entries win
- record.environment = env
- block = complexity_block(record)
- if block is not None:
- record.extra["complexity"] = block
- return record
-
-
-def complexity_block(record: RunRecord) -> dict[str, Any] | None:
- """``record.extra["complexity"]``: the last measured round's vector, plus plan parts and the per-round trail."""
- cx = next((c for c in (round_complexity(r) for r in reversed(record.rounds)) if c), None)
- if cx is None:
- return None
- plan_parts = len(record.plan.parts) if record.plan is not None else 0
- block = dict(cx)
- block["plan_parts"] = plan_parts
- block["parts_per_plan_part"] = (
- round(int(cx.get("part_count") or 0) / plan_parts, 3) if plan_parts else None
- )
- trail = [c["index"] for c in (round_complexity(r) for r in record.rounds) if c]
- block["by_round"] = trail
- return block
-
-
-# --------------------------------------------------------------------------- io
-def package_run(ws: Workspace, record: RunRecord) -> None:
- """Read the run's total off its ledger (``record.total_usage`` = its sum at list price).
-
- Best-effort: a failure is logged and the run still gets its ``record.json``."""
- from c3d_flywheel.record.telemetry import build_telemetry
-
- if existing_ledger_path(ws.root) is not None:
- record.total_usage = ledger_usage(load_ledger(ws.root))
- try:
- ws.ensure_layout()
- except OSError as e: # read-only mount
- log.warning("run layout not created in %s: %s", ws.root, e)
- return
- try:
- record.telemetry = build_telemetry(ws, record)
- except Exception as e: # noqa: BLE001 - never fail a finished run over accounting
- log.warning("telemetry not written for %s: %s", ws.root, e)
-
-
-def repackage(ws: Workspace) -> RunRecord | None:
- """Re-package a finished run's ``record.json`` after its ledger grew; None when there is no ledger or record."""
- if existing_ledger_path(ws.root) is None:
- return None
- try:
- record = load_record(ws)
- except RecordError:
- return None
- package_run(ws, record)
- ws.write_json(ws.record_path, record)
- return record
-
-
-@contextmanager
-def join_post_run(ws: Workspace) -> Iterator[None]:
- """Work on a finished run that may spend money: rows join its ledger, and the record is
- re-packaged on the way out (even on error) so ``total_usage`` stays the ledger's sum."""
- from c3d_flywheel.cost.instrument import run_ledger
-
- try:
- with run_ledger(ws.root, run=ws.root.name, create=False):
- yield
- finally:
- repackage(ws)
-
-
-def finalize_record(ws: Workspace, record: RunRecord) -> Path:
- """Fill derived fields + environment, build the telemetry block and write ``record.json``."""
- record.workspace = record.workspace or str(ws.root)
- fill_derived(record)
- package_run(ws, record)
- ws.write_json(ws.record_path, record)
- return ws.record_path
-
-
def load_record(ws: Workspace | Path | str) -> RunRecord:
ws = ws if isinstance(ws, Workspace) else Workspace(ws)
if not ws.record_path.is_file():
diff --git a/flywheel/c3d_flywheel/record/telemetry.py b/flywheel/c3d_flywheel/record/telemetry.py
index 5ca20915..7b9108e8 100644
--- a/flywheel/c3d_flywheel/record/telemetry.py
+++ b/flywheel/c3d_flywheel/record/telemetry.py
@@ -1,102 +1,19 @@
-"""Run telemetry, stored in ``record.telemetry``: how the run was configured (the settings snapshot)
-and a summary of the ledger rows. Builders degrade to what could be read."""
+"""The cost summary of one run, from its ledger (``record.telemetry`` when the record lacks one)."""
from __future__ import annotations
-import hashlib
-import json
import logging
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.run import (
- CostSummary,
- RoleSettings,
- RunRecord,
- RunTelemetry,
- SettingsSnapshot,
- StageCost,
-)
+from codeverse3d.workspace import Workspace
+
from c3d_flywheel.cost.ledger import load_ledger, summarise
from c3d_flywheel.cost.types import STAGE_ORDER, CallCost
-from c3d_flywheel.record.record import effective_score, rubric_of
-from codeverse3d.workspace import Workspace
+from c3d_flywheel.legacy import CostSummary, RunRecord, RunTelemetry, StageCost
+from c3d_flywheel.record.record import effective_score
log = logging.getLogger(__name__)
-# --------------------------------------------------------------------------- settings snapshot
-def price_table_version() -> str:
- """Content hash of the USD price table."""
- from c3d_flywheel.models.pricing import PRICES
-
- payload = json.dumps(
- {f"{p}/{m}": [v.input, v.output, v.cached, v.cache_write, v.approximate] for (p, m), v in sorted(PRICES.items())},
- sort_keys=True,
- )
- return hashlib.sha256(payload.encode()).hexdigest()[:12]
-
-
-def backend_kind(model_id: str) -> str:
- """``gemini:x`` → ``gemini``."""
- return model_id.split(":", 1)[0] if ":" in model_id else model_id
-
-
-def rubric_hash(name: str) -> str:
- if not name:
- return ""
- try:
- from codeverse3d.judges.rubrics import load_rubric
-
- return load_rubric(name).content_hash()
- except Exception as e: # unknown rubric / package missing
- log.debug("rubric hash unavailable for %s: %s", name, e)
- return ""
-
-
-def _role_settings(record: RunRecord) -> list[RoleSettings]:
- """The model per role from the spec; sampling knobs from ``record.extra["sampling"]``, else left empty."""
- b = record.spec.backends
- sampling = record.extra.get("sampling") or {}
- roles: list[RoleSettings] = []
- for role, model_id in (("planner", b.planner), ("generator", b.generator), ("judge", b.judge),
- ("captioner", b.captioner)):
- knobs = sampling.get(role) or {}
- temperature = knobs.get("temperature")
- roles.append(RoleSettings(
- role=role, model=model_id, backend=backend_kind(model_id), thinking=str(knobs.get("thinking") or ""),
- temperature=float(temperature) if isinstance(temperature, (int, float)) else None,
- n_samples=knobs.get("n_samples") if role == "judge" else None, source="run" if knobs else "spec",
- ))
- return roles
-
-
-def settings_snapshot(record: RunRecord) -> SettingsSnapshot:
- """The resolved configuration block (no secrets)."""
- spec = record.spec
- rubric = rubric_of(record)
- s = get_settings()
- render = s.render.model_dump(mode="json")
- keys = len(s.gemini_api_keys)
- env = dict(record.environment)
- return SettingsSnapshot(
- roles=_role_settings(record),
- budget=spec.budget.model_dump(mode="json"),
- rubric=rubric,
- rubric_hash=rubric_hash(rubric),
- prompt_hashes=dict(record.prompt_hashes),
- tool_versions={k: v for k, v in env.items() if k in _TOOL_KEYS},
- harness_version=env.get("codeverse3d", ""),
- harness_git_sha=env.get("harness_git_sha", ""),
- key_pool_size=keys,
- price_table_version=price_table_version(),
- render=render,
- )
-
-
-_TOOL_KEYS = frozenset({"python", "platform", "host", "blender", "blender_path", "node", "three",
- "chrome", "puppeteer", "codeverse3d", "harness_git_sha"})
-
-
# --------------------------------------------------------------------------- the ledger (c3d_flywheel.cost)
def ledger_rows(ws: Workspace) -> list[CallCost]:
"""The run's priced per-call rows (none without a ledger)."""
@@ -130,7 +47,6 @@ def cost_summary(record: RunRecord, rows: list[CallCost]) -> CostSummary:
)
-# --------------------------------------------------------------------------- build
def build_telemetry(ws: Workspace, record: RunRecord) -> RunTelemetry:
- """Compute the settings snapshot and the cost summary of one run (``record.telemetry``)."""
- return RunTelemetry(settings=settings_snapshot(record), cost=cost_summary(record, ledger_rows(ws)))
+ """The cost summary of one run, computed from its ledger (no settings snapshot: that is only ever read)."""
+ return RunTelemetry(cost=cost_summary(record, ledger_rows(ws)))
diff --git a/flywheel/c3d_flywheel/skill_targets.py b/flywheel/c3d_flywheel/skill_targets.py
deleted file mode 100644
index f6d2bfbe..00000000
--- a/flywheel/c3d_flywheel/skill_targets.py
+++ /dev/null
@@ -1,392 +0,0 @@
-"""What each bundle CLAIMS to move — one deterministic quantity per skill."""
-
-from __future__ import annotations
-
-import importlib
-import re
-import tomllib
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-from codeverse3d.skills.model import Skill
-
-DOWN = "down"
-UP = "up"
-
-SRC_GATE = "gate_kinds"
-SRC_BUILD = "build"
-SRC_GLB = "glb"
-SRC_FRAMES = "frames"
-SRC_ARTIFACT = "artifact"
-
-
-@dataclass(frozen=True)
-class Target:
-
- skill: str
- metric: str
- direction: str
- unit: str
- source: str
- why: str
- kinds: tuple[str, ...] = ()
- severities: tuple[str, ...] = ()
- languages: tuple[str, ...] = field(default_factory=tuple)
- measurable: bool = True
- caveat: str = ""
-
-
-OBJECT_LANGS = ("blender", "urdf_blender", "cadquery", "threejs")
-
-TARGETS: tuple[Target, ...] = (
- Target(
- skill="c3d-part-contact",
- metric="interpenetrating_pairs",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("connectivity/penetration",),
- languages=OBJECT_LANGS,
- why="the penetrating-pair count is the strongest single predictor of assembly_fit (r = -0.258, eval/docs/COMPLEXITY.md §6)",
- ),
- Target(
- skill="c3d-bbox-contract",
- metric="contract_findings",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("contract/part_bbox", "contract/overall_bbox", "contract/footprint_offset",
- "contract/ground_gap"),
- languages=OBJECT_LANGS,
- why="the bundle is the contract gate's own tolerance arithmetic: a part that is there at the wrong size or place",
- caveat="contract/instance_* is c3d-repeats-and-mirrors; contract/missing_part is a language-API failure, not a tolerance one.",
- ),
- Target(
- skill="c3d-repeats-and-mirrors",
- metric="contract_instance_findings",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("contract/instance_bbox", "contract/instance_count"),
- languages=OBJECT_LANGS,
- why="metadata.owns already names these two kinds; they are the whole claim",
- ),
- Target(
- skill="c3d-blender-forms",
- metric="blender_lint_findings",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("lint/*",),
- languages=("blender", "urdf_blender"),
- why="the bundle is written around the five bpy traps the static lint already counts",
- ),
- Target(
- skill="c3d-urdf-joints",
- metric="joint_sweep_errors",
- direction=DOWN,
- unit="ERROR findings per run",
- source=SRC_GATE,
- kinds=("joint_sweep/penetration", "joint_sweep/unattached"),
- severities=("error",),
- languages=("urdf_blender",),
- why="653 of 691 joint_sweep findings in the corpus are moved-pose ERRORs, which the clearance rule targets",
- ),
- Target(
- skill="c3d-cadquery-forms",
- metric="build_failure_rate",
- direction=DOWN,
- unit="fraction of rounds whose build failed",
- source=SRC_BUILD,
- languages=("cadquery",),
- why="an OCC kernel refusal is a build failure, not a gate finding",
- caveat="no baseline exists: eval/bench/out holds zero graded cadquery runs.",
- ),
- Target(
- skill="c3d-threejs-forms",
- metric="missing_parts",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("contract/missing_part",),
- languages=("threejs",),
- why="the traps that make a threejs part silently ship nothing count as a missing part",
- caveat="no baseline exists: eval/bench/out holds zero graded threejs runs.",
- ),
- Target(
- skill="c3d-scene-composition",
- metric="camera_placement_findings",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("scene_frames/camera_*",),
- languages=("scene_threejs",),
- why="cameras below ground or inside geometry are arithmetic failures the bundle derives from measured bounds",
- caveat="frame coverage (content_small, hero_*) also routes the bundle (R15) but is not in this count.",
- ),
- Target(
- skill="c3d-scene-lighting",
- metric="dark_or_flat_frames",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("scene_frames/dark_frame", "scene_frames/flat_frame", "scene_frames/blown_frame"),
- languages=("scene_threejs",),
- why="the bundle is the exposure thresholds of frame_metrics.py with a recipe",
- ),
- Target(
- skill="c3d-scene-motion",
- metric="min_authored_changed_frac",
- direction=UP,
- unit="fraction of pixels changed on the WEAKEST authored camera, t=0 → t=1.5 s",
- source=SRC_FRAMES,
- languages=("scene_threejs",),
- measurable=False,
- why="reported as a regression guard only",
- caveat="NOT a target: no deterministic instrument attributes motion to a named plan item; a drop below MOVING_FRAC is still a regression.",
- ),
- Target(
- skill="c3d-threejs-shader-traps",
- metric="shader_preflight_findings",
- direction=DOWN,
- unit="WARN+ERROR findings per run",
- source=SRC_ARTIFACT,
- kinds=("shader_preflight/*",),
- languages=("scene_threejs", "threejs"),
- why="every trap the bundle teaches lands in the one shader_preflight report",
- caveat="zero headroom in today's corpus (all 4 recorded scene runs are clean): detects a regression only.",
- ),
- Target(
- skill="c3d-glsl-craft",
- metric="mean_edge_density",
- direction=UP,
- unit="mean edge density over the 5 sampled frames (frame_stats)",
- source=SRC_FRAMES,
- languages=("glsl_shader",),
- why="the gl_frames gate fired zero findings on 9 graded glsl runs; edge density is the continuous quantity it thresholds",
- ),
- Target(
- skill="c3d-opengl-pipeline",
- metric="gl_frame_findings",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("gl_frames/*",),
- languages=("opengl_python",),
- why="every gl_frames finding in the graphics corpus came from opengl_python",
- ),
- # ------------------------------------------------------------- recipe port
- Target(
- skill="c3d-scene-atmosphere",
- metric="env_bake_orientation_flags",
- direction=DOWN,
- unit="inverted bakes per run (prospective census)",
- source=SRC_ARTIFACT,
- languages=("scene_threejs",),
- measurable=False,
- why="the reference audit found the sky bake inverted in 4 of 9 delivered scenes",
- caveat="ported claim: reference-harness evidence, our own A/B pending.",
- ),
- Target(
- skill="c3d-scene-water",
- metric="metallic_water_planes",
- direction=DOWN,
- unit="water materials with metalness above 0.3 (prospective census)",
- source=SRC_ARTIFACT,
- languages=("scene_threejs",),
- measurable=False,
- why="dielectric water against a live environment bake is the reference's most-shipped material fix",
- caveat="NOT yet measurable here: judged quality only; a fresnel/metalness census is the candidate instrument.",
- ),
- Target(
- skill="c3d-scene-night",
- metric="overbright_shaft_findings",
- direction=DOWN,
- unit="solid-reading light shafts per run (prospective census)",
- source=SRC_ARTIFACT,
- languages=("scene_threejs",),
- measurable=False,
- why="night briefs are the dark_or_flat gate's richest source",
- caveat="ported claim: reference-harness evidence, our own A/B pending.",
- ),
- Target(
- skill="c3d-scene-materials",
- metric="flat_albedo_surfaces",
- direction=DOWN,
- unit="large surfaces with zero value variance (prospective census)",
- source=SRC_ARTIFACT,
- languages=("scene_threejs",),
- measurable=False,
- why="flat albedo and identical twins are the reference audit's most-cited surface defects",
- caveat="NOT yet measurable here: judged quality only; a per-material value-spread census is the candidate instrument.",
- ),
- # ------------------------------------------------------------- scene_blender
- Target(
- skill="c3d-bpy-scene-placement",
- metric="bpy_placement_errors",
- direction=DOWN,
- unit="ERROR findings per run",
- source=SRC_GATE,
- kinds=("scene_placement/floating", "scene_placement/sunken", "scene_placement/unsupported",
- "scene_placement/interpenetration"),
- severities=("error",),
- languages=("scene_blender",),
- why="the bundle seats every placed row on height_at or its support; the floating+sunken+interpenetration ERROR count is the number to move",
- caveat="unsupported, cantilevered and free_exempt route the bundle (R30) but are WARN-only or per-report, so they add nothing to the count.",
- ),
- Target(
- skill="c3d-bpy-scene-lighting",
- metric="bpy_exposure_findings",
- direction=DOWN,
- unit="findings per run",
- source=SRC_GATE,
- kinds=("scene_frames/dark_frame", "scene_frames/flat_frame", "scene_frames/blown_frame"),
- languages=("scene_blender",),
- why="the bundle is the exposure thresholds of frame_metrics.py with the Cycles/AgX bands",
- caveat="the same three kinds as c3d-scene-lighting over a disjoint language; the uniqueness test keys on (kind, language).",
- ),
- Target(
- skill="c3d-bpy-scene-materials",
- metric="flat_principled_materials",
- direction=DOWN,
- unit="materials whose Base Color, Roughness and Metallic are all unlinked constants (prospective census)",
- source=SRC_ARTIFACT,
- languages=("scene_blender",),
- measurable=False,
- why="placeholder_material, flat_ground and monotonous_vegetation are judge defects with no deterministic twin",
- caveat="NOT yet measurable here: judged quality only; a census of linked Principled sockets is the candidate instrument.",
- ),
-)
-
-BY_SKILL: dict[str, Target] = {t.skill: t for t in TARGETS}
-
-
-def target_for(skill: str) -> Target | None:
- return BY_SKILL.get(skill)
-
-
-# ===================================================================== claims
-# A claim pins a number some text states to the code constant that produces it (``_claims/<name>.toml``,
-# ``prompts/_claims.toml``); the text must stand as its own token and, with ``context``, share a line with that phrase.
-CLAIMS_DIR = "_claims"
-PROMPT_CLAIMS = "_claims.toml"
-
-
-def claims_path(skill: Skill | str, root: Path | None = None) -> Path:
- from codeverse3d.skills import skills_dir
-
- name = skill.name if isinstance(skill, Skill) else skill
- base = Path(root) if root is not None else skills_dir()
- if isinstance(skill, Skill):
- own = skill.dir.parent / CLAIMS_DIR / f"{name}.toml"
- if own.is_file():
- return own
- return base / CLAIMS_DIR / f"{name}.toml"
-
-
-def _load_rows(p: Path) -> list[dict[str, Any]]:
- if not p.is_file():
- return []
- rows = tomllib.loads(p.read_text()).get("claim") or []
- if not isinstance(rows, list):
- raise ValueError(f"{p}: [[claim]] must be an array of tables")
- return [dict(r) for r in rows]
-
-
-def load_claims(skill: Skill | str, root: Path | None = None) -> list[dict[str, Any]]:
- return _load_rows(claims_path(skill, root))
-
-
-def resolve(dotted: str) -> Any:
- mod, _, attr = dotted.partition(":") if ":" in dotted else dotted.rpartition(".")
- if not mod or not attr:
- raise ValueError(f"claim python target {dotted!r} must be 'module:NAME'")
- return getattr(importlib.import_module(mod), attr)
-
-
-def resolve_js(target: str) -> float:
- rel, _, name = target.rpartition(":")
- if not rel or not name:
- raise ValueError(f"claim js target {target!r} must be '<path>:NAME'")
- head, _, tail = rel.partition("/")
- if head == "runtime_js":
- from codeverse3d.spatial.node import runtime_js_dir
-
- path = runtime_js_dir() / tail
- else:
- path = Path(__file__).resolve().parents[1] / rel
- m = re.search(rf"^\s*(?:export\s+)?const\s+{re.escape(name)}\s*=\s*([-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)\s*[;,]",
- path.read_text(), re.M)
- if m is None:
- raise ValueError(f"{path} states no numeric `const {name} = …`")
- return float(m.group(1))
-
-
-def _target(row: dict[str, Any]) -> tuple[str, Any]:
- if row.get("js"):
- return f"js:{row['js']}", resolve_js(str(row["js"]))
- return str(row["python"]), resolve(str(row["python"]))
-
-
-def render_claim(row: dict[str, Any]) -> str:
- value = _target(row)[1]
- scale = row.get("scale")
- fmt = row.get("format")
- if row.get("sep"):
- items = [str(fmt or "{:g}").format(v * scale if scale is not None else v) for v in value]
- head, last = items[:-1], items[-1:]
- return str(row["sep"]).join(head) + (str(row.get("last_sep") or row["sep"]) if head else "") + "".join(last)
- if scale is not None:
- value = value * scale
- return format(value, "") if not fmt else str(fmt).format(value)
-
-
-def stated(text: str, doc: str, context: str = "") -> bool:
- pat = re.compile(rf"(?<![\w.]){re.escape(text)}(?!\w|\.\d)")
- return any(pat.search(line) and context in line for line in doc.splitlines())
-
-
-def check_rows(rows: list[dict[str, Any]], read: Any) -> list[str]:
- issues: list[str] = []
- for i, row in enumerate(rows):
- where = f"claim[{i}] {row.get('key', '?')}"
- missing = [k for k in ("key", "text") if not row.get(k)]
- if not row.get("python") and not row.get("js"):
- missing.append("python` or `js")
- if missing:
- issues += [f"{where}: missing `{k}`" for k in missing]
- continue
- try:
- live = render_claim(row)
- except Exception as e: # noqa: BLE001
- issues.append(f"{where}: cannot resolve {row.get('js') or row.get('python')!r}: {e}")
- continue
- try:
- doc = read(row.get("file"))
- except OSError as e:
- issues.append(f"{where}: cannot read {row.get('file')!r}: {e}")
- continue
- if live != row["text"]:
- issues.append(f"{where}: the constant now renders {live!r}, the text says {row['text']!r}")
- elif not stated(row["text"], doc, str(row.get("context", ""))):
- ctx = f" on a line with {row['context']!r}" if row.get("context") else ""
- issues.append(f"{where}: {row['text']!r} no longer appears{ctx} in {row.get('file') or 'the body'}")
- return issues
-
-
-def check_claims(skill: Skill, root: Path | None = None) -> list[str]:
- try:
- rows = load_claims(skill, root)
- except (ValueError, OSError) as e:
- return [f"claims file unreadable: {e}"]
- return check_rows(rows, lambda f: skill.body if not f else (skill.dir / f).read_text())
-
-
-def check_prompt_claims(root: Path | None = None) -> list[str]:
- base = Path(root) if root is not None else Path(__file__).resolve().parents[1] / "prompts"
- try:
- rows = _load_rows(base / PROMPT_CLAIMS)
- except (ValueError, OSError) as e:
- return [f"prompt claims file unreadable: {e}"]
- return check_rows(rows, lambda f: (base / f).read_text() if f else "")
diff --git a/flywheel/c3d_flywheel/vlm_judge.py b/flywheel/c3d_flywheel/vlm_judge.py
deleted file mode 100644
index c8cf8e2d..00000000
--- a/flywheel/c3d_flywheel/vlm_judge.py
+++ /dev/null
@@ -1,405 +0,0 @@
-"""``VlmJudge``: rubric YAML + labelled renders + measurements → ``Judgment``.
-
-Per sample, one ``ChatRequest`` with a per-rubric JSON schema; samples differ by montage order so the
-n-sample std measures judge noise. Score, penalties, floors, caps and pass are computed in code
-(``rubrics.aggregate_samples``). Retries stay inside one ``sample_budget_s``; if no sample succeeds a
-*degraded* Judgment (``raw.status='degraded'``) is returned, which the orchestrator treats as a glitch.
-"""
-
-from __future__ import annotations
-
-import contextlib
-import logging
-import shutil
-import time
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Literal
-
-from codeverse3d.config import get_settings
-from codeverse3d.contracts.artifacts import Judgment
-from codeverse3d.contracts.chat import ChatRequest, ChatResponse
-from codeverse3d.contracts.common import TRACK_INFO, Track, Usage
-from codeverse3d.judges.base import SLICE_TRACKS, JudgeInput
-from codeverse3d.judges.prompt_builder import (
- _key,
- _stat,
- build_judge_messages,
- connectivity_error_pairs,
- default_cache_dir,
- judge_prompt_hash,
-)
-from codeverse3d.judges.rubrics import (
- JudgeOutput,
- JudgeParseError,
- Rubric,
- aggregate_samples,
- degraded_judgment,
- load_rubric,
- parse_judge_output,
- wire_schema,
-)
-from c3d_flywheel.models import get_chat_model
-from c3d_flywheel.models.base import ChatModel, ModelError
-from codeverse3d.proc import fan_out, unique_tmp
-from codeverse3d.reference import conflict_note, dimension_conflict
-from codeverse3d.spatial.silhouette import (
- CompareFn,
- ReferenceMatch,
- compare_silhouette,
- measure_reference,
-)
-
-log = logging.getLogger(__name__)
-
-#: retry budget for ONE judge sample, every attempt included
-SAMPLE_BUDGET_S = 900.0
-SAMPLE_MIN_WAIT_S = 20.0
-
-SLICE_LABELS = {
- "front_back": "cross-section front-back · interior view",
- "left_right": "cross-section left-right · interior view",
-}
-
-THINKING_LEVELS: tuple[str, ...] = ("off", "low", "medium", "high")
-
-#: per-track montage ceilings under the ``Settings.judge.montages`` dial
-TRACK_MONTAGE_CAP: dict[str, int] = {"scene": 4, "graphics": 3}
-
-
-@dataclass
-class JudgeContext:
-
- measured_scores: dict[str, float] = field(default_factory=dict)
- extra_text: str = ""
- extra_images: list[tuple[str, str]] = field(default_factory=list)
- #: what building the context cost; folded into the verdict so budgets see it
- usage: Usage = field(default_factory=Usage)
-
-
-class VlmJudge:
-
- name = "vlm"
-
- def __init__(
- self,
- rubric: str | Rubric = "static_object",
- model_id: str | None = None,
- n_samples: int = 1,
- temperature: float = 0.2,
- *,
- thinking: Literal["off", "low", "medium", "high"] | None = None,
- max_attempts: int = 3,
- chat_model: ChatModel | None = None,
- cache_dir: Path | None = None,
- label: str = "judge",
- sample_budget_s: float | None = None,
- fixed_order: bool = False,
- ):
- self.rubric: Rubric = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)
- # fixed_order sends the identical prompt n times (re-judge sigma) instead of shuffling view order
- self.fixed_order = bool(fixed_order)
- self.model_id = model_id or get_settings().default_judge
- self.n_samples = max(1, int(n_samples))
- if self.n_samples % 2 == 0:
- log.warning("judge n_samples=%d is even: an exact vote tie on a defect reads as absent and a tie on an "
- "acceptance item follows the representative sample (rubrics.aggregate_samples); an odd n "
- "gives a true majority", self.n_samples)
- self.temperature = temperature
- jd = get_settings().judge
- # ``thinking=None`` reads ``Settings.judge.thinking``
- level = str(jd.thinking if thinking is None else thinking)
- if level not in THINKING_LEVELS:
- raise ValueError(f"judge thinking must be one of {' | '.join(THINKING_LEVELS)}, not {level!r}")
- self.thinking: str = level
- self.max_attempts = max(1, int(max_attempts))
- self.sample_budget_s = SAMPLE_BUDGET_S if sample_budget_s is None else float(sample_budget_s)
- self.max_montages = max(1, int(jd.montages))
- self.detail_crops = max(0, int(jd.detail_crops))
- self.max_px = int(jd.max_px)
- self._model = chat_model
- self.cache_dir = cache_dir
- self.label = label
-
- def _montages_for(self, inp: JudgeInput) -> int:
- """Per-track montage cap (``TRACK_MONTAGE_CAP``); the object rig takes the dial."""
- cap = TRACK_MONTAGE_CAP.get(inp.spec.track.value)
- if cap is None:
- return self.max_montages
- return min(self.max_montages, cap)
-
- @property
- def prompt_hash(self) -> str:
- return judge_prompt_hash(self.rubric)
-
- @property
- def model(self) -> ChatModel:
- if self._model is None:
- self._model = get_chat_model(self.model_id)
- return self._model
-
- def judge(self, inp: JudgeInput) -> Judgment:
- """Judge one round; clay/normals views in ``inp.renders`` are routed to the geometry montage."""
- geometry_views = inp.geometry_views
- acceptance_ids = [a.id for a in inp.acceptance]
- schema = wire_schema(self.rubric, acceptance_ids)
- ctx = self.context(inp)
- measured, extra_text, extra_images = ctx.measured_scores, ctx.extra_text, ctx.extra_images
- usage = ctx.usage
- slice_images, elicit = self.slice_payload(inp)
- samples: list[JudgeOutput] = []
- errors: list[str] = []
- reqs: list[ChatRequest] = []
- for k in range(self.n_samples):
- seed = None if (self.fixed_order or (self.n_samples == 1 and k == 0)) else (inp.round_index * 1000 + k)
- system, messages = build_judge_messages(
- inp, self.rubric, shuffle_seed=seed, geometry_views=geometry_views, max_montages=self._montages_for(inp),
- detail_crops=self.detail_crops, max_px=self.max_px, cache_dir=self.cache_dir,
- extra_images=extra_images, extra_text=extra_text,
- slice_images=slice_images, provenance_elicitation=elicit,
- )
- reqs.append(ChatRequest(
- messages=messages, system=system, response_schema=schema, temperature=self.temperature,
- thinking=self.thinking, # type: ignore[arg-type]
- # fan_out copies contextvars into every sample thread; the label adds what only this call knows
- label=f"{self.label}:{self.rubric.name}:r{inp.round_index:02d}:s{k}",
- ))
- if len(reqs) == 1:
- outcomes = [self._sample(reqs[0], acceptance_ids, measured)]
- else:
- self.model # noqa: B018
- results = fan_out(
- reqs, lambda r: self._sample(r, acceptance_ids, measured),
- max_workers=len(reqs), label=f"{self.label}:samples", item_name=lambda r: r.label,
- )
- outcomes = [
- r if not isinstance(r, Exception) else (None, Usage(), f"{type(r).__name__}: {r}")
- for r in results
- ]
- for out, u, err in outcomes:
- usage = usage + u
- if out is not None:
- samples.append(out)
- else:
- errors.append(err)
- if not samples:
- return degraded_judgment(
- self.rubric, " || ".join(errors)[:2000], usage=usage, judge_backend=self.model_id, n_requested=self.n_samples,
- judge_prompt_hash=self.prompt_hash,
- )
- return aggregate_samples(
- self.rubric, samples, gates=inp.gates, acceptance_items=inp.acceptance,
- console_errors=inp.renders.console_errors, views=list(inp.renders.views), usage=usage,
- judge_backend=self.model_id, n_requested=self.n_samples, sample_errors=errors,
- judge_prompt_hash=self.prompt_hash,
- )
-
- def context(self, inp: JudgeInput) -> JudgeContext:
- """Per-call extras: measured-criterion scores, extra text and extra images (none in the base judge)."""
- if self.rubric.measured_criteria():
- raise ValueError(
- f"rubric {self.rubric.name} has measured criteria "
- f"{[c.id for c in self.rubric.measured_criteria()]} but VlmJudge computes none; use a specialised judge"
- )
- return JudgeContext()
-
- def slice_payload(self, inp: JudgeInput) -> tuple[list[tuple[str, str]], bool]:
- """``([(label, png_path)…], provenance_elicitation)`` for the slice channel.
-
- Fires only when ``Settings.judge.slices == "on-error"``, the track is an object track, ``inp.glb_path``
- exists and the connectivity gate has an ERROR; otherwise ``([], False)``. Never raises.
- """
- if get_settings().judge.slices != "on-error":
- return [], False
- if inp.spec.track.value not in SLICE_TRACKS or not inp.glb_path:
- return [], False
- glb = Path(inp.glb_path)
- if not glb.is_file():
- return [], False
- if not any(g.gate == "connectivity" and g.errors for g in inp.gates):
- return [], False
- pairs = connectivity_error_pairs(inp.gates)
- try:
- out_dir, manifest = self._render_slices(glb, pairs)
- except ImportError as e:
- log.warning("judge slices disabled: %s (install the 'mesh' extra)", e)
- return [], False
- except Exception as e: # noqa: BLE001
- log.warning("judge slice render failed for %s: %s", glb, e)
- return [], True
- return [(SLICE_LABELS[s.name], str(out_dir / s.png)) for s in manifest.rendered()], True
-
- def _render_slices(self, glb: Path, pairs: list[tuple[str, str]]):
- """Render (or reuse) the slice set for ``glb``, cached by path + mtime + size + error pairs."""
- from codeverse3d.spatial.sections import SliceManifest, judge_slices
-
- cache = Path(self.cache_dir) if self.cache_dir else default_cache_dir()
- out_dir = cache / f"slices_{_key([_stat(str(glb)), str(pairs)])}"
- mpath = out_dir / "manifest.json"
- if not mpath.is_file():
- tmp = unique_tmp(out_dir)
- shutil.rmtree(tmp, ignore_errors=True)
- try:
- judge_slices(glb, pairs, tmp)
- with contextlib.suppress(OSError):
- tmp.rename(out_dir)
- finally:
- shutil.rmtree(tmp, ignore_errors=True)
- return out_dir, SliceManifest.model_validate_json(mpath.read_text())
-
- def _sample(
- self, req: ChatRequest, acceptance_ids: list[str], measured: dict[str, float]
- ) -> tuple[JudgeOutput | None, Usage, str]:
- usage = Usage()
- last = ""
- t_start = time.monotonic()
- for attempt in range(1, self.max_attempts + 1):
- remaining = self.sample_budget_s - (time.monotonic() - t_start)
- if remaining <= 0:
- last = (f"sample budget of {self.sample_budget_s:.0f}s spent after {attempt - 1} attempt(s)"
- + (f"; last: {last}" if last else ""))
- log.warning("judge %s: %s", req.label, last[:300])
- break
- req = req.model_copy(update={"max_wait_s": max(SAMPLE_MIN_WAIT_S, remaining)})
- t0 = time.monotonic()
- try:
- resp: ChatResponse = self.model.generate(req)
- except ModelError as e:
- usage = usage + (getattr(e, "usage", None) or Usage())
- last = f"ModelError(attempt {attempt}): {e}"
- log.warning("judge %s: %s", req.label, last)
- if not e.retryable:
- break
- continue
- usage = usage + resp.usage
- payload = resp.parsed if resp.parsed is not None else resp.text
- try:
- out = parse_judge_output(payload, self.rubric, acceptance_ids, measured_scores=measured)
- log.debug("judge %s ok in %.1fs", req.label, time.monotonic() - t0)
- return out, usage, ""
- except JudgeParseError as e:
- last = f"parse(attempt {attempt}): {e}"
- log.warning("judge %s: %s", req.label, last[:300])
- req = req.model_copy(update={"temperature": min(1.0, self.temperature + 0.1 * attempt)})
- return None, usage, last or "unknown judge failure"
-
-
-
-#: IoU → score: ≤ 0.25 → 0, ≥ 0.85 → 1, linear between
-IOU_LOW, IOU_HIGH = 0.25, 0.85
-
-#: score the measured criterion gets when the measurement cannot be trusted
-NEUTRAL_SCORE = 0.5
-
-def iou_to_score(iou: float) -> float:
- return max(0.0, min(1.0, (float(iou) - IOU_LOW) / (IOU_HIGH - IOU_LOW)))
-
-
-class ReferenceJudge(VlmJudge):
-
- name = "reference"
-
- def __init__(
- self,
- model_id: str | None = None,
- n_samples: int = 1,
- temperature: float = 0.2,
- *,
- rubric: str | Rubric = "reference",
- silhouette_fn: CompareFn | None = None,
- **kwargs: Any,
- ):
- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
- self.silhouette_fn = silhouette_fn or compare_silhouette
-
- def context(self, inp: JudgeInput) -> JudgeContext:
- refs = [r for r in inp.spec.references if Path(r.path).is_file()]
- images: list[tuple[str, str]] = []
- for i, r in enumerate(refs[:3], 1):
- note = f" — {r.note}" if r.note else ""
- images.append((f"REFERENCE {i}/{min(len(refs), 3)} ({r.role}){note}", r.path))
- blocks: list[str] = []
- conflict = self.dimension_conflict(inp, refs)
- note = conflict_note(conflict)
- if note:
- blocks.append(note)
- measured_ids = [c.id for c in self.rubric.measured_criteria()]
- measured_scores: dict[str, float] = {}
- match = measure_reference(inp.renders.views, inp.spec, compare=self.silhouette_fn, conflict=conflict) if refs else None
- if measured_ids:
- if match is None:
- raise ReferenceJudgeError(f"cannot score measured criteria {measured_ids}: "
- + ("no render view compared" if refs else "spec has no readable reference images"))
- score, why = self.silhouette_score(match)
- measured_scores = {cid: score for cid in measured_ids}
- extra = match.model_dump(exclude={"iou", "reference", "conflict", "per_view"}, exclude_none=True)
- blocks.append(
- f"MEASURED SILHOUETTE (harness): {match.view} render vs reference {Path(match.reference).name}: "
- f"IoU {match.iou:.3f} → silhouette_match score {score:.2f}{why}. extra: {extra}")
- return JudgeContext(measured_scores=measured_scores, extra_text="\n\n".join(blocks), extra_images=images)
-
- def dimension_conflict(self, inp: JudgeInput, refs: list[Any]) -> dict[str, Any]:
- """Does the target reference's own outline contradict the brief's dimensions? If so the brief wins."""
- targets = [r for r in refs if r.role == "target"] or refs
- if not targets:
- return {"conflict": False}
- try:
- return dimension_conflict(inp.spec, targets[0].path)
- except Exception as e: # noqa: BLE001
- log.warning("dimension-conflict check failed: %s", e)
- return {"conflict": False, "error": str(e)}
-
- def silhouette_score(self, match: ReferenceMatch) -> tuple[float, str]:
- """IoU → measured score, neutral when the reference contradicts the brief or the mask is unreliable."""
- if match.conflict:
- return NEUTRAL_SCORE, " (NEUTRAL: the reference's proportions contradict the brief's dimensions)"
- if not match.reliable:
- return NEUTRAL_SCORE, " (NEUTRAL: the background mask was unreliable)"
- return iou_to_score(match.iou), ""
-
-
-LIKENESS_NOTE = (
- "REAL-WORLD REFERENCE PHOTOS are attached (labelled REAL-WORLD REFERENCE n/N). They are not a "
- "composition to copy; they are what the thing the brief names ACTUALLY looks like. Score "
- "brief fidelity, colour/light and richness against them: dominant colour and where the "
- "secondary colours sit, how the structure folds / layers / thins out, where the brightness "
- "concentrates and how much of the frame stays dark, what the real texture is at fine scale. A "
- "frame that ticks every noun in the brief but would never be mistaken for the photographed "
- "thing (a row of evenly spaced bars standing in for curtains, a flat band standing in for a "
- "gradient of light, cartoon-saturated colour where the photo is subtle) is a 0.4-0.5 on brief "
- "fidelity, not a 1.0. Foreground, framing and landscape in the photos are incidental unless "
- "the brief asks for them."
-)
-
-
-class LikenessJudge(VlmJudge):
- """``VlmJudge`` + reference photos, for tracks with no silhouette: the photos are attached beside the
- frames with :data:`LIKENESS_NOTE` and the rubric is unchanged."""
-
- name = "likeness"
-
- def __init__(self, model_id: str | None = None, n_samples: int = 1, temperature: float = 0.2, *,
- rubric: str | Rubric = "shader", **kwargs: Any):
- super().__init__(rubric, model_id, n_samples, temperature, **kwargs)
-
- def context(self, inp: JudgeInput) -> JudgeContext:
- if self.rubric.measured_criteria():
- raise ValueError(f"rubric {self.rubric.name} has measured criteria; LikenessJudge measures nothing")
- refs = [r for r in inp.spec.references if Path(r.path).is_file()][:3]
- images = [(f"REAL-WORLD REFERENCE {i}/{len(refs)}" + (f" — {r.note}" if r.note else ""), r.path)
- for i, r in enumerate(refs, 1)]
- return JudgeContext(extra_text=LIKENESS_NOTE if refs else "", extra_images=images)
-
-
-class ReferenceJudgeError(RuntimeError):
- """The reference judge cannot compute its measured criteria."""
-
-
-def judge_for(track: Track, rubric: str | Rubric, *, references: bool) -> type[VlmJudge]:
- """The judge class that scores a round, shared by the live run and every replay: :class:`ReferenceJudge`
- for measured criteria or object-track references, :class:`LikenessJudge` for graphics/scene references,
- else :class:`VlmJudge`."""
- rub = rubric if isinstance(rubric, Rubric) else load_rubric(rubric)
- if rub.measured_criteria() or (references and not TRACK_INFO[track].likeness_refs):
- return ReferenceJudge
- return LikenessJudge if references else VlmJudge