Official Codebase of 3DCodeVerse
Python
delete what nothing calls: the trajectory miner, the unwired cost guard, the one-off time audit
gaoypeng2bc4188eee79
diff --git a/harness/README.md b/harness/README.mdindex c714a5ca..a4c3bd7b 100644--- a/harness/README.md+++ b/harness/README.md@@ -8,10 +8,10 @@ renders, judges, refines, textures and records every run as data-flywheel materi* **Languages:** `blender` · `cadquery` · `threejs` · `urdf_blender` ·`scene_threejs` · `glsl_shader` · `opengl_python`* **Backends:** Gemini / Anthropic / OpenAI chat models; coding agents `gemini-cli`,- `claude-code`, `codex`, `agy` (Antigravity), the in-process `api-agent` tool loop, and+ `claude-code`, `codex`, `agy` (Antigravity) — always a vendor's CLI — and`single-shot` generation on any chat model.-* **Spatial tools** (direct, MCP server `3dcv` for agentic CLIs, native tools for- api-agent): build, measure, render_views/sheet, isolate, cross_section,+* **Spatial tools** (direct, or over the MCP server `3dcv` for the agentic CLIs):+ build, measure, render_views/sheet, isolate, cross_section,check_connectivity, check_contract, compare_silhouette, joint_sweep, shader_probe,scene_probe, scene_views, gl_probe, gl_frames, texture_pass, texture_preview,read_cookbook.diff --git a/harness/bench/time_audit/calls.py b/harness/bench/time_audit/calls.pydeleted file mode 100644index 8adaf6b5..00000000--- a/harness/bench/time_audit/calls.py+++ /dev/null@@ -1,128 +0,0 @@-"""Task 2: per-model-call latency distributions (baseline vs storm day), calls per-run, calls with >= 1 retry, and total retry-wait per run; plus the worker.log storm-lines for the fancy_v1 cells (the only corpus whose driver keeps a per-cell log)."""-from __future__ import annotations--import re-import sys-from collections import Counter, defaultdict-from pathlib import Path--sys.path.insert(0, str(Path(__file__).parent))-from corpus import RunRef, cost_rows, discover, fmt, med_p90, trajectories # noqa: E402-from turns import parse_turns # noqa: E402--RETRY_THRESHOLD_S = 1.0 # min storm sleep is 1.5 s (2^1 * 0.75); 429 rotation pause 0.5 s-STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")-FAIL_RE = re.compile(r"gemini (\S+) attempt (\d)/6 failed \((.*?)\); retrying in ([\d.]+)s")-GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")---def pct(xs: list[float]) -> str:- if not xs:- return "-"- s = sorted(xs)- q = lambda p: s[min(len(s) - 1, int(round(p * (len(s) - 1))))] # noqa: E731- return f"n={len(s)} p50={q(.5):.1f} p90={q(.9):.1f} p99={q(.99):.1f} max={q(1):.0f}"---def latency_table(refs: list[RunRef]) -> None:- lat: dict[tuple[str, str, str], list[float]] = defaultdict(list)- per_run: dict[tuple[str, str], list[int]] = defaultdict(list)- err: Counter[tuple[str, str]] = Counter()- for ref in refs:- n: Counter[str] = Counter()- for r in cost_rows(ref):- key = (ref.bucket, r["model"], r["role"])- if r["outcome"] == "ok":- lat[key].append(r["latency_ms"] / 1000)- n[r["model"]] += 1- else:- err[(ref.bucket, r["model"])] += 1- for m, c in n.items():- per_run[(ref.bucket, m)].append(c)- print("## successful-call latency (s) by corpus x model x role")- for k in sorted(lat):- print(f" {k[0]:14s} {k[1]:24s} {k[2]:9s} {pct(lat[k])}")- print("## ok calls per run (median / p90) and error rows")- for k in sorted(per_run):- m, p, cnt = med_p90([float(x) for x in per_run[k]])- print(f" {k[0]:14s} {k[1]:24s} runs={cnt} calls/run {fmt(m)}/{fmt(p)} error rows={err[k]}")---def retry_table(refs: list[RunRef]) -> None:- print("## generator turns (transcripts): waiting = wall - recorded latency")- agg: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))- for ref in refs:- turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]- if not turns:- continue- real = [t for t in turns if t.latency_s > 0]- retried = [t for t in real if t.wait_s > RETRY_THRESHOLD_S]- a = agg[(ref.bucket, ref.track)]- a["calls"].append(len(real))- a["retried"].append(len(retried))- a["retried_pct"].append(100.0 * len(retried) / max(1, len(real)))- a["wait_s"].append(sum(t.wait_s for t in real) + sum(t.error_s for t in turns))- a["wait_per_retried"].extend(t.wait_s for t in retried)- a["errors"].append(sum(1 for t in turns if t.error_s > 0))- for k in sorted(agg):- a = agg[k]- c, cp, n = med_p90(a["calls"])- r, rp, _ = med_p90(a["retried"])- pc, _, _ = med_p90(a["retried_pct"])- w, wp, _ = med_p90(a["wait_s"])- wr, wrp, _ = med_p90(a["wait_per_retried"])- e = sum(a["errors"])- print(f" {k[0]:14s} {k[1]:16s} runs={n:2d} calls/run {fmt(c)}/{fmt(cp)} retried/run {fmt(r)}/{fmt(rp)} "- f"({fmt(pc)}%) wait/run {fmt(w)}/{fmt(wp)} s wait/retried-call {fmt(wr, 1)}/{fmt(wrp, 1)} s model_error rows={e}")---def worker_logs(refs: list[RunRef]) -> None:- print("## fancy_v1 worker.log (per cell; sleeps only — the failed round-trips are not logged)")- per: dict[str, list[float]] = defaultdict(list)- tot: Counter[str] = Counter()- sleep: Counter[str] = Counter()- depth: Counter[int] = Counter()- for ref in refs:- cd = ref.cell_dir- if cd is None:- continue- s_flash = s_pro = 0.0- for line in (cd / "worker.log").read_text(errors="replace").splitlines():- m = STORM_RE.match(line)- if m:- tot[f"storm:{m.group(1)}"] += 1- sleep[m.group(1)] += int(m.group(3))- depth[int(m.group(2))] += 1- if "flash" in m.group(1):- s_flash += int(m.group(3))- else:- s_pro += int(m.group(3))- continue- m = FAIL_RE.match(line)- if m:- tot[f"fail:{m.group(1)}:{m.group(3)[:28]}"] += 1- continue- m = GIVEUP_RE.match(line)- if m:- tot[f"giveup:{m.group(1)}"] += 1- per["flash_sleep_s"].append(s_flash)- per["pro_sleep_s"].append(s_pro)- for k, v in sorted(tot.items()):- print(f" {v:5d} {k}")- for k, v in sleep.items():- print(f" total logged sleep {k}: {v} s over {len(per['flash_sleep_s'])} cells")- for k in ("flash_sleep_s", "pro_sleep_s"):- m, p, n = med_p90(per[k])- print(f" {k} per cell med/p90 = {fmt(m)}/{fmt(p)} (n={n})")- mx = max(depth) if depth else 0- print(f" storm streak depth: max {mx}; streaks >= 10: {sum(v for d, v in depth.items() if d >= 10)} lines; "- f"streak 1: {depth[1]} (= distinct storm episodes)")---if __name__ == "__main__":- refs = discover()- latency_table(refs)- retry_table(refs)- worker_logs(refs)diff --git a/harness/bench/time_audit/corpus.py b/harness/bench/time_audit/corpus.pydeleted file mode 100644index dd1c4908..00000000--- a/harness/bench/time_audit/corpus.py+++ /dev/null@@ -1,126 +0,0 @@-"""Read-only corpus discovery + loaders for the time audit (no side effects)."""-from __future__ import annotations--import glob-import json-import statistics-from dataclasses import dataclass, field-from pathlib import Path-from typing import Any--OUT = Path("/home/yipeng/3dcodeverse/harness/bench/out")-STORM_DAY = "storm(08-26)"-BASELINE = "base(08-23/25)"--# corpus name -> (glob under bench/out, corpus bucket)-CORPORA: dict[str, tuple[str, str]] = {- "fancy_v1": ("fancy_v1/*/arms/*/cells/*/harness_api-agent_gemini_gemini-3.7-flash/run", STORM_DAY),- "h2h_brilliana_v1": ("h2h_brilliana_v1/*/runs/*", STORM_DAY),- "h2h_scene_v1": ("h2h_scene_v1/runs/*", STORM_DAY),- "refs_v1_graphics": ("refs_v1_graphics/runs/*", STORM_DAY),- "teaser": ("teaser/runs/*", BASELINE),- "static_v2_flash": ("static_v2_flash/runs/*", BASELINE),- "scenes_v1_flash": ("scenes_v1_flash/runs/*", BASELINE),-}---@dataclass-class RunRef:- corpus: str- bucket: str- run: Path- track: str = ""- events: list[dict[str, Any]] = field(default_factory=list)-- @property- def cell_dir(self) -> Path | None:- return self.run.parent if (self.run.parent / "worker.log").is_file() else None-- @property- def name(self) -> str:- return self.run.parent.parent.name if self.cell_dir else self.run.name---def read_jsonl(path: Path) -> list[dict[str, Any]]:- rows: list[dict[str, Any]] = []- if not path.is_file():- return rows- with path.open() as fh:- for line in fh:- line = line.strip()- if not line:- continue- try:- rows.append(json.loads(line))- except json.JSONDecodeError:- continue- return rows---def track_of(events: list[dict[str, Any]]) -> str:- start = next((e for e in events if e.get("event") == "run.start"), None)- if start is None:- return "?"- tr, lang = start.get("track", "?"), start.get("language", "?")- if tr == "static_object":- return f"static/{lang}"- if tr == "articulated_object":- return "articulated"- return tr---def discover(corpora: dict[str, tuple[str, str]] = CORPORA) -> list[RunRef]:- refs: list[RunRef] = []- for corpus, (pat, bucket) in corpora.items():- for d in sorted(glob.glob(str(OUT / pat))):- run = Path(d)- ev = read_jsonl(run / "events.jsonl")- if not ev:- continue- refs.append(RunRef(corpus=corpus, bucket=bucket, run=run, track=track_of(ev), events=ev))- return refs---def cost_rows(ref: RunRef) -> list[dict[str, Any]]:- return read_jsonl(ref.run / "telemetry" / "cost.jsonl")---def trajectories(ref: RunRef) -> list[tuple[str, dict[str, Any], list[dict[str, Any]]]]:- """(session label, result.json, transcript rows) for every trajectory dir."""- out: list[tuple[str, dict[str, Any], list[dict[str, Any]]]] = []- tdir = ref.run / "trajectories"- if not tdir.is_dir():- return out- for sd in sorted(p for p in tdir.iterdir() if p.is_dir()):- res: dict[str, Any] = {}- rp = sd / "result.json"- if rp.is_file():- try:- res = json.loads(rp.read_text())- except json.JSONDecodeError:- res = {}- out.append((sd.name, res, read_jsonl(sd / "transcript.jsonl")))- return out---def rounds(ref: RunRef) -> list[dict[str, Any]]:- out: list[dict[str, Any]] = []- for p in sorted((ref.run / "rounds").glob("r*.json")):- try:- out.append(json.loads(p.read_text()))- except (OSError, json.JSONDecodeError):- continue- return out---def med_p90(xs: list[float]) -> tuple[float, float, int]:- """(median, p90, n); p90 by nearest-rank."""- if not xs:- return (float("nan"), float("nan"), 0)- s = sorted(xs)- p90 = s[min(len(s) - 1, int(round(0.9 * (len(s) - 1))))]- return (statistics.median(s), p90, len(s))---def fmt(x: float, nd: int = 0) -> str:- return "-" if x != x else f"{x:.{nd}f}"diff --git a/harness/bench/time_audit/deadline.py b/harness/bench/time_audit/deadline.pydeleted file mode 100644index 869c9227..00000000--- a/harness/bench/time_audit/deadline.py+++ /dev/null@@ -1,59 +0,0 @@-"""Deadline overshoots: sessions past their 1800-s job timeout, runs killed by the-bench max_minutes with no round completed, and the storm-streak depth at each-'giving up after 900 s' (=> seconds per failed 503 attempt, independent of transcripts)."""-from __future__ import annotations--import json-import re-import sys-from pathlib import Path--sys.path.insert(0, str(Path(__file__).parent))-from corpus import discover, fmt, med_p90, trajectories # noqa: E402--STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 ")-GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")-BUDGET_RE = re.compile(r"elapsed ([\d.]+) min exceeds max_minutes ([\d.]+)")-SESSION_TIMEOUT_S = 1800.0---def main() -> None:- refs = discover()- over, killed, zero_rounds, depths = [], [], 0, []- for ref in refs:- if ref.bucket != "storm(08-26)":- continue- for _, res, _ in trajectories(ref):- d = float(res.get("duration_s") or 0)- if d > SESSION_TIMEOUT_S and res.get("exit_reason") in ("timeout", "error"):- over.append(d - SESSION_TIMEOUT_S)- rounds_done = sum(1 for e in ref.events if e["event"] == "round.done")- for e in ref.events:- if e["event"] == "budget.exceeded":- m = BUDGET_RE.search(str(e.get("reason", "")))- if m:- killed.append((float(m.group(1)) - float(m.group(2))) * 60)- zero_rounds += rounds_done == 0- cd = ref.cell_dir- if cd is None:- continue- last = 0- for line in (cd / "worker.log").read_text(errors="replace").splitlines():- m = STORM_RE.match(line)- if m and "flash" in m.group(1):- last = int(m.group(2))- elif GIVEUP_RE.match(line) and "flash" in line:- depths.append(last)- m, p, n = med_p90(over)- print(f"sessions past the {SESSION_TIMEOUT_S:.0f}-s job timeout (exit timeout/error): n={n} overshoot med/p90 = {fmt(m)}/{fmt(p)} s, sum={sum(over):.0f} s")- m, p, n = med_p90(killed)- print(f"runs stopped by bench max_minutes: n={n}, overshoot past the ceiling med/p90 = {fmt(m)}/{fmt(p)} s; with ZERO completed rounds: {zero_rounds}")- m, p, n = med_p90([float(d) for d in depths])- if n:- print(f"flash give-ups in fancy_v1 logs: n={n}; storm-streak depth at give-up med/p90 = {fmt(m)}/{fmt(p)} "- f"=> 900 s / depth = {fmt(900 / m, 1)} s per failed attempt (sleep included, <= 5 s of it)")- print(json.dumps({"overshoot_sum_s": round(sum(over)), "killed": len(killed), "zero_rounds": zero_rounds}))---if __name__ == "__main__":- main()diff --git a/harness/bench/time_audit/judge_plan.py b/harness/bench/time_audit/judge_plan.pydeleted file mode 100644index 2d417d1a..00000000--- a/harness/bench/time_audit/judge_plan.py+++ /dev/null@@ -1,55 +0,0 @@-"""Judge + planner stages: stage span vs recorded model latency (the difference is-retry/timeout waiting), per corpus; judge samples that errored and what they cost."""-from __future__ import annotations--import sys-from collections import defaultdict-from pathlib import Path--sys.path.insert(0, str(Path(__file__).parent))-from corpus import cost_rows, discover, fmt, med_p90 # noqa: E402---def main() -> None:- J: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))- for ref in discover():- rows = cost_rows(ref)- by_round: dict[int, list[dict]] = defaultdict(list)- for r in rows:- if r["role"] == "judge":- by_round[int(r.get("round") or 0)].append(r)- b = J[ref.bucket]- for e in ref.events:- if e["event"] == "judge.done":- d = float(e.get("duration_s") or 0)- rs = by_round.get(int(e.get("round") or 0), [])- ok = sum(r["latency_ms"] / 1000 for r in rs if r["outcome"] == "ok")- err = [r["latency_ms"] / 1000 for r in rs if r["outcome"] != "ok"]- b["judge_s"].append(d)- b["judge_ok_lat"].append(ok)- b["judge_wait"].append(max(0.0, d - ok))- b["judge_n_ok"].append(float(sum(1 for r in rs if r["outcome"] == "ok")))- if err:- b["judge_err_rows"].extend(err)- b["judge_rounds_with_err"].append(d)- elif e["event"] == "stage.done" and e.get("stage") == "plan":- d = float(e.get("duration_s") or 0)- pl = [r for r in rows if r["role"] == "planner"]- ok = sum(r["latency_ms"] / 1000 for r in pl if r["outcome"] == "ok")- b["plan_s"].append(d)- b["plan_ok_lat"].append(ok)- b["plan_calls"].append(float(len(pl)))- b["plan_wait"].append(max(0.0, d - ok))- for bucket, b in sorted(J.items()):- print(f"## {bucket}")- for k in ("judge_s", "judge_ok_lat", "judge_wait", "judge_n_ok", "plan_s", "plan_ok_lat", "plan_calls", "plan_wait"):- m, p, n = med_p90(b[k])- print(f" {k:14s} n={n:3d} med={fmt(m, 1):>7s} p90={fmt(p, 1):>7s} sum={sum(b[k]):.0f}")- er = b["judge_err_rows"]- print(f" judge error rows: n={len(er)} sum={sum(er):.0f} s each: {[round(x) for x in sorted(er, reverse=True)[:8]]}")- rw = b["judge_rounds_with_err"]- print(f" judge rounds with an errored sample: {len(rw)} / {len(b['judge_s'])}; their judge_s: {[round(x) for x in sorted(rw, reverse=True)[:8]]}")---if __name__ == "__main__":- main()diff --git a/harness/bench/time_audit/keys.py b/harness/bench/time_audit/keys.pydeleted file mode 100644index 64cc9f10..00000000--- a/harness/bench/time_audit/keys.py+++ /dev/null@@ -1,60 +0,0 @@-"""Task 4: does any persisted telemetry row say WHICH Gemini key served a call?-Scans every field name in cost.jsonl / transcript.jsonl / result.json / rounds /-record.json for something key-like, and greps the raw text for the '…xxxx' redaction-that gemini.py puts into ChatResponse.raw."""-from __future__ import annotations--import json-import re-import sys-from collections import Counter-from pathlib import Path--sys.path.insert(0, str(Path(__file__).parent))-import contextlib--from corpus import discover, read_jsonl # noqa: E402--KEYLIKE = re.compile(r"key|api_key|credential", re.I)-REDACTED = re.compile(r'"key":\s*"…[^"]{0,8}"')---def walk(obj: object, prefix: str, found: Counter[str]) -> None:- if isinstance(obj, dict):- for k, v in obj.items():- if KEYLIKE.search(str(k)):- found[f"{prefix}.{k}"] += 1- walk(v, f"{prefix}.{k}", found)- elif isinstance(obj, list):- for v in obj[:50]:- walk(v, prefix + "[]", found)---def main() -> None:- found: Counter[str] = Counter()- redacted = 0- files = 0- for ref in discover():- for p in [ref.run / "telemetry" / "cost.jsonl", ref.run / "record.json", *(ref.run / "rounds").glob("r*.json"),- *(ref.run / "trajectories").glob("*/result.json"), *(ref.run / "trajectories").glob("*/transcript.jsonl")]:- if not p.is_file():- continue- files += 1- text = p.read_text(errors="replace")- redacted += len(REDACTED.findall(text))- if p.suffix == ".jsonl":- for row in read_jsonl(p):- walk(row, p.name, found)- else:- with contextlib.suppress(json.JSONDecodeError):- walk(json.loads(text), p.name, found)- print(f"scanned {files} files; '\"key\": \"…xxxx\"' occurrences: {redacted}")- print("key-like field names found (name -> count):")- for k, v in found.most_common(20):- print(f" {v:7d} {k}")- if not any("api" in k.lower() or k.endswith(".key") for k in found):- print("=> no per-call key identity is persisted anywhere in the run telemetry.")---if __name__ == "__main__":- main()diff --git a/harness/bench/time_audit/sessions.py b/harness/bench/time_audit/sessions.pydeleted file mode 100644index 55ffd861..00000000--- a/harness/bench/time_audit/sessions.py+++ /dev/null@@ -1,58 +0,0 @@-"""Task 3: per agent session (api-agent transcripts) — turns, tool calls by name-with durations, and the model-thinking / tool-execution / waiting split, one table-per track x corpus."""-from __future__ import annotations--import sys-from collections import defaultdict-from pathlib import Path--sys.path.insert(0, str(Path(__file__).parent))-from corpus import discover, fmt, med_p90, trajectories # noqa: E402-from turns import parse_turns, split # noqa: E402---def main() -> None:- sess: dict[tuple[str, str], list[tuple[int, float, float, float, float]]] = defaultdict(list)- tools: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))- for ref in discover():- k = (ref.bucket, ref.track)- for _, res, tr in trajectories(ref):- turns = parse_turns(tr)- if not turns:- continue- think, tool, wait = split(turns)- sess[k].append((len([t for t in turns if t.latency_s > 0]), float(res.get("duration_s") or 0), think, tool, wait))- for t in turns:- for name, d in t.tools:- tools[k][name].append(d)- print("## per session: turns, duration, and the split (medians; shares are of summed session time)")- print("| corpus | track | sessions | turns med/p90 | dur s med/p90 | think s | tool s | wait s | think% | tool% | wait% |")- print("|---|---|---|---|---|---|---|---|---|---|---|")- for k in sorted(sess):- rows = sess[k]- tm, tp, n = med_p90([float(r[0]) for r in rows])- dm, dp, _ = med_p90([r[1] for r in rows])- th, _, _ = med_p90([r[2] for r in rows])- to, _, _ = med_p90([r[3] for r in rows])- wa, _, _ = med_p90([r[4] for r in rows])- T = sum(r[2] + r[3] + r[4] for r in rows) or 1.0- print(f"| {k[0]} | {k[1]} | {n} | {fmt(tm)}/{fmt(tp)} | {fmt(dm)}/{fmt(dp)} | {fmt(th)} | {fmt(to)} | {fmt(wa)} | "- f"{100 * sum(r[2] for r in rows) / T:.0f} | {100 * sum(r[3] for r in rows) / T:.0f} | {100 * sum(r[4] for r in rows) / T:.0f} |")- print("\n## tool calls by name: count, median s, p90 s, total s per session (top tools per track; storm-day + baseline pooled by track)")- pooled: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))- nsess: dict[str, int] = defaultdict(int)- for k, d in tools.items():- for name, xs in d.items():- pooled[k[1]][name].extend(xs)- nsess[k[1]] += len(sess[k])- for track in sorted(pooled):- items = sorted(pooled[track].items(), key=lambda kv: -sum(kv[1]))- print(f" {track} ({nsess[track]} sessions):")- for name, xs in items[:12]:- m, p, n = med_p90(xs)- print(f" {name:18s} n={n:5d} med={fmt(m, 2):>6s} p90={fmt(p, 1):>6s} total/session={sum(xs) / max(1, nsess[track]):6.1f} s")---if __name__ == "__main__":- main()diff --git a/harness/bench/time_audit/stage_times.py b/harness/bench/time_audit/stage_times.pydeleted file mode 100644index 3176f630..00000000--- a/harness/bench/time_audit/stage_times.py+++ /dev/null@@ -1,156 +0,0 @@-"""Task 1: per track x corpus, median / p90 of run wall and every stage, plus-"waiting on the provider" (retry sleeps + failed attempts), derived from transcript-timestamps (agent sessions) and from judge/planner stage span minus recorded latency.-Writes stage_runs.jsonl (one row per run) and prints the table."""-from __future__ import annotations--import json-import sys-from collections import defaultdict-from pathlib import Path-from typing import Any--sys.path.insert(0, str(Path(__file__).parent))-from corpus import RunRef, cost_rows, discover, fmt, med_p90, rounds, trajectories # noqa: E402-from turns import parse_turns, split # noqa: E402--HERE = Path(__file__).parent---def round_segments(ev: list[dict[str, Any]]) -> list[dict[str, Any]]:- """One dict per round.start..round.done segment (an abandoned segment ends at the- next round.start / run.start and is flagged)."""- segs: list[dict[str, Any]] = []- cur: dict[str, Any] | None = None- cand: dict[Any, float] = {} # best-of-N candidates are generated BEFORE round.start- for e in ev:- k = e["event"]- if k == "candidates.start":- cand[e.get("round")] = e["t"]- if k in ("round.start", "run.start"):- if cur is not None:- cur["abandoned"] = True- cur["end"] = e["t"]- segs.append(cur)- cur = None- if k == "round.start":- cur = {"round": e.get("round"), "start": min(e["t"], cand.pop(e.get("round"), e["t"])), "abandoned": False}- continue- if cur is None:- continue- if k == "build.done":- cur.setdefault("first_build", e["t"])- cur["last_build"] = e["t"]- cur["build_ms"] = cur.get("build_ms", 0) + int(e.get("duration_ms") or 0)- cur["n_builds"] = cur.get("n_builds", 0) + 1- elif k == "gates.done":- cur["gates_t"] = e["t"]- elif k == "judge.done":- cur["judge_s"] = float(e.get("duration_s") or 0)- cur["judge_t"] = e["t"]- elif k == "round.done":- cur["end"] = e["t"]- cur["duration_s"] = float(e.get("duration_s") or (e["t"] - cur["start"]))- segs.append(cur)- cur = None- if cur is not None:- cur["abandoned"] = True- cur["end"] = ev[-1]["t"]- segs.append(cur)- return segs---def analyse(ref: RunRef) -> dict[str, Any]:- ev = ref.events- done = [e for e in ev if e["event"] == "run.done"]- row: dict[str, Any] = {- "corpus": ref.corpus, "bucket": ref.bucket, "track": ref.track, "run": ref.name,- "finished": bool(done), "wall_s": ev[-1]["t"] - ev[0]["t"],- "status": done[-1].get("status") if done else "RUNNING",- }- plan_s = sum(float(e.get("duration_s") or 0) for e in ev if e["event"] == "stage.done" and e.get("stage") == "plan")- row["plan_s"] = plan_s- scene_gen = sum(float(e.get("duration_s") or 0) for e in ev- if e["event"] == "stage.done" and e.get("stage") in ("assets", "env", "zones", "assemble"))- row["plan_cached"] = any(e["event"] == "stage.cached" and e.get("stage") == "plan" for e in ev)- rows = cost_rows(ref)- lat = lambda role, ok=True: sum( # noqa: E731- r["latency_ms"] / 1000 for r in rows if r["role"] == role and (r["outcome"] == "ok") == ok)- row["plan_model_s"] = lat("planner")- row["plan_wait_s"] = max(0.0, plan_s - lat("planner")) if plan_s else 0.0- row["judge_err_s"] = lat("judge", ok=False) # error rows carry the whole failed span- segs = [s for s in round_segments(ev) if not s["abandoned"]]- row["abandoned_s"] = sum(s["end"] - s["start"] for s in round_segments(ev) if s["abandoned"])- rd = {r.get("index"): r for r in rounds(ref)}- gen, build, gates, render, judge, other = [], [], [], [], [], []- for s in segs:- if "gates_t" not in s or "last_build" not in s:- continue- b = s["build_ms"] / 1000- gen.append(s["last_build"] - s["start"] - b)- build.append(b)- gates.append(s["gates_t"] - s["last_build"])- r = rd.get(s["round"], {})- rnd = float((r.get("renders") or {}).get("duration_ms") or 0) / 1000- render.append(rnd)- j = s.get("judge_s", 0.0)- judge.append(j)- if "judge_t" in s:- other.append(max(0.0, s["judge_t"] - s["gates_t"] - j - rnd))- judge_ok = lat("judge")- row.update(gen_s=sum(gen) + scene_gen, build_s=sum(build), gates_s=sum(gates), render_s=sum(render),- judge_s=sum(judge), other_s=sum(other), n_rounds=len(gen),- judge_model_s=judge_ok, judge_wait_s=max(0.0, sum(judge) - judge_ok))- think = tools = wait = sess = 0.0- n_sess = 0- for _, res, tr in trajectories(ref):- a, b2, c = split(parse_turns(tr))- think, tools, wait, sess, n_sess = think + a, tools + b2, wait + c, sess + float(res.get("duration_s") or 0), n_sess + 1- row.update(sess_think_s=think, sess_tool_s=tools, sess_wait_s=wait, sess_total_s=sess, n_sessions=n_sess)- row["wait_total_s"] = wait + row["plan_wait_s"] + row["judge_wait_s"]- return row---def main() -> None:- refs = discover()- out = [analyse(r) for r in refs]- (HERE / "stage_runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in out))- cols = ["wall_s", "plan_s", "gen_s", "build_s", "gates_s", "render_s", "judge_s", "wait_total_s", "sess_wait_s", "judge_wait_s", "judge_err_s", "abandoned_s"]- groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)- for r in out:- groups[(r["bucket"], r["track"])].append(r)- print("| corpus | track | n(fin) | " + " | ".join(f"{c} med/p90" for c in cols) + " |")- print("|" + "---|" * (len(cols) + 3))- for (bucket, track), rs in sorted(groups.items()):- fin = [r for r in rs if r["finished"]]- cells = []- for c in cols:- src = fin if c == "wall_s" else rs- m, p, _ = med_p90([float(r[c]) for r in src])- cells.append(f"{fmt(m)}/{fmt(p)}")- print(f"| {bucket} | {track} | {len(rs)}({len(fin)}) | " + " | ".join(cells) + " |")- print("\nshare of run wall: waiting / thinking / tools (medians of per-run ratios, finished runs)")- for (bucket, track), rs in sorted(groups.items()):- fin = [r for r in rs if r["finished"] and r["wall_s"] > 0]- if not fin:- continue- w = med_p90([100 * r["wait_total_s"] / r["wall_s"] for r in fin])[0]- th = med_p90([100 * r["sess_think_s"] / r["wall_s"] for r in fin])[0]- to = med_p90([100 * (r["sess_tool_s"] + r["build_s"] + r["gates_s"] + r["render_s"]) / r["wall_s"] for r in fin])[0]- jm = med_p90([100 * r["judge_model_s"] / r["wall_s"] for r in fin])[0]- print(f" {bucket:14s} {track:16s} wait {fmt(w)}% think {fmt(th)}% tools+build+gates+render {fmt(to)}% judge model {fmt(jm)}%")- # per-round view (generate + judge per round) for the storm/baseline contrast- print("\nper-round: gen_s and judge_s med/p90 (rounds with a build+gates)")- per: dict[tuple[str, str], tuple[list[float], list[float]]] = defaultdict(lambda: ([], []))- for r in out:- n = max(1, r["n_rounds"])- per[(r["bucket"], r["track"])][0].append(r["gen_s"] / n)- per[(r["bucket"], r["track"])][1].append(r["judge_s"] / n)- for k, (g, j) in sorted(per.items()):- gm, gp, _ = med_p90(g)- jm, jp, _ = med_p90(j)- print(f" {k[0]:14s} {k[1]:16s} gen {fmt(gm)}/{fmt(gp)} judge {fmt(jm)}/{fmt(jp)}")---if __name__ == "__main__":- main()diff --git a/harness/bench/time_audit/storm_attempts.py b/harness/bench/time_audit/storm_attempts.pydeleted file mode 100644index 601d947a..00000000--- a/harness/bench/time_audit/storm_attempts.py+++ /dev/null@@ -1,78 +0,0 @@-"""Price the pieces of "waiting": backoff sleeps (worker.log) vs failed 503 round-trips (transcript wait minus sleep, per storm line) vs 900-s give-ups (model_error-rows / turns with wait >= 850 s), plus cell wall vs run span for the fancy_v1 cells."""-from __future__ import annotations--import json-import re-import sys-from collections import Counter-from pathlib import Path--sys.path.insert(0, str(Path(__file__).parent))-from corpus import discover, fmt, med_p90, trajectories # noqa: E402-from turns import parse_turns # noqa: E402--STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")-GIVEUP_S = 850.0---def hist(xs: list[float], edges: list[float]) -> str:- c = Counter()- for x in xs:- c[next((e for e in edges if x < e), float("inf"))] += 1- tot = max(1, len(xs))- return " ".join(f"<{e:g}s:{100 * c[e] / tot:.0f}%" for e in [*edges, float("inf")])---def main() -> None:- refs = discover()- for bucket in ("base(08-23/25)", "storm(08-26)"):- waits, big, errs = [], [], []- for ref in [r for r in refs if r.bucket == bucket]:- for _, _, tr in trajectories(ref):- for t in parse_turns(tr):- if t.latency_s > 0:- waits.append(t.wait_s)- if t.error_s > 0:- errs.append(t.error_s)- tot = sum(waits) + sum(errs)- big = [w for w in waits if w >= GIVEUP_S]- print(f"## {bucket}: {len(waits)} generator turns; wait histogram: {hist(waits, [1, 5, 30, 120, 850])}")- print(f" total wait {tot:.0f} s; turns with wait>=850 s: {len(big)} = {100 * sum(big) / max(1, tot):.0f}% of all wait; "- f"model_error spans: n={len(errs)} sum={sum(errs):.0f} s ({100 * sum(errs) / max(1, tot):.0f}%) "- f"med/p90={fmt(med_p90(errs)[0])}/{fmt(med_p90(errs)[1])} s")- print(f" wait>=850s turn: {hist(big, [900, 1800, 2700])}")- print("## fancy_v1 cells: transcript wait vs logged sleep -> cost of one failed 503 round trip")- rt, per_cell, outside = [], [], []- for ref in refs:- cd = ref.cell_dir- if cd is None:- continue- lines = (cd / "worker.log").read_text(errors="replace").splitlines()- n = sleep = 0- for m in filter(None, map(STORM_RE.match, lines)):- if "flash" in m.group(1):- n += 1- sleep += int(m.group(3))- turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]- w = sum(t.wait_s for t in turns) + sum(t.error_s for t in turns)- w_nogiveup = sum(min(t.wait_s, 0) if t.wait_s >= GIVEUP_S else t.wait_s for t in turns)- if n:- rt.append((w_nogiveup - sleep) / n)- per_cell.append((ref.name, n, sleep, round(w), round(w_nogiveup)))- try:- cell = json.loads((cd / "cell.json").read_text())- outside.append(float(cell.get("wall_s") or 0) - (ref.events[-1]["t"] - ref.events[0]["t"]))- except (OSError, json.JSONDecodeError):- pass- m, p, k = med_p90(rt)- print(f" per-cell (wait_excluding_giveups - sleep) / storm_lines: med {fmt(m, 1)} s p90 {fmt(p, 1)} s (n={k} cells)")- print(f" cell wall_s - run event span (pairwise + final pro scoring outside the run): med/p90 = "- f"{fmt(med_p90(outside)[0])}/{fmt(med_p90(outside)[1])} s")- for row in sorted(per_cell, key=lambda r: -r[3])[:5]:- print(" worst:", row)---if __name__ == "__main__":- main()diff --git a/harness/bench/time_audit/turns.py b/harness/bench/time_audit/turns.pydeleted file mode 100644index e831f541..00000000--- a/harness/bench/time_audit/turns.py+++ /dev/null@@ -1,67 +0,0 @@-"""Per-turn reconstruction of an api-agent transcript.--The ledger's ``latency_ms`` is the stopwatch of the SUCCESSFUL attempt only-(``gemini.py:_once``); backoff sleeps, failed attempts and KeyPool waits are not-recorded anywhere. Transcript rows are stamped when the model returns-(``assistant``) and when a tool finishes (``tool_result``), so-``model_wall = t(assistant) - t(previous row)`` and ``wait = model_wall - latency``.-"""-from __future__ import annotations--from dataclasses import dataclass, field-from typing import Any---@dataclass-class Turn:- turn: int- model_wall_s: float- latency_s: float- wait_s: float- tools: list[tuple[str, float]] = field(default_factory=list)- error_s: float = 0.0 # time spent in a call that ended in a model_error row---def is_gemini_api_session(rows: list[dict[str, Any]]) -> bool:- """Only in-process api-agent sessions on Gemini stamp a per-turn latency."""- return any(r.get("kind") == "assistant" and (r.get("usage") or {}).get("backend") == "gemini" for r in rows)---def parse_turns(rows: list[dict[str, Any]]) -> list[Turn]:- turns: list[Turn] = []- if not rows or not is_gemini_api_session(rows):- return turns- prev_end = float(rows[0]["t"])- cur: Turn | None = None- tool_start = prev_end- pending_err = 0.0- for r in rows:- t = float(r["t"])- kind = r.get("kind")- if kind == "assistant":- wall = max(0.0, t - prev_end)- lat = float((r.get("usage") or {}).get("latency_ms") or 0) / 1000.0- cur = Turn(int(r.get("turn", len(turns))), wall, lat, max(0.0, wall - lat), error_s=pending_err)- pending_err = 0.0- turns.append(cur)- prev_end = tool_start = t- elif kind == "tool_result":- if cur is not None:- cur.tools.append((str(r.get("name", "?")), max(0.0, t - tool_start)))- tool_start = prev_end = t- elif kind == "model_error":- pending_err += max(0.0, t - prev_end)- prev_end = t- elif kind in ("nudge", "compact", "system"):- prev_end = t- if pending_err > 0: # session died in a model_error after the last good turn- turns.append(Turn(len(turns), 0.0, 0.0, 0.0, error_s=pending_err))- return turns---def split(turns: list[Turn]) -> tuple[float, float, float]:- """(model thinking s, tool execution s, waiting s) over a session."""- think = sum(t.latency_s for t in turns)- tools = sum(d for t in turns for _, d in t.tools)- wait = sum(t.wait_s + t.error_s for t in turns)- return think, tools, waitdiff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.pyindex efb5cef9..ac4a180c 100644--- a/harness/codeverse/cli/flywheel_cmd.py+++ b/harness/codeverse/cli/flywheel_cmd.py@@ -69,12 +69,11 @@ def pairs_cmd(runs_dir: Annotated[Path, typer.Argument()],out_jsonl: Annotated[Path, typer.Argument()],min_delta: Annotated[float, typer.Option("--min-delta")] = 0.05,- trajectories: Annotated[bool, typer.Option("--trajectories/--no-trajectories", help="also mine in-session repair pairs from agent transcripts")] = True,) -> None:"""Preference / repair (round + in-session) / cross-backend pairs → JSONL."""from codeverse.flywheel.pairs import build_pairs- n = build_pairs(runs_dir, out_jsonl, min_delta=min_delta, trajectories=trajectories)+ n = build_pairs(runs_dir, out_jsonl, min_delta=min_delta)ok(f"{n} pairs → {out_jsonl}")diff --git a/harness/codeverse/cost/__init__.py b/harness/codeverse/cost/__init__.pyindex 54c2b749..e66894dd 100644--- a/harness/codeverse/cost/__init__.py+++ b/harness/codeverse/cost/__init__.py@@ -8,7 +8,7 @@ Public API::from codeverse.cost import record_call, load_ledger, summarisefrom codeverse.cost import audit_runs, markdown # the audit + its report- from codeverse.cost import estimate_call, CostGuard # decide before you send+ from codeverse.cost import estimate_call # estimate before you sendfrom codeverse.cost import Block, order_blocks # cache-friendly promptsfrom codeverse.cost import default_route, pro_break_even # model routing@@ -28,13 +28,7 @@ from codeverse.cost.caching import (render_blocks,)from codeverse.cost.context import CallContext, bind_run, call_context-from codeverse.cost.guard import (- CostEstimate,- CostGuard,- cheapest_affordable,- estimate_call,- text_tokens,-)+from codeverse.cost.guard import CostEstimate, estimate_call, text_tokensfrom codeverse.cost.instrument import (MeteredAgent,MeteredChatModel,@@ -72,7 +66,6 @@ __all__ = ["CallCost","CostBucket","CostEstimate",- "CostGuard","CostLedger","MeteredAgent","MeteredChatModel",@@ -90,7 +83,6 @@ __all__ = ["bind_run","cache_efficiency","call_context",- "cheapest_affordable","console","default_route","estimate_call",diff --git a/harness/codeverse/cost/guard.py b/harness/codeverse/cost/guard.pyindex 986a28d5..daf78477 100644--- a/harness/codeverse/cost/guard.py+++ b/harness/codeverse/cost/guard.py@@ -1,9 +1,8 @@"""Estimate what a call will cost **before** sending it, and pick a tier.The budget guard in ``orchestrator/budget.py`` stops a run *after* the money is-gone. This is the cheap check that belongs in front of a call: how many tokens-am I about to send, what will they cost on this model, and is there a cheaper-model that still fits the job?+gone. This is the cheap estimate that belongs in front of a call: how many+tokens am I about to send, and what will they cost on this model?Token counts are estimates (≈4 characters per token for prose/code, a flatper-image count for vision parts) — good to ±15% for prompt sizing, which is all@@ -14,7 +13,7 @@ the provider returns (``codeverse.cost.record_call``).from __future__ import annotationsfrom collections.abc import Iterable, Sequence-from dataclasses import dataclass, field+from dataclasses import dataclassfrom codeverse.cost.types import normalise_idsfrom codeverse.models.pricing import per_image_usd, price_provenance, unit_prices@@ -97,66 +96,3 @@ def estimate_call(usd=usd, price_source=row.match if row.price else "unknown",approximate=row.approximate)--@dataclass-class CostGuard:- """A per-run (or per-stage) allowance checked *before* each call.-- ``spent`` is advanced by :meth:`commit` with what the call really cost, so- the guard converges on the truth even when the estimate was off."""-- budget_usd: float- spent_usd: float = 0.0- reserve_usd: float = 0.0 # keep this much back for the judge / finalise- calls: int = 0- rejected: list[str] = field(default_factory=list)-- @property- def remaining_usd(self) -> float:- return max(0.0, self.budget_usd - self.reserve_usd - self.spent_usd)-- def affordable(self, estimate: CostEstimate) -> bool:- return estimate.usd <= self.remaining_usd-- def check(self, estimate: CostEstimate) -> tuple[bool, str]:- """``(allow, reason)``. An unpriced model is allowed but flagged: refusing- to run because we cannot price a model would be worse than running it."""- if not estimate.known:- return True, f"no price row for {estimate.model_id}: cost unknown, not blocked"- if self.affordable(estimate):- return True, ""- msg = (f"{estimate.model_id} would cost ${estimate.usd:.4f} but only "- f"${self.remaining_usd:.4f} is left of ${self.budget_usd:.2f}")- self.rejected.append(msg)- return False, msg-- def commit(self, usd: float) -> None:- self.spent_usd += max(0.0, usd)- self.calls += 1---def cheapest_affordable(- model_ids: Sequence[str],- *,- input_tokens: int = 0,- output_tokens: int = 0,- cached_tokens: int = 0,- budget_usd: float | None = None,-) -> tuple[str, CostEstimate] | None:- """Pick the cheapest model in ``model_ids`` that fits ``budget_usd``.-- ``model_ids`` should be ordered best-quality-first; the caller decides how- much quality it is prepared to trade (see :mod:`codeverse.cost.routing`)."""- scored: list[tuple[float, str, CostEstimate]] = []- for mid in model_ids:- est = estimate_call(mid, input_tokens=input_tokens, output_tokens=output_tokens,- cached_tokens=cached_tokens)- if not est.known:- continue- if budget_usd is not None and est.usd > budget_usd:- continue- scored.append((est.usd, mid, est))- if not scored:- return None- scored.sort(key=lambda s: s[0])- return scored[0][1], scored[0][2]diff --git a/harness/codeverse/flywheel/__init__.py b/harness/codeverse/flywheel/__init__.pyindex 55f9d005..bb26cfd9 100644--- a/harness/codeverse/flywheel/__init__.py+++ b/harness/codeverse/flywheel/__init__.py@@ -9,6 +9,5 @@ Public API (see docs/INTERFACES.md):* ``captions`` ``caption_sample(ws, record, model_id)`` → ``Captions``* ``dedupe`` ``code_fingerprint`` · ``mesh_fingerprint`` · ``near_duplicates``* ``quality`` ``quality_tier`` (A/B/C/D) · ``prompt_hash`` · exact (code, prompt) duplicate groups-* ``trajectories`` in-session repair pairs mined from api-agent transcripts* ``index`` ``build_index(runs_dir, out_sqlite)`` + query helpers"""diff --git a/harness/codeverse/flywheel/deliverable.py b/harness/codeverse/flywheel/deliverable.pyindex 323806f7..70d96161 100644--- a/harness/codeverse/flywheel/deliverable.py+++ b/harness/codeverse/flywheel/deliverable.py@@ -49,10 +49,6 @@ _ARTIFACT_ROLES: tuple[tuple[str, str], ...] = ()-class DeliverableError(RuntimeError):- """The run has nothing that can be handed over."""--def _sha256(data: bytes) -> str:return hashlib.sha256(data).hexdigest()diff --git a/harness/codeverse/flywheel/pairs.py b/harness/codeverse/flywheel/pairs.pyindex 3a016969..b9fe3463 100644--- a/harness/codeverse/flywheel/pairs.py+++ b/harness/codeverse/flywheel/pairs.py@@ -18,9 +18,6 @@ Output is JSONL, one object per pair. All kinds share::(``notes`` say "repair attempts: N (fixed)"): the ``rNN <kind>: generated``commit is the rejected state, the round's final commit the chosen one; theerror message comes from the round's failing ``build.done`` events.-* ``repair`` (``source: "trajectory"``) — the same shape mined from an agent- session's transcript: last failing ``build`` → next successful ``build``- (see ``trajectories.py``); ``error`` is the build tool's message.* ``cross_backend`` the same (prompt, track, language) run under ≥ 2 generators;best vs each other candidate with Δ ≥ ``min_delta``."""@@ -36,11 +33,10 @@ from codeverse.contracts.run import RoundRecord, RunRecordfrom codeverse.flywheel import _gitfrom codeverse.flywheel.quality import prompt_hashfrom codeverse.flywheel.record import effective_judgment, iter_runs-from codeverse.flywheel.trajectories import mine_runfrom codeverse.workspace import Workspace__all__ = ["build_pairs", "preference_pairs", "repair_pairs", "in_round_repair_pairs",- "trajectory_repair_pairs", "cross_backend_pairs", "prompt_hash"]+ "cross_backend_pairs", "prompt_hash"]MAX_INLINE_CODE = 200_000@@ -207,28 +203,6 @@ def in_round_repair_pairs(ws: Workspace, rec: RunRecord, *, slug: str | None = Nreturn out-def trajectory_repair_pairs(ws: Workspace, rec: RunRecord, *, slug: str | None = None) -> list[dict[str, Any]]:- """In-session repairs (failing build → fixing build) mined from api-agent transcripts."""- out = []- for tr in mine_run(ws):- pair = _base(ws, rec, "repair", slug=slug)- pair.update(- source="trajectory",- trajectory=tr.trajectory,- chosen={"round": tr.round_index, "kind": tr.stage, "commit": "", "turn": tr.fixed_turn, "score": None,- "passed": None, "build_ok": True, "files": tr.chosen, "truncated_files": []},- rejected={"round": tr.round_index, "kind": tr.stage, "commit": "", "turn": tr.broken_turn, "score": None,- "passed": None, "build_ok": False, "files": tr.rejected, "truncated_files": []},- delta=None,- error={"type": "", "message": tr.error, "file": "", "line": None, "stderr_tail": "", "stdout_tail": "",- "gate_errors": []},- reason=[f"in-session fix of a failing build ({tr.n_failures} failing build(s) folded)"],- changed_files=tr.changed_files,- )- out.append(pair)- return out--def cross_backend_pairs(groups: dict[tuple[str, str, str], list[tuple[Workspace, RunRecord, str]]], *, min_delta: float) -> list[dict[str, Any]]:@@ -269,12 +243,8 @@ def cross_backend_pairs(return out-def build_pairs(- runs_dir: Path | str, out_jsonl: Path | str, *, min_delta: float = 0.05, trajectories: bool = True-) -> int:- """Write all pair kinds for the runs under ``runs_dir``; returns the number written.-- ``trajectories`` adds in-session repair pairs mined from agent transcripts."""+def build_pairs(runs_dir: Path | str, out_jsonl: Path | str, *, min_delta: float = 0.05) -> int:+ """Write all pair kinds for the runs under ``runs_dir``; returns the number written."""out = Path(out_jsonl)out.parent.mkdir(parents=True, exist_ok=True)groups: dict[tuple[str, str, str], list[tuple[Workspace, RunRecord, str]]] = defaultdict(list)@@ -286,8 +256,6 @@ def build_pairs(groups[(prompt_hash(rec.spec.prompt), rec.spec.track.value, rec.spec.language.value)].append((ws, rec, slug))pairs = (preference_pairs(ws, rec, min_delta=min_delta, slug=slug)+ repair_pairs(ws, rec, slug=slug) + in_round_repair_pairs(ws, rec, slug=slug))- if trajectories:- pairs += trajectory_repair_pairs(ws, rec, slug=slug)for pair in pairs:fh.write(json.dumps(pair, ensure_ascii=False) + "\n")n += 1diff --git a/harness/codeverse/flywheel/trajectories.py b/harness/codeverse/flywheel/trajectories.pydeleted file mode 100644index ba7fcfb9..00000000--- a/harness/codeverse/flywheel/trajectories.py+++ /dev/null@@ -1,203 +0,0 @@-"""Mine repair pairs from agent trajectories (``trajectories/<stage>_rNN/transcript.jsonl``).--Round-level repair pairs (``pairs.repair_pairs``) need a round whose build-failed. Most build failures are fixed *inside* one agent session instead —-the ``api-agent`` transcript records every ``write_file`` / ``edit_file`` /-``build`` tool call, so the file state can be replayed::-- write_file ... → build FAIL (error) → edit/write ... → build OK- ^ rejected snapshot ^ chosen snapshot--Only the ``api-agent`` transcript format (``assistant`` turns with-``tool_calls`` + ``tool_result`` turns) is structured enough; CLI backends-(gemini-cli / codex / claude-code) log raw stdout lines and yield no pairs.--The base file state is the ``pre:<label>`` git commit taken by the harness-right before the session (nearest one not after the first transcript event);-``read_file`` results are not used (they may be truncated).-"""--from __future__ import annotations--import subprocess-from pathlib import Path-from typing import Any--from pydantic import BaseModel, Field--from codeverse.flywheel import _git-from codeverse.proc import read_jsonl_lenient-from codeverse.workspace import Workspace--BUILD_TOOLS = frozenset({"build", "run_build"})-MAX_PAIR_CODE = 200_000---class TrajectoryRepair(BaseModel):- """One in-session repair: the last failing snapshot before a successful build."""-- trajectory: str = Field(description="trajectory dir name, e.g. refine_r00")- stage: str- round_index: int | None- broken_turn: int- fixed_turn: int- error: str- rejected: dict[str, str] = Field(description="path → code at the failing build")- chosen: dict[str, str] = Field(description="path → code at the next successful build")- changed_files: list[str]- n_failures: int = Field(description="consecutive failing builds folded into this pair")---def parse_trajectory_name(name: str) -> tuple[str, int | None]:- """``refine_r00`` → ``("refine", 0)``; ``zone_koi_pond_r01`` → ``("zone_koi_pond", 1)``."""- stem, _, tail = name.rpartition("_r")- if stem and tail.isdigit():- return stem, int(tail)- return name, None---def _base_commit(ws: Workspace, label: str, not_after: float | None) -> str | None:- """Latest ``pre:<label>`` commit at or before ``not_after`` (epoch seconds)."""- try:- proc = subprocess.run(- ["git", "log", "--format=%H %ct %s"], cwd=ws.root, capture_output=True, text=True, check=False, timeout=30- )- except (OSError, subprocess.TimeoutExpired):- return None- if proc.returncode != 0:- return None- for line in proc.stdout.splitlines(): # newest first- sha, _, rest = line.partition(" ")- ts, _, subject = rest.partition(" ")- if subject.strip() != f"pre:{label}":- continue- if not_after is None or float(ts) <= not_after + 1.0:- return sha- return None---def _tracked(path: str) -> bool:- parts = Path(path).parts- return bool(parts) and parts[0] in _git.CODE_ROOTS---def _normalise_path(ws: Workspace, raw: str) -> str:- p = Path(raw)- if p.is_absolute():- try:- p = p.relative_to(ws.root)- except ValueError:- return raw- return p.as_posix()---class _Replay:- """Applies write/edit tool calls to an in-memory tree; tracks whether it is exact."""-- def __init__(self, base: dict[str, str]):- self.files = dict(base)- self.exact = True # False once an edit touched a file we never saw in full- self.changed: set[str] = set()-- def write(self, path: str, content: str) -> None:- self.files[path] = content- self.changed.add(path)-- def edit(self, path: str, old: str, new: str, all_: bool) -> None:- if path not in self.files:- self.exact = False- return- text = self.files[path]- if not old or old not in text:- self.exact = False- return- self.files[path] = text.replace(old, new) if all_ else text.replace(old, new, 1)- self.changed.add(path)---def read_events(transcript: Path) -> list[dict[str, Any]]:- return read_jsonl_lenient(transcript)---def mine_trajectory(ws: Workspace, traj_dir: Path) -> list[TrajectoryRepair]:- """Repair pairs from one trajectory folder (empty for non api-agent transcripts)."""- transcript = traj_dir / "transcript.jsonl"- if not transcript.is_file():- return []- events = read_events(transcript)- calls: dict[str, dict[str, Any]] = {}- for ev in events:- if ev.get("kind") == "assistant":- for c in ev.get("tool_calls") or []:- if isinstance(c, dict) and c.get("id"):- calls[str(c["id"])] = c- if not calls:- return []- stage, rnd = parse_trajectory_name(traj_dir.name)- t0 = next((float(ev["t"]) for ev in events if "t" in ev), None)- base_sha = _base_commit(ws, stage, t0)- base: dict[str, str] = {}- if base_sha:- try:- base, _ = _git.decode_text_files(_git.read_tree_at(ws, base_sha))- except _git.GitReadError:- base = {}- rp = _Replay(base)- broken: tuple[int, dict[str, str], str, set[str]] | None = None # turn, snapshot, error, changed-before- n_fail = 0- out: list[TrajectoryRepair] = []- for ev in events:- if ev.get("kind") != "tool_result":- continue- call = calls.get(str(ev.get("call_id")))- if call is None:- continue- name, args = str(call.get("name")), dict(call.get("arguments") or {})- ok = bool(ev.get("ok"))- turn = int(ev.get("turn", -1))- if name == "write_file" and ok:- path = _normalise_path(ws, str(args.get("path", "")))- if _tracked(path):- rp.write(path, str(args.get("content", "")))- elif name == "edit_file" and ok:- path = _normalise_path(ws, str(args.get("path", "")))- if _tracked(path):- rp.edit(path, str(args.get("old", "")), str(args.get("new", "")), bool(args.get("all", False)))- elif name in BUILD_TOOLS:- if not ok:- n_fail += 1- broken = (turn, dict(rp.files), str(ev.get("content", ""))[:4000], set(rp.changed))- continue- if broken is not None and rp.exact:- b_turn, b_files, b_err, _ = broken- changed = sorted(p for p in set(b_files) | set(rp.files) if b_files.get(p) != rp.files.get(p))- if changed:- out.append(TrajectoryRepair(- trajectory=traj_dir.name, stage=stage, round_index=rnd, broken_turn=b_turn,- fixed_turn=turn, error=b_err, rejected=_limit(b_files), chosen=_limit(rp.files),- changed_files=changed, n_failures=n_fail,- ))- broken, n_fail = None, 0- return out---def _limit(files: dict[str, str], max_total: int = MAX_PAIR_CODE) -> dict[str, str]:- out: dict[str, str] = {}- total = 0- for p in sorted(files):- if total + len(files[p]) > max_total:- break- out[p] = files[p]- total += len(files[p])- return out---def mine_run(ws: Workspace) -> list[TrajectoryRepair]:- """All in-session repair pairs of one run (every ``trajectories/*`` folder)."""- root = ws.root / "trajectories"- if not root.is_dir():- return []- out: list[TrajectoryRepair] = []- for d in sorted(p for p in root.iterdir() if p.is_dir()):- out.extend(mine_trajectory(ws, d))- return outdiff --git a/harness/codeverse/gallery/model.py b/harness/codeverse/gallery/model.pyindex 962b92fc..08f4b8b0 100644--- a/harness/codeverse/gallery/model.py+++ b/harness/codeverse/gallery/model.py@@ -24,7 +24,6 @@ RunState = Literal["ok", "pending", "broken"]#: filter names the index page and ``/api/runs`` both understandFILTER_KEYS = ("q", "track", "lang", "tier", "backend", "pass", "verdict", "battery")-SORT_KEYS = ("score", "cost", "time", "complexity", "name")class RunLink(BaseModel):diff --git a/harness/codeverse/gallery/static_site.py b/harness/codeverse/gallery/static_site.pyindex a8c54550..67914626 100644--- a/harness/codeverse/gallery/static_site.py+++ b/harness/codeverse/gallery/static_site.py@@ -30,11 +30,6 @@ def render_static(index: GalleryIndex, *, title: str = "3dcv gallery", embed: botitle=title, sort=sort, view=view, note=note, extra_html=extra_html)-def write_atomic(path: Path, text: str) -> Path:- """tmp + rename, so a reader never sees a partial gallery."""- return write_text_atomic(path, text)--def build_static(roots: list[Path] | list[str], out_html: Path | str, *, title: str | None = None,embed: bool = False, thumb_px: int = THUMB_PX) -> tuple[Path, int, GalleryIndex]:"""Scan ``roots`` and write one self-contained page; ``(path, n_runs, index)``."""@@ -42,4 +37,4 @@ def build_static(roots: list[Path] | list[str], out_html: Path | str, *, title:label = title or ("3dcv gallery — " + ", ".join(s.label for s in index.sections[:4])+ ("…" if len(index.sections) > 4 else ""))html = render_static(index, title=label, embed=embed, thumb_px=thumb_px)- return write_atomic(Path(out_html), html), len(index.entries()), index+ return write_text_atomic(Path(out_html), html), len(index.entries()), indexdiff --git a/harness/codeverse/judges/metrics.py b/harness/codeverse/judges/metrics.pydeleted file mode 100644index 73a5e2bf..00000000--- a/harness/codeverse/judges/metrics.py+++ /dev/null@@ -1,60 +0,0 @@-"""Small, pure helpers over judgments and round scores (no I/O, no models)."""--from __future__ import annotations--import statistics-from collections.abc import Sequence--from pydantic import BaseModel, Field--from codeverse.contracts.judgment import Judgment---class Agreement(BaseModel):- mean: float- std: float- per_criterion_std: dict[str, float] = Field(default_factory=dict)- n: int---def judge_agreement(judgments: Sequence[Judgment]) -> Agreement:- """Mean/std of ``overall`` and per-criterion std across repeated judgments of the same thing."""- if not judgments:- raise ValueError("judge_agreement needs at least one judgment")- overalls = [j.overall for j in judgments]- crit: dict[str, list[float]] = {}- for j in judgments:- for k, v in j.scores.items():- crit.setdefault(k, []).append(float(v))- return Agreement(- mean=round(statistics.fmean(overalls), 4),- std=round(statistics.pstdev(overalls), 4) if len(overalls) > 1 else 0.0,- per_criterion_std={k: (round(statistics.pstdev(v), 4) if len(v) > 1 else 0.0) for k, v in crit.items()},- n=len(judgments),- )---def plateau(scores: Sequence[float], window: int = 2, min_delta: float = 0.02) -> bool:- """True when the best of the last ``window`` scores did not beat the best before them by ≥ ``min_delta``.-- Needs at least ``window + 1`` scores; fewer → False (keep going).- """- if window < 1:- raise ValueError("window must be ≥ 1")- if len(scores) < window + 1:- return False- before = max(scores[:-window])- recent = max(scores[-window:])- return (recent - before) < min_delta---def best_index(rounds: Sequence[tuple[float, int]]) -> int:- """Index of the best round: higher score, tie → fewer errors, tie → later round."""- if not rounds:- raise ValueError("best_index needs at least one round")- best = 0- for i, (score, n_err) in enumerate(rounds):- bs, be = rounds[best]- if score > bs or (score == bs and n_err < be) or (score == bs and n_err == be):- best = i- return bestdiff --git a/harness/codeverse/models/health.py b/harness/codeverse/models/health.pyindex 2ab3c295..b905298c 100644--- a/harness/codeverse/models/health.py+++ b/harness/codeverse/models/health.py@@ -225,36 +225,6 @@ def _is_delegating_driver(args: list[str]) -> bool:return args[i + 1 : i + 2] != ["cell"]-def sibling_processes() -> int:- """How many OTHER harness processes are running on this machine.-- ``shared_pool`` keeps one KeyPool per PROCESS, so each sibling believes it owns the- whole key quota and gets its own ``max_in_flight``. N siblings multiply the real- concurrency by N against a quota that is shared: on 2026-08-24 six concurrent- batteries took gemini-3.7-flash from 25 % success to 0 % (``docs/COST.md`` §23).-- Best-effort — reads /proc and returns 0 when it cannot tell, because a wrong number- here must never block a run.- """- import os-- me, found = os.getpid(), 0- try:- entries = list(Path("/proc").iterdir())- except OSError:- return 0- for entry in entries:- if not entry.name.isdigit() or int(entry.name) == me:- continue- try:- raw = (entry / "cmdline").read_bytes()- except OSError:- continue # exited between listing and reading- if _is_harness_argv([a for a in raw.decode(errors="replace").split("\0") if a]):- found += 1- return found--#: the measured per-process knee (docs/COST.md §20); the machine-wide budget is the same#: number, because the provider sees one machine's worth of traffic, not one process'sPOOL_KNEE = 64@@ -321,8 +291,8 @@ def pool_budget() -> PoolBudget:so the SUM of ``max_in_flight`` across processes must stay at the knee, not eachprocess alone. Counting *processes* (the first version) made every agent refuse tolaunch while any sibling existed, and on 2026-08-24 that stalled a whole A/B wave- behind two batteries that were themselves parked. Best-effort like- :func:`sibling_processes`: on any /proc trouble it reports no siblings and no usage.+ behind two batteries that were themselves parked. Best-effort: on any /proc+ trouble it reports no siblings and no usage."""import osdiff --git a/harness/codeverse/orchestrator/rounds.py b/harness/codeverse/orchestrator/rounds.pyindex bf05dd18..11555b95 100644--- a/harness/codeverse/orchestrator/rounds.py+++ b/harness/codeverse/orchestrator/rounds.py@@ -34,7 +34,6 @@ from dataclasses import dataclass, replacefrom typing import Literalfrom codeverse.contracts.run import RoundRecord-from codeverse.judges.metrics import best_indexlog = logging.getLogger(__name__)@@ -351,10 +350,22 @@ def last_gain(history: Sequence[RoundRecord]) -> float | None:return float(history[-1].score) - before+def best_index(rounds: Sequence[tuple[float, int]]) -> int:+ """Index of the best round: higher score, tie → fewer errors, tie → later round."""+ if not rounds:+ raise ValueError("best_index needs at least one round")+ best = 0+ for i, (score, n_err) in enumerate(rounds):+ bs, be = rounds[best]+ if score > bs or (score == bs and n_err < be) or (score == bs and n_err == be):+ best = i+ return best++class BestSelector:"""Best round = highest score, then fewer gate errors, then later.- ``codeverse.judges.metrics.best_index`` owns the ranking rule. A round that+ ``best_index`` above owns the ranking rule. A round thatdid not build is never picked (it has no score, and delivering code that doesnot run is never an improvement); when NO round has a score the fallback isthe built round with the fewest gate errors — later on a tie — so a round that@@ -385,7 +396,6 @@ def _build_failed(r: RoundRecord) -> bool:# Moved to codeverse.orchestrator.refine_tasks; imported here so existing importers# (tracks/*, reference/mismatch, tests) keep working unchanged.from codeverse.orchestrator.refine_tasks import ( # noqa: E402- FileForTarget,RefineTask,TaskGroup,build_refine_instructions,@@ -395,7 +405,7 @@ from codeverse.orchestrator.refine_tasks import ( # noqa: E402)__all__ = [- "BestSelector", "DEFAULT_JUDGE_SIGMA", "DETAIL_KIND", "FileForTarget", "KIND_FOR_STRATEGY",+ "BestSelector", "DEFAULT_JUDGE_SIGMA", "DETAIL_KIND", "KIND_FOR_STRATEGY","REWRITE_KIND", "RefineTask", "RoundPolicy", "StopDecision", "StopPolicy", "StopReason","Strategy", "TaskGroup", "best_score", "build_refine_instructions", "compact_instructions","detail_blocked", "gate_error_count", "judge_sigma", "kind_for_strategy", "last_gain",diff --git a/harness/codeverse/orchestrator/runner.py b/harness/codeverse/orchestrator/runner.pyindex a16c264e..a9bb241f 100644--- a/harness/codeverse/orchestrator/runner.py+++ b/harness/codeverse/orchestrator/runner.py@@ -56,10 +56,6 @@ def _jsonable(obj: Any) -> Any:return obj-class StageError(RuntimeError):- """A stage function raised; the original exception is ``__cause__``."""--class StageRunner:"""Runs stages with resume semantics on top of ``RunState`` + ``EventLog``."""diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.pyindex 3020bdee..a7c958df 100644--- a/harness/codeverse/proc.py+++ b/harness/codeverse/proc.py@@ -431,7 +431,7 @@ def read_json_or_none(path: Path | str, *, errors: str | None = None) -> dict[streplaced identical ``try: json.loads(read_text()) except (OSError, ValueError)``copies in ``flywheel/telemetry.read_json``, ``bench/complexity_report._read_json``,``gallery/index._measurement_complexity`` / ``_spec_fields``,- ``cli/main._best_round_of_record``, ``runlock._holder``,+ ``cli/inspect_cmd._best_round_of_record``, ``runlock._holder``,``languages/threejs/lint._load_plan``, ``cost/reconstruct._read_json``,``spatial/_render_common.read_json`` and ``cli/_judge.plan_summary_for`` — allthe same algorithm, differing only in whether they also checked ``isinstance(dict)``diff --git a/harness/codeverse/tracks/plan_features.py b/harness/codeverse/tracks/plan_features.pyindex c961efc7..c7a05cad 100644--- a/harness/codeverse/tracks/plan_features.py+++ b/harness/codeverse/tracks/plan_features.py@@ -1,56 +1,37 @@-"""Feature switches for the plan-loop-engineering wave, one env variable for all of them.--Every change in that wave (the plan fit check, the rendered contact graph, the brief-attachment graph, …) ships OFF and is A/B'd one at a time with ``bench/ab_plan.py``,-whose arms differ only by environment. So the switch has to be an env variable, it-has to be read at call time (never cached in ``Settings``), and every change has to-read the SAME variable — a second spelling per change is how a control arm quietly-turns into a variant.--``CV3D_PLAN_FEATURES="fit,contacts"`` turns two features on; ``all`` turns every known-one on and ``all,-graph`` all but one. Unknown names are ignored (and logged once)-rather than fatal: a typo in a bench command must produce a control run, not a crash-half-way through a battery.--STATUS (CQ-5): none of the six features below is implemented, and nothing outside this-module reads ``CV3D_PLAN_FEATURES`` — the plan-loop changes that DID land each invented-their own spelling instead (see :data:`LIVE_SWITCHES`). The switch is therefore DEAD,-and an A/B whose arms differ only by it is a no-op by construction: one such run is on-record printing "keep, mean delta +0.344" for two byte-identical arms. :data:`DEAD_SWITCHES`-exists so ``bench/ab_plan.py`` refuses that A/B instead of producing a verdict. Delete-this module, or implement a feature and move its variable into ``LIVE_SWITCHES``.+"""The A/B switch registry: which env switches any code path ACTUALLY reads.++Born as the switchboard for the plan-loop-engineering wave (CQ-5): six features were+declared under ``CV3D_PLAN_FEATURES``, none was ever implemented, and one A/B run is on+record printing "keep, mean delta +0.344" for two byte-identical arms. The feature+runtime (``parse_features`` / ``plan_feature_on``) was deleted 2026-08-28 per its own+instruction; the six names stay only so ``pin_plan_blockers`` can classify a pinned+A/B, and what else remains is the guard rail — :data:`LIVE_SWITCHES` / :data:`DEAD_SWITCHES` (kept honest by+``tests/orchestrator_tracks/test_plan_features.py``, which greps the tree) and the+``--pin-plan`` blocker rule ``bench/ab_plan.py`` consults so a paired A/B cannot+silently pin away the very thing it is testing."""from __future__ import annotations-import logging-import os--log = logging.getLogger(__name__)-PLAN_FEATURES_ENV = "CV3D_PLAN_FEATURES"-#: the switches the wave defines, in the order the design doc lists them-#: (``scratchpad/planloop/design/DESIGN.md``). Add a name here in the same commit as-#: the code that reads it; a feature that is not listed cannot be turned on.-FIT = "fit" # plan-level fit check: undeclared deep overlaps, declared gaps → one repair re-ask-CONTACTS = "contacts" # every declared contact rendered to the builder with its mating faces and overlap rule-GRAPH = "graph" # brief sub-assemblies carry `mounts_on` edges; parts are grouped by node-GRAPH_BUDGET = "graph_budget" # plan budget derived from the brief graph instead of the flat floor-GRAPH_EXAMPLE = "graph_example" # the planner's worked example written as a graph (touches + assembly filled)-CONSISTENCY = "consistency" # plan overall bbox checked against the brief's dimension rows+#: the six switch names the wave defined. Nothing reads them at runtime any more —+#: they remain because ``pin_plan_blockers`` still classifies a CV3D_PLAN_FEATURES+#: value inside a MIXED A/B env, and that classification must not silently widen.+FIT = "fit"+CONTACTS = "contacts"+GRAPH = "graph"+GRAPH_BUDGET = "graph_budget"+GRAPH_EXAMPLE = "graph_example"+CONSISTENCY = "consistency"KNOWN_FEATURES: tuple[str, ...] = (FIT, CONTACTS, GRAPH, GRAPH_BUDGET, GRAPH_EXAMPLE, CONSISTENCY)--#: Which stage a switch changes. This is not documentation — ``bench/ab_plan.py --pin-plan``-#: reads it, and pinning a PLAN-side switch would silently disable the very thing under test-#: (both arms would share one plan, so a change that only alters planning becomes a no-op the-#: rig would then report as "no effect"). A switch missing from here is treated as plan-side:-#: refusing to pin costs one noisy A/B, pinning wrongly costs a confident wrong answer.-#: docs/EVAL.md §8.1 measured why pinning matters — the A/A's worst pair differed 1 part vs 10.+#: which stage each switch would have changed — read by ``pin_plan_blockers``PLAN_SIDE: frozenset[str] = frozenset({FIT, GRAPH, GRAPH_BUDGET, GRAPH_EXAMPLE, CONSISTENCY})GENERATION_SIDE: frozenset[str] = frozenset({CONTACTS})-#: env switches outside CV3D_PLAN_FEATURES, same rule-PLAN_SIDE_ENV: frozenset[str] = frozenset({"CV3D_PLAN_BRIEF", "CV3D_SCOPED_PARTS"})+#: env switches that act AFTER planning — ``--pin-plan`` may share one plan across arms+#: that differ only by these. Anything not listed is treated as plan-side: refusing to+#: pin costs one noisy A/B, pinning wrongly costs a confident wrong answer+#: (docs/EVAL.md §8.1 measured it — the A/A's worst pair differed 1 part vs 10).GENERATION_SIDE_ENV: frozenset[str] = frozenset({"CV3D_SKILLS", "CV3D_SKILLS_MAX", "CV3D_SKILLS_UNVERIFIED","CV3D_SKILLS_ONLY","CV3D_DETAIL_ROUNDS", "CV3D_REFERENCE_DIFF",@@ -111,49 +92,6 @@ def dead_env_keys(env: dict[str, str]) -> list[str]:return sorted(k for k in env if k in DEAD_SWITCHES)-_warned: set[str] = set()---def parse_features(raw: str | None) -> frozenset[str]:- """``"fit, contacts"`` → ``{fit, contacts}``; ``"all,-graph"`` → everything but graph.-- Pure, so it is testable without touching the environment. Unknown names are- dropped and logged once per process (see the module docstring for why not fatal)."""- if not raw or not raw.strip():- return frozenset()- on: set[str] = set()- for tok in (t.strip().lower() for t in raw.split(",")):- if not tok:- continue- negate = tok.startswith("-")- name = tok[1:] if negate else tok- names = set(KNOWN_FEATURES) if name == "all" else {name}- unknown = names - set(KNOWN_FEATURES)- for u in unknown - _warned:- _warned.add(u)- log.warning("%s: unknown plan feature %r ignored (known: %s)", PLAN_FEATURES_ENV, u, ", ".join(KNOWN_FEATURES))- names -= unknown- if negate:- on -= names- else:- on |= names- return frozenset(on)---def plan_features() -> frozenset[str]:- """The features switched on for this process, read from the environment NOW."""- return parse_features(os.environ.get(PLAN_FEATURES_ENV))---def plan_feature_on(name: str) -> bool:- """Is feature ``name`` on? Raises on a name the wave does not define, because a- caller asking about a feature that cannot be switched on is a bug, not a config."""- if name not in KNOWN_FEATURES:- raise ValueError(f"unknown plan feature {name!r}; known: {', '.join(KNOWN_FEATURES)}")- return name in plan_features()---__all__ = ["CONSISTENCY", "CONTACTS", "DEAD_SWITCHES", "FIT", "GENERATION_SIDE", "GENERATION_SIDE_ENV",- "GRAPH", "GRAPH_BUDGET", "GRAPH_EXAMPLE", "PLAN_SIDE", "PLAN_SIDE_ENV", "pin_plan_blockers",- "KNOWN_FEATURES", "LIVE_SWITCHES", "PLAN_FEATURES_ENV", "dead_env_keys", "parse_features",- "plan_feature_on", "plan_features"]+__all__ = ["CONSISTENCY", "CONTACTS", "DEAD_SWITCHES", "FIT", "GENERATION_SIDE",+ "GENERATION_SIDE_ENV", "GRAPH", "GRAPH_BUDGET", "GRAPH_EXAMPLE", "KNOWN_FEATURES",+ "LIVE_SWITCHES", "PLAN_FEATURES_ENV", "PLAN_SIDE", "dead_env_keys", "pin_plan_blockers"]diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.mdindex d974fea5..e2f0a101 100644--- a/harness/docs/ARCHITECTURE.md+++ b/harness/docs/ARCHITECTURE.md@@ -24,11 +24,11 @@ layered on top of the language — for four **tracks**:…driven by **any** of these backends behind two protocols:* `ChatModel` (API): `gemini:*`, `anthropic:*`, `openai:*` — planner, judge,- captioner, single-shot generation, and the in-process `api-agent`.-* `CodingAgent` (agentic session on a workspace): `gemini-cli:*`, `claude-code:*`,- `codex:*`, `agy:*` (Antigravity), `api-agent:<chat-model>` (our own tool loop so- API-only users get parity). Plus `single-shot:<chat-model>` (one envelope of- files, no tools) handled inside `tracks/generation.py`.+ captioner, single-shot generation.+* `CodingAgent` (agentic session on a workspace): always a vendor CLI —+ `gemini-cli:*`, `claude-code:*`, `codex:*`, `agy:*` (Antigravity); the in-process+ `api-agent` was deleted 2026-08-28. Plus `single-shot:<chat-model>` (one envelope+ of files, no tools) handled inside `tracks/generation.py`.Every run is: **spec → plan → skeleton → [scene stages] → baseline round(optionally best-of-N candidates) → refine rounds → finalise → record**, with an@@ -100,10 +100,9 @@ codeverse/fallback), retry.py (MAX_WAIT_S: no single wait > 3 s), storm.py (shared 503 gate —measured, ships OFF, see docs/COST.md §21), health.py (preflight probe: is the modelserving? no retries, no backoff), schema_utils.py (strict schema), registry.py- agents/ CodingAgent; gemini_cli.py claude_code.py codex.py antigravity.py api_agent.py- (+ api_tools.py run_shell policy, api_skills.py: the read_skill tool every- backend WITHOUT a native loader gets — measured, prose in a system prompt is- not an affordance), materialize.py, cli_common.py (sessions, retry+ agents/ CodingAgent; gemini_cli.py claude_code.py codex.py antigravity.py+ (api_agent.py + api_tools.py + api_skills.py deleted 2026-08-28 —+ the vendors already ship the loop), materialize.py, cli_common.py (sessions, retrytrajectory naming, files_changed attribution), watchdog.py, transcript.py, registry.pylanguages/ LanguageRuntime; blender/ (multi-file: layout.py, model.py + parts/*.py) cadquery/threejs/ (+ templates.py) urdf/ scene_threejs/ glsl_shader/ (wrap.py header+line-map)@@ -178,8 +177,7 @@ codeverse/skills_hook.py (the round's view of codeverse/skills: attach before generating,probe reads after — a no-op unless CV3D_SKILLS is on)flywheel/ record.py, export.py, pack.py, sample.py, pairs.py, migrate.py (schema moves),- deliverable.py, telemetry.py, trajectories.py (repair-pair- mining), captions.py, quality.py (tiers + dedupe), dedupe.py, index.py,+ deliverable.py, telemetry.py, captions.py, quality.py (tiers + dedupe), dedupe.py, index.py,code_quality.py (the delivered CODE's own vector — magic numbers per 100 LOC,function length, dead functions, duplication, docstrings → record.extra["code_quality"].index, a flywheel filter beside score and complexity)@@ -295,8 +293,8 @@ summary:`@tool(name, ArgsModel, description, *, tracks=(), languages=(), cost_hint)` registers`fn(ctx, args) -> Observation` → (a) direct call from tracks, (b) the stdio MCP-server (name `3dcv`) for gemini-cli / claude-code / codex, (c) native tool schema-for `api-agent`, (d) a prompt card. Tools: `build`, `measure`, `render_views`,+server (name `3dcv`) for the vendor CLIs, (c) a native tool schema for any embedder+(`ToolDef.schema()`), (d) a prompt card. Tools: `build`, `measure`, `render_views`,`render_sheet`, `isolate`, `cross_section`, `check_connectivity`, `check_contract`,`compare_silhouette`, `joint_sweep` (articulated), `shader_probe`, `scene_probe`,`scene_views` (scene), `gl_probe`, `gl_frames` (graphics), `texture_pass`,@@ -437,8 +435,7 @@ $0.02–0.03 (flash) / ~$0.2 (pro); api-agent generation 3–6 min per object ro* Scenes: fps is a relative cost; camera-in-geometry can miss open-back enclosures.* threejs: textures are stripped on GLB export (the texture pass re-adds them as aderived pack); `userData.tick` cannot survive export.-* Trajectory repair mining only understands api-agent transcripts; CLI backends- produce raw stdout. agy exposes no per-workspace MCP, cost or served model.+* agy exposes no per-workspace MCP, cost or served model.* Anthropic / OpenAI backends are mock-tested only (no keys on this box).* Budget checks run between steps: a refine round that finishes its judge and thentrips the budget is not promoted to best — give scenes `--max-minutes 60 --max-usd 4`.@@ -456,9 +453,9 @@ passed, C best ≥ 0.6, D else), acceptance checklists, gate summaries and`(code fingerprint, prompt)` dedupe (`--drop-duplicates`; side-car captions via`--captions-dir`); `metadata.parquet`/`.jsonl` + sqlite index carry tier, gateerrors, cost, fingerprints, `duplicate_of`; `--pack` tars with byte-range locators.-`flywheel pairs` emits preference pairs (round i < j by judge Δ ≥ τ), round-level-repair pairs and **in-session repair pairs mined from api-agent transcripts**-(replay-verified against the git snapshots); `flywheel caption` adds+`flywheel pairs` emits preference pairs (round i < j by judge Δ ≥ τ) and round-level+repair pairs (in-session trajectory mining died with the api-agent, 2026-08-28 —+vendor CLIs log raw stdout, not structured tool turns); `flywheel caption` adds{detailed, instruction, factory} captions (image-grounded, brand-free, `--out` forside-car mode); `flywheel gallery` is an alias of `3dcv gallery build --embed`(the flywheel package has no renderer of its own).diff --git a/harness/docs/COST.md b/harness/docs/COST.mdindex 171fd3c0..783f1f47 100644--- a/harness/docs/COST.md+++ b/harness/docs/COST.md@@ -1233,7 +1233,7 @@ guard is now `len(failed_keys) < len(pool)`.-**Follow-up, same day, from the time audit (51 storm-day runs vs 52 baseline; `time_audit/REPORT.md`).**+**Follow-up, same day, from the time audit (51 storm-day runs vs 52 baseline; scripts retired 2026-08-28 — findings preserved in `docs/TIME_AUDIT_2026-08-26.md`).**Three accelerations, all additive and on by default:*A caller clips the retry budget to what it can afford* (`ChatRequest.max_wait_s`, None = thediff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.mdindex 54140eab..55f4f0a8 100644--- a/harness/docs/INTERFACES.md+++ b/harness/docs/INTERFACES.md@@ -12,7 +12,7 @@ file was reconciled against it on 2026-08-23 (waves 2–3 + fix batch 1).* Chat models: `<provider>:<model>` — `gemini:*` · `anthropic:*` · `openai:*`(`models.registry.parse_model_id`).* Coding agents: `<kind>:<model>` — `gemini-cli:*` · `claude-code:*` · `codex:*` ·- `agy:*` · `api-agent:<provider>:<model>` (`agents.registry.parse_agent_id`).+ `agy:*` (`agents.registry.parse_agent_id`).* **Δ** Single-shot generation ids: `single-shot:<provider>:<model>` — handled onlyinside `tracks/generation.py` (`is_single_shot`, `single_shot_model_id`); neverregistered in `agents/`. `Spec.backends.generator` accepts any of the three.@@ -107,7 +107,7 @@ with run_ledger(ws.root, run=slug): # binds the run, points record_call a# models.get_chat_model returns a MeteredChatModel → ONE row per ChatModel.generate# agents.get_coding_agent returns a MeteredAgent → ambient round/stage for the session, the# profile's turn cap (only ever LOWERS job.max_turns), and — iff NOT meters_own_calls(agent),- # i.e. the backend is not in IN_PROCESS_AGENT_KINDS = {"api-agent"} — one session row.+ # a declaration no shipped backend makes — one session row.# The rule is the BACKEND, never "did a row get written while it ran": a CLI session with one# in-process tool call used to be dropped entirely, an in-process session whose turns ran in# another thread used to be counted twice. A backend may declare `meters_own_calls`.@@ -118,7 +118,7 @@ from codeverse.cost.context import CallContext, call_context, bind_run, context_# texture pass that bills a model INSIDE an agent session is filed under its own stage, not the# session's; a generation label yields to the session (best-of-N: kind="candidate" beats# label="baseline"); Stage.OTHER means "the label said nothing" and displaces nothing.- # labels understood: api-agent:<job label>:t<turn> · judge:<rubric>:r<NN>:s<k> · planner · pairwise:… · texture… · caption…+ # labels understood: judge:<rubric>:r<NN>:s<k> · planner · pairwise:… · texture… · caption… (api-agent:<label>:t<turn> in historical ledgers only)from codeverse.cost.caching import Block, order_blocks, prefix_report, session_cache, session_keysession_cache(rows) -> [SessionCache] # per session: cold first call, cached share, saved_usd, cold_usdfrom codeverse.cost.profiles import get_profile, PROFILES # economy | balanced | quality@@ -167,10 +167,9 @@ one workspace; harness-owned paths and sibling sessions' hinted files are droppethe entry file), `read_only: list[str]` (harness-owned files INSIDE the write roots thesession may read but never write — `contracts.common.HARNESS_OWNED_SRC[language]`, i.e.`src/recipes.glsl` for glsl_shader; `tracks/generation.py` sets it on every job of the-run and `FileTools(read_only=…)` answers a `write_file` / `edit_file` on one with-`PathDenied("… is harness-owned and read-only … call them from your own files")`)-and `job.api: ApiAgentOptions(max_usd, temperature, thinking, allow_shell)` (api-agent-only). **Δ legacy lift**: the same keys passed inside `extra={...}` are lifted into+run and `agents/cli_common._enforce_scope` reverts a post-session write to one and+fails that session). **Δ legacy lift**: the job keys (`round`, `kind`, `language`,+`track`, `files_hint`, `mcp_command`) passed inside `extra={...}` are lifted intothe typed fields at validation (extra itself is left untouched), so old constructorsand serialized jobs keep working; `extra` stays for one-off backend hints.Trajectories: `ws/trajectories/<label>_rNN/{prompt.md, transcript.jsonl, stdout.json,@@ -185,14 +184,6 @@ applied LAST and `mcp.allowed` replaces, so an agent-planted server in ws/.gemincached; `tokens.input` is the uncached count) and each served model is priced at itsown rate. KeyPoolExhausted never escapes `run()` (→ `exit_reason=budget`); retriesprefer a different key (10 s wait) before re-using the same one.-api-agent tools: `read_file`, `write_file`, `edit_file`, `list_files`, `run_shell`-+ every spatial tool (`build` keeps its real name; `run_build` is an alias).-**Δ** `run_shell` is a *policy filter*, not an OS sandbox: allow-listed programs only,-no inline-code/REPL/loader flags (`-c/-e/-p/-i/--eval/--require/--import/…`),-`python -m` restricted to a small allow-list, every path-like argument-realpath-confined to the workspace (and never `.git/`). `FileTools.call` never-raises — bad arguments / non-UTF-8 files come back as `ToolOutcome(is_error=True)`.-## languages/```pythonfrom codeverse.languages import get_runtime # (Language | str) -> LanguageRuntime@@ -436,8 +427,7 @@ from codeverse.flywheel.export import export_samples # (runs_dir, out_dir, *,# overwrite=True, include_unbuilt=False, captions_dir=None, drop_duplicates=False) -> ExportReport{…, n_duplicates, duplicates, tiers}from codeverse.flywheel.quality import quality_tier, prompt_hash, find_duplicates # tiers: A passed & 0 gate errors, B passed,# C best ≥ 0.6, D else; dedupe = (code fingerprint, prompt)-from codeverse.flywheel.pairs import build_pairs # (runs_dir, out_jsonl, *, min_delta=0.05, trajectories=True) -> n-from codeverse.flywheel.trajectories import mine_run # in-session repair pairs from api-agent transcripts (replay-verified)+from codeverse.flywheel.pairs import build_pairs # (runs_dir, out_jsonl, *, min_delta=0.05) -> nfrom codeverse.flywheel.captions import caption_sample # Δ (ws, record, model_id, *, model=None, out_dir=None) -> Captions;# out_dir → side-car <out_dir>/<slug>.json, run untouchedfrom codeverse.gallery import build_index, default_roots, build_static, serve, GalleryApp # THE local gallerydiff --git a/harness/docs/RUNBOOK.md b/harness/docs/RUNBOOK.mdindex ca20d366..0c45b9cd 100644--- a/harness/docs/RUNBOOK.md+++ b/harness/docs/RUNBOOK.md@@ -155,7 +155,7 @@ unless you type `--host` yourself; it never serves a path outside the declared r3dcv tools list [--cards] · 3dcv tools measure --workspace runs/<slug> · 3dcv tools gl_frames --workspace … --json '{"times":[0,1,2.5]}'3dcv flywheel export runs/ dataset/ [--min-score 0.7] [--only-passed] [--pack] [--include-unbuilt][--captions-dir caps/] [--drop-duplicates]-3dcv flywheel pairs runs/ pairs.jsonl [--min-delta 0.05] [--no-trajectories]+3dcv flywheel pairs runs/ pairs.jsonl [--min-delta 0.05]3dcv flywheel caption <slug> [--model …] [--out caps/] # --out = side-car mode, run untouched3dcv flywheel gallery runs/ gallery.html [--title …] # alias of `3dcv gallery build --embed` (§4)3dcv flywheel index runs/ runs_index.sqlite · 3dcv flywheel dedupe dataset/diff --git a/harness/docs/SKILLS.md b/harness/docs/SKILLS.mdindex ce390384..24d2fa04 100644--- a/harness/docs/SKILLS.md+++ b/harness/docs/SKILLS.md@@ -10,7 +10,9 @@ names the cookbook section to fetch. Two libraries of truth would be worse than`SKILL.md` is an open standard (agentskills.io, Dec 2025), which is why the format below isnot ours to bend: **claude-code, codex, gemini-cli and agy discover these bundles-themselves** — proven live, see §7. Only `api-agent` needs harness-side injection.+themselves** — proven live, see §7. (The in-process `api-agent`, the one backend that+needed harness-side injection, was deleted 2026-08-28; an unclassified backend still+gets the routed index as the safe default.)Everything is behind `CV3D_SKILLS`. **It ships OFF**, because `api-agent` read 0 of 5routed bundles in the measured A/B while the three subscription CLIs read all five.@@ -32,7 +34,6 @@ one file, `skills/delivery.py`:|---|---|---|| `claude-code` | yes | `.claude/skills` || `codex`, `gemini-cli`, `agy` | yes | `.agents/skills` |-| `api-agent` | no | `.agents/skills` || *anything unclassified* | **no** (safe default) | `.agents/skills` |Both bits were read out of the shipped binaries, not assumed. Everything else is derived:@@ -203,7 +204,7 @@ answerable. Per session, `telemetry/skills.jsonl` gets one line.3dcv doctor --skills # library + discovery wiring```-Targets: ≥ 60% deep-read for CLI backends, ≥ 80% for api-agent, counted only over sessions+Targets: ≥ 60% deep-read for CLI backends (the api-agent's 80% target died with it, 2026-08-28), counted only over sessionsthe control (§1) says were measurable. **A skill under 20% over 20 such sessions is mergedor deleted.** A library that only ever grows is how this ends as bloat.The read rate is the FIRST readout, before any score: `read_cookbook` was called by 0 of 16diff --git a/harness/tests/cost/test_guard_caching_routing.py b/harness/tests/cost/test_guard_caching_routing.pyindex 8f7c7306..4782eceb 100644--- a/harness/tests/cost/test_guard_caching_routing.py+++ b/harness/tests/cost/test_guard_caching_routing.py@@ -8,9 +8,7 @@ import pytestfrom codeverse.contracts.common import Usagefrom codeverse.cost import (Block,- CostGuard,cache_efficiency,- cheapest_affordable,estimate_call,order_blocks,prefix_report,@@ -43,28 +41,6 @@ def test_estimate_of_an_image_model_uses_the_per_image_price():assert est.usd == pytest.approx(500 * 0.5 / 1e6 + 2 * 0.067)-def test_guard_blocks_only_what_it_can_price():- guard = CostGuard(budget_usd=0.10, reserve_usd=0.02)- cheap = estimate_call("gemini:gemini-3.7-flash", input_tokens=10_000, output_tokens=500)- allow, why = guard.check(cheap)- assert allow and not why- guard.commit(cheap.usd)- dear = estimate_call("gemini:gemini-3.1-pro-preview", input_tokens=2_000_000, output_tokens=50_000)- allow, why = guard.check(dear)- assert not allow and "left of" in why and guard.rejected- unknown = estimate_call("gemini:no-such-model", input_tokens=10_000_000, output_tokens=1)- assert guard.check(unknown)[0] is True # unpriceable ≠ blocked---def test_cheapest_affordable_picks_a_tier():- pick = cheapest_affordable(- ["gemini:gemini-3.1-pro-preview", "gemini:gemini-3.7-flash", "gemini:gemini-3.1-flash-lite"],- input_tokens=50_000, output_tokens=2_000)- assert pick is not None and pick[0] == "gemini:gemini-3.1-flash-lite"- assert cheapest_affordable(["gemini:gemini-3.1-pro-preview"], input_tokens=10_000_000,- output_tokens=0, budget_usd=0.001) is None--# ------------------------------------------------------------------ cachingdef test_order_blocks_puts_the_stable_prefix_first():blocks = [Block("round", "round 3 of 4", order=1),diff --git a/harness/tests/flywheel_cli/test_quality_gallery.py b/harness/tests/flywheel_cli/test_quality_gallery.pyindex 291ea64f..f2efe3b8 100644--- a/harness/tests/flywheel_cli/test_quality_gallery.py+++ b/harness/tests/flywheel_cli/test_quality_gallery.py@@ -1,4 +1,4 @@-"""Quality tiers, (code, prompt) dedupe, richer meta.json, captions side-car, gallery, trajectory repair mining."""+"""Quality tiers, (code, prompt) dedupe, richer meta.json, captions side-car, gallery."""from __future__ import annotations@@ -8,10 +8,8 @@ from pathlib import Pathfrom codeverse.contracts.common import Languagefrom codeverse.flywheel.captions import caption_samplefrom codeverse.flywheel.export import export_samples, load_captions-from codeverse.flywheel.pairs import build_pairsfrom codeverse.flywheel.quality import find_duplicates, mark_duplicates, prompt_hash, quality_tierfrom codeverse.flywheel.record import load_record-from codeverse.flywheel.trajectories import mine_run, parse_trajectory_namefrom codeverse.gallery import (GalleryIndex,RootSection,@@ -20,7 +18,6 @@ from codeverse.gallery import (build_static,render_static,)-from codeverse.workspace import Workspacefrom tests.flywheel_cli.conftest import make_fake_runfrom tests.flywheel_cli.test_captions import GOOD, FakeModel@@ -172,75 +169,3 @@ def test_gallery_from_runs(runs_dir: Path, tmp_path: Path):RunEntry(battery="t", slug="y", path="", state="broken", error="boom <b>")])html = render_static(GalleryIndex(sections=[section]), title="t")assert "no render" in html and "boom <b>" in html---# --------------------------------------------------------------------------- trajectory repair pairs---def _write_transcript(ws: Workspace, label: str, rnd: int, events: list[dict]) -> Path:- d = ws.root / "trajectories" / f"{label}_r{rnd:02d}"- d.mkdir(parents=True, exist_ok=True)- (d / "transcript.jsonl").write_text("\n".join(json.dumps(e) for e in events) + "\n")- return d---def _call(turn: int, t: float, cid: str, name: str, **args) -> dict:- return {"t": t, "kind": "assistant", "turn": turn, "text": "", "tool_calls": [{"id": cid, "name": name, "arguments": args}]}---def _result(turn: int, t: float, cid: str, name: str, ok: bool, content: str) -> dict:- return {"t": t, "kind": "tool_result", "turn": turn, "name": name, "call_id": cid, "ok": ok, "content": content}---def test_trajectory_repair_mining(fake_run, tmp_path: Path):- ws, rec = fake_run- assert parse_trajectory_name("zone_koi_pond_r01") == ("zone_koi_pond", 1)- # the base state = the `pre:baseline` snapshot the harness takes before the session- (ws.src / "model.py").write_text("import bpy\nA = 1\n")- ws.commit("pre:baseline")- import time-- t0 = time.time() + 5 # transcript starts after the snapshot- ev = [- {"t": t0, "kind": "system", "text": "rules"},- _call(0, t0 + 1, "c1", "edit_file", path="src/model.py", old="A = 1", new="A = 2)"),- _result(0, t0 + 1, "c1", "edit_file", True, "edited"),- _call(1, t0 + 2, "c2", "build"), _result(1, t0 + 2, "c2", "build", False, "build failed: SyntaxError"),- _call(2, t0 + 3, "c3", "write_file", path="src/model.py", content="import bpy\nA = 3\nB = 4\n"),- _result(2, t0 + 3, "c3", "write_file", True, "overwrote"),- _call(3, t0 + 4, "c4", "build"), _result(3, t0 + 4, "c4", "build", False, "build failed: NameError"),- _call(4, t0 + 5, "c5", "edit_file", path="src/model.py", old="B = 4", new="B = A"),- _result(4, t0 + 5, "c5", "edit_file", True, "edited"),- _call(5, t0 + 6, "c6", "build"), _result(5, t0 + 6, "c6", "build", True, "BUILD OK"),- _call(6, t0 + 7, "c7", "build"), _result(6, t0 + 7, "c7", "build", True, "BUILD OK"), # no new pair- ]- _write_transcript(ws, "baseline", 0, ev)- pairs = mine_run(ws)- assert len(pairs) == 1- p = pairs[0]- assert p.trajectory == "baseline_r00" and p.stage == "baseline" and p.round_index == 0- assert p.broken_turn == 3 and p.fixed_turn == 5 and p.n_failures == 2 and "NameError" in p.error- assert p.rejected["src/model.py"] == "import bpy\nA = 3\nB = 4\n" and p.chosen["src/model.py"] == "import bpy\nA = 3\nB = A\n"- assert "src/parts/leg.py" in p.chosen and p.changed_files == ["src/model.py"] # untouched files ride along- # a CLI-backend transcript (raw lines) yields nothing- _write_transcript(ws, "refine", 0, [{"t": t0, "kind": "invoke", "argv": ["gemini"]}, {"t": t0, "kind": "line", "text": "x"}])- assert len(mine_run(ws)) == 1- out = tmp_path / "pairs.jsonl"- n = build_pairs(ws.root.parent, out)- kinds = [(json.loads(line)["kind"], json.loads(line).get("source", "")) for line in out.read_text().splitlines()]- assert n == 2 and ("repair", "trajectory") in kinds and ("preference", "") in kinds- assert build_pairs(ws.root.parent, tmp_path / "p2.jsonl", trajectories=False) == 1---def test_trajectory_inexact_base_is_skipped(fake_run):- """An edit on a file we never saw in full (no pre: commit, no write_file) must not produce a pair."""- ws, rec = fake_run- t0 = 1.0- ev = [- _call(0, t0, "c1", "edit_file", path="src/model.py", old="zzz", new="y"), _result(0, t0, "c1", "edit_file", True, "ok"),- _call(1, t0, "c2", "build"), _result(1, t0, "c2", "build", False, "fail"),- _call(2, t0, "c3", "edit_file", path="src/model.py", old="y", new="w"), _result(2, t0, "c3", "edit_file", True, "ok"),- _call(3, t0, "c4", "build"), _result(3, t0, "c4", "build", True, "ok"),- ]- _write_transcript(ws, "baseline", 0, ev)- assert mine_run(ws) == []diff --git a/harness/tests/judges/test_metrics.py b/harness/tests/judges/test_metrics.pydeleted file mode 100644index cf0dc47a..00000000--- a/harness/tests/judges/test_metrics.py+++ /dev/null@@ -1,32 +0,0 @@-import pytest--from codeverse.contracts.judgment import Judgment-from codeverse.judges.metrics import best_index, judge_agreement, plateau---def _j(o, **sc):- return Judgment(rubric="r", scores=sc or {"a": o}, overall=o, passed=o > 0.7)---def test_agreement():- a = judge_agreement([_j(0.7, a=0.6, b=0.8), _j(0.9, a=0.8, b=1.0)])- assert a.mean == 0.8 and a.std == pytest.approx(0.1) and a.per_criterion_std == {"a": 0.1, "b": 0.1} and a.n == 2- assert judge_agreement([_j(0.5)]).std == 0.0- with pytest.raises(ValueError):- judge_agreement([])---def test_plateau():- assert plateau([0.5, 0.6]) is False # too few- assert plateau([0.5, 0.6, 0.61, 0.6]) is True- assert plateau([0.5, 0.6, 0.61, 0.7]) is False- assert plateau([0.5, 0.52, 0.53], window=2, min_delta=0.02) is False # 0.53-0.5 = 0.03 ≥ 0.02- assert plateau([0.5, 0.51, 0.515], window=2, min_delta=0.02) is True---def test_best_index():- assert best_index([(0.5, 0), (0.7, 2), (0.7, 1), (0.6, 0)]) == 2- assert best_index([(0.7, 1), (0.7, 1)]) == 1 # later wins ties- assert best_index([(0.9, 3), (0.8, 0)]) == 0- with pytest.raises(ValueError):- best_index([])diff --git a/harness/tests/models/test_health.py b/harness/tests/models/test_health.pyindex 0dcb3bf0..84daccb6 100644--- a/harness/tests/models/test_health.py+++ b/harness/tests/models/test_health.py@@ -102,19 +102,6 @@ def test_sibling_detection_matches_arguments_not_the_repo_path():assert not _is_harness_argv(["python", "-c", "print(1)", "/home/u/.local/bin/3dcv"])-def test_sibling_count_never_raises(monkeypatch):- """A wrong number here must never block a run, so /proc trouble returns 0."""- from pathlib import Path-- import codeverse.models.health as health-- def boom(_self):- raise OSError("no /proc on this platform")-- monkeypatch.setattr(Path, "iterdir", boom)- assert health.sibling_processes() == 0--def test_probe_model_treats_a_503_as_final(monkeypatch):""" "No retries" must include the storm branch. rotate_with_retries' capacity-stormbranch does NOT consume max_attempts, so max_attempts=1 alone still retried a 503diff --git a/harness/tests/orchestrator_tracks/test_best_index.py b/harness/tests/orchestrator_tracks/test_best_index.pynew file mode 100644index 00000000..c9077957--- /dev/null+++ b/harness/tests/orchestrator_tracks/test_best_index.py@@ -0,0 +1,16 @@+"""``orchestrator.rounds.best_index`` — the ranking rule BestSelector delegates to+(moved here from tests/judges/test_metrics.py when judges/metrics.py was deleted)."""++from __future__ import annotations++import pytest++from codeverse.orchestrator.rounds import best_index+++def test_best_index():+ assert best_index([(0.5, 0), (0.7, 2), (0.7, 1), (0.6, 0)]) == 2+ assert best_index([(0.7, 1), (0.7, 1)]) == 1 # later wins ties+ assert best_index([(0.9, 3), (0.8, 0)]) == 0+ with pytest.raises(ValueError):+ best_index([])diff --git a/harness/tests/orchestrator_tracks/test_plan_features.py b/harness/tests/orchestrator_tracks/test_plan_features.pyindex 64bca37a..bdccd73e 100644--- a/harness/tests/orchestrator_tracks/test_plan_features.py+++ b/harness/tests/orchestrator_tracks/test_plan_features.py@@ -1,5 +1,5 @@-"""``tracks/plan_features.py``: the plan-loop switch registry, and the guard that keeps-it honest about which switches any code actually reads (CQ-5)."""+"""``tracks/plan_features.py``: the switch registry that keeps A/B rigs honest about+which env switches any code actually reads (CQ-5)."""from __future__ import annotations@@ -9,44 +9,6 @@ import pytestfrom codeverse.tracks import plan_features as F--def test_empty_means_nothing_on(monkeypatch):- monkeypatch.delenv(F.PLAN_FEATURES_ENV, raising=False)- assert F.plan_features() == frozenset()- assert not F.plan_feature_on(F.FIT)- monkeypatch.setenv(F.PLAN_FEATURES_ENV, " ")- assert F.plan_features() == frozenset()---def test_parse_names_case_and_spaces():- assert F.parse_features("fit, Contacts ,") == {F.FIT, F.CONTACTS}---def test_all_and_negation():- assert F.parse_features("all") == frozenset(F.KNOWN_FEATURES)- assert F.parse_features("all,-graph") == frozenset(F.KNOWN_FEATURES) - {F.GRAPH}- # order matters: a later positive re-enables- assert F.GRAPH in F.parse_features("-graph,graph")---def test_unknown_names_are_dropped_not_fatal(caplog):- with caplog.at_level("WARNING"):- assert F.parse_features("fit,typo") == {F.FIT}- assert any("typo" in r.message for r in caplog.records)---def test_reads_env_at_call_time(monkeypatch):- monkeypatch.setenv(F.PLAN_FEATURES_ENV, "fit")- assert F.plan_feature_on(F.FIT) and not F.plan_feature_on(F.CONTACTS)- monkeypatch.setenv(F.PLAN_FEATURES_ENV, "contacts")- assert F.plan_feature_on(F.CONTACTS) and not F.plan_feature_on(F.FIT)---def test_asking_about_an_undefined_feature_is_a_bug():- with pytest.raises(ValueError):- F.plan_feature_on("not_a_feature")--# --------------------------------------------------------------------------- CQ-5HARNESS = Path(__file__).resolve().parents[2]