Skip to content

Official Codebase of 3DCodeVerse

Python

delete what nothing calls: the trajectory miner, the unwired cost guard, the one-off time audit

gaoypeng2bc4188eee79
diff --git a/harness/README.md b/harness/README.md
index c714a5ca..a4c3bd7b 100644
--- a/harness/README.md
+++ b/harness/README.md
@@ -8,10 +8,10 @@ renders, judges, refines, textures and records every run as data-flywheel materi
* **Languages:** `blender` · `cadquery` · `threejs` · `urdf_blender` ·
`scene_threejs` · `glsl_shader` · `opengl_python`
* **Backends:** Gemini / Anthropic / OpenAI chat models; coding agents `gemini-cli`,
- `claude-code`, `codex`, `agy` (Antigravity), the in-process `api-agent` tool loop, and
+ `claude-code`, `codex`, `agy` (Antigravity) — always a vendor's CLI — and
`single-shot` generation on any chat model.
-* **Spatial tools** (direct, MCP server `3dcv` for agentic CLIs, native tools for
- api-agent): build, measure, render_views/sheet, isolate, cross_section,
+* **Spatial tools** (direct, or over the MCP server `3dcv` for the agentic CLIs):
+ build, measure, render_views/sheet, isolate, cross_section,
check_connectivity, check_contract, compare_silhouette, joint_sweep, shader_probe,
scene_probe, scene_views, gl_probe, gl_frames, texture_pass, texture_preview,
read_cookbook.
diff --git a/harness/bench/time_audit/calls.py b/harness/bench/time_audit/calls.py
deleted file mode 100644
index 8adaf6b5..00000000
--- a/harness/bench/time_audit/calls.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""Task 2: per-model-call latency distributions (baseline vs storm day), calls per
-run, calls with >= 1 retry, and total retry-wait per run; plus the worker.log storm
-lines for the fancy_v1 cells (the only corpus whose driver keeps a per-cell log)."""
-from __future__ import annotations
-
-import re
-import sys
-from collections import Counter, defaultdict
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from corpus import RunRef, cost_rows, discover, fmt, med_p90, trajectories # noqa: E402
-from turns import parse_turns # noqa: E402
-
-RETRY_THRESHOLD_S = 1.0 # min storm sleep is 1.5 s (2^1 * 0.75); 429 rotation pause 0.5 s
-STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")
-FAIL_RE = re.compile(r"gemini (\S+) attempt (\d)/6 failed \((.*?)\); retrying in ([\d.]+)s")
-GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")
-
-
-def pct(xs: list[float]) -> str:
- if not xs:
- return "-"
- s = sorted(xs)
- q = lambda p: s[min(len(s) - 1, int(round(p * (len(s) - 1))))] # noqa: E731
- return f"n={len(s)} p50={q(.5):.1f} p90={q(.9):.1f} p99={q(.99):.1f} max={q(1):.0f}"
-
-
-def latency_table(refs: list[RunRef]) -> None:
- lat: dict[tuple[str, str, str], list[float]] = defaultdict(list)
- per_run: dict[tuple[str, str], list[int]] = defaultdict(list)
- err: Counter[tuple[str, str]] = Counter()
- for ref in refs:
- n: Counter[str] = Counter()
- for r in cost_rows(ref):
- key = (ref.bucket, r["model"], r["role"])
- if r["outcome"] == "ok":
- lat[key].append(r["latency_ms"] / 1000)
- n[r["model"]] += 1
- else:
- err[(ref.bucket, r["model"])] += 1
- for m, c in n.items():
- per_run[(ref.bucket, m)].append(c)
- print("## successful-call latency (s) by corpus x model x role")
- for k in sorted(lat):
- print(f" {k[0]:14s} {k[1]:24s} {k[2]:9s} {pct(lat[k])}")
- print("## ok calls per run (median / p90) and error rows")
- for k in sorted(per_run):
- m, p, cnt = med_p90([float(x) for x in per_run[k]])
- print(f" {k[0]:14s} {k[1]:24s} runs={cnt} calls/run {fmt(m)}/{fmt(p)} error rows={err[k]}")
-
-
-def retry_table(refs: list[RunRef]) -> None:
- print("## generator turns (transcripts): waiting = wall - recorded latency")
- agg: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
- for ref in refs:
- turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]
- if not turns:
- continue
- real = [t for t in turns if t.latency_s > 0]
- retried = [t for t in real if t.wait_s > RETRY_THRESHOLD_S]
- a = agg[(ref.bucket, ref.track)]
- a["calls"].append(len(real))
- a["retried"].append(len(retried))
- a["retried_pct"].append(100.0 * len(retried) / max(1, len(real)))
- a["wait_s"].append(sum(t.wait_s for t in real) + sum(t.error_s for t in turns))
- a["wait_per_retried"].extend(t.wait_s for t in retried)
- a["errors"].append(sum(1 for t in turns if t.error_s > 0))
- for k in sorted(agg):
- a = agg[k]
- c, cp, n = med_p90(a["calls"])
- r, rp, _ = med_p90(a["retried"])
- pc, _, _ = med_p90(a["retried_pct"])
- w, wp, _ = med_p90(a["wait_s"])
- wr, wrp, _ = med_p90(a["wait_per_retried"])
- e = sum(a["errors"])
- print(f" {k[0]:14s} {k[1]:16s} runs={n:2d} calls/run {fmt(c)}/{fmt(cp)} retried/run {fmt(r)}/{fmt(rp)} "
- f"({fmt(pc)}%) wait/run {fmt(w)}/{fmt(wp)} s wait/retried-call {fmt(wr, 1)}/{fmt(wrp, 1)} s model_error rows={e}")
-
-
-def worker_logs(refs: list[RunRef]) -> None:
- print("## fancy_v1 worker.log (per cell; sleeps only — the failed round-trips are not logged)")
- per: dict[str, list[float]] = defaultdict(list)
- tot: Counter[str] = Counter()
- sleep: Counter[str] = Counter()
- depth: Counter[int] = Counter()
- for ref in refs:
- cd = ref.cell_dir
- if cd is None:
- continue
- s_flash = s_pro = 0.0
- for line in (cd / "worker.log").read_text(errors="replace").splitlines():
- m = STORM_RE.match(line)
- if m:
- tot[f"storm:{m.group(1)}"] += 1
- sleep[m.group(1)] += int(m.group(3))
- depth[int(m.group(2))] += 1
- if "flash" in m.group(1):
- s_flash += int(m.group(3))
- else:
- s_pro += int(m.group(3))
- continue
- m = FAIL_RE.match(line)
- if m:
- tot[f"fail:{m.group(1)}:{m.group(3)[:28]}"] += 1
- continue
- m = GIVEUP_RE.match(line)
- if m:
- tot[f"giveup:{m.group(1)}"] += 1
- per["flash_sleep_s"].append(s_flash)
- per["pro_sleep_s"].append(s_pro)
- for k, v in sorted(tot.items()):
- print(f" {v:5d} {k}")
- for k, v in sleep.items():
- print(f" total logged sleep {k}: {v} s over {len(per['flash_sleep_s'])} cells")
- for k in ("flash_sleep_s", "pro_sleep_s"):
- m, p, n = med_p90(per[k])
- print(f" {k} per cell med/p90 = {fmt(m)}/{fmt(p)} (n={n})")
- mx = max(depth) if depth else 0
- print(f" storm streak depth: max {mx}; streaks >= 10: {sum(v for d, v in depth.items() if d >= 10)} lines; "
- f"streak 1: {depth[1]} (= distinct storm episodes)")
-
-
-if __name__ == "__main__":
- refs = discover()
- latency_table(refs)
- retry_table(refs)
- worker_logs(refs)
diff --git a/harness/bench/time_audit/corpus.py b/harness/bench/time_audit/corpus.py
deleted file mode 100644
index dd1c4908..00000000
--- a/harness/bench/time_audit/corpus.py
+++ /dev/null
@@ -1,126 +0,0 @@
-"""Read-only corpus discovery + loaders for the time audit (no side effects)."""
-from __future__ import annotations
-
-import glob
-import json
-import statistics
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any
-
-OUT = Path("/home/yipeng/3dcodeverse/harness/bench/out")
-STORM_DAY = "storm(08-26)"
-BASELINE = "base(08-23/25)"
-
-# corpus name -> (glob under bench/out, corpus bucket)
-CORPORA: dict[str, tuple[str, str]] = {
- "fancy_v1": ("fancy_v1/*/arms/*/cells/*/harness_api-agent_gemini_gemini-3.7-flash/run", STORM_DAY),
- "h2h_brilliana_v1": ("h2h_brilliana_v1/*/runs/*", STORM_DAY),
- "h2h_scene_v1": ("h2h_scene_v1/runs/*", STORM_DAY),
- "refs_v1_graphics": ("refs_v1_graphics/runs/*", STORM_DAY),
- "teaser": ("teaser/runs/*", BASELINE),
- "static_v2_flash": ("static_v2_flash/runs/*", BASELINE),
- "scenes_v1_flash": ("scenes_v1_flash/runs/*", BASELINE),
-}
-
-
-@dataclass
-class RunRef:
- corpus: str
- bucket: str
- run: Path
- track: str = ""
- events: list[dict[str, Any]] = field(default_factory=list)
-
- @property
- def cell_dir(self) -> Path | None:
- return self.run.parent if (self.run.parent / "worker.log").is_file() else None
-
- @property
- def name(self) -> str:
- return self.run.parent.parent.name if self.cell_dir else self.run.name
-
-
-def read_jsonl(path: Path) -> list[dict[str, Any]]:
- rows: list[dict[str, Any]] = []
- if not path.is_file():
- return rows
- with path.open() as fh:
- for line in fh:
- line = line.strip()
- if not line:
- continue
- try:
- rows.append(json.loads(line))
- except json.JSONDecodeError:
- continue
- return rows
-
-
-def track_of(events: list[dict[str, Any]]) -> str:
- start = next((e for e in events if e.get("event") == "run.start"), None)
- if start is None:
- return "?"
- tr, lang = start.get("track", "?"), start.get("language", "?")
- if tr == "static_object":
- return f"static/{lang}"
- if tr == "articulated_object":
- return "articulated"
- return tr
-
-
-def discover(corpora: dict[str, tuple[str, str]] = CORPORA) -> list[RunRef]:
- refs: list[RunRef] = []
- for corpus, (pat, bucket) in corpora.items():
- for d in sorted(glob.glob(str(OUT / pat))):
- run = Path(d)
- ev = read_jsonl(run / "events.jsonl")
- if not ev:
- continue
- refs.append(RunRef(corpus=corpus, bucket=bucket, run=run, track=track_of(ev), events=ev))
- return refs
-
-
-def cost_rows(ref: RunRef) -> list[dict[str, Any]]:
- return read_jsonl(ref.run / "telemetry" / "cost.jsonl")
-
-
-def trajectories(ref: RunRef) -> list[tuple[str, dict[str, Any], list[dict[str, Any]]]]:
- """(session label, result.json, transcript rows) for every trajectory dir."""
- out: list[tuple[str, dict[str, Any], list[dict[str, Any]]]] = []
- tdir = ref.run / "trajectories"
- if not tdir.is_dir():
- return out
- for sd in sorted(p for p in tdir.iterdir() if p.is_dir()):
- res: dict[str, Any] = {}
- rp = sd / "result.json"
- if rp.is_file():
- try:
- res = json.loads(rp.read_text())
- except json.JSONDecodeError:
- res = {}
- out.append((sd.name, res, read_jsonl(sd / "transcript.jsonl")))
- return out
-
-
-def rounds(ref: RunRef) -> list[dict[str, Any]]:
- out: list[dict[str, Any]] = []
- for p in sorted((ref.run / "rounds").glob("r*.json")):
- try:
- out.append(json.loads(p.read_text()))
- except (OSError, json.JSONDecodeError):
- continue
- return out
-
-
-def med_p90(xs: list[float]) -> tuple[float, float, int]:
- """(median, p90, n); p90 by nearest-rank."""
- if not xs:
- return (float("nan"), float("nan"), 0)
- s = sorted(xs)
- p90 = s[min(len(s) - 1, int(round(0.9 * (len(s) - 1))))]
- return (statistics.median(s), p90, len(s))
-
-
-def fmt(x: float, nd: int = 0) -> str:
- return "-" if x != x else f"{x:.{nd}f}"
diff --git a/harness/bench/time_audit/deadline.py b/harness/bench/time_audit/deadline.py
deleted file mode 100644
index 869c9227..00000000
--- a/harness/bench/time_audit/deadline.py
+++ /dev/null
@@ -1,59 +0,0 @@
-"""Deadline overshoots: sessions past their 1800-s job timeout, runs killed by the
-bench max_minutes with no round completed, and the storm-streak depth at each
-'giving up after 900 s' (=> seconds per failed 503 attempt, independent of transcripts)."""
-from __future__ import annotations
-
-import json
-import re
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from corpus import discover, fmt, med_p90, trajectories # noqa: E402
-
-STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 ")
-GIVEUP_RE = re.compile(r"gemini (\S+) giving up after (\d+) s")
-BUDGET_RE = re.compile(r"elapsed ([\d.]+) min exceeds max_minutes ([\d.]+)")
-SESSION_TIMEOUT_S = 1800.0
-
-
-def main() -> None:
- refs = discover()
- over, killed, zero_rounds, depths = [], [], 0, []
- for ref in refs:
- if ref.bucket != "storm(08-26)":
- continue
- for _, res, _ in trajectories(ref):
- d = float(res.get("duration_s") or 0)
- if d > SESSION_TIMEOUT_S and res.get("exit_reason") in ("timeout", "error"):
- over.append(d - SESSION_TIMEOUT_S)
- rounds_done = sum(1 for e in ref.events if e["event"] == "round.done")
- for e in ref.events:
- if e["event"] == "budget.exceeded":
- m = BUDGET_RE.search(str(e.get("reason", "")))
- if m:
- killed.append((float(m.group(1)) - float(m.group(2))) * 60)
- zero_rounds += rounds_done == 0
- cd = ref.cell_dir
- if cd is None:
- continue
- last = 0
- for line in (cd / "worker.log").read_text(errors="replace").splitlines():
- m = STORM_RE.match(line)
- if m and "flash" in m.group(1):
- last = int(m.group(2))
- elif GIVEUP_RE.match(line) and "flash" in line:
- depths.append(last)
- m, p, n = med_p90(over)
- print(f"sessions past the {SESSION_TIMEOUT_S:.0f}-s job timeout (exit timeout/error): n={n} overshoot med/p90 = {fmt(m)}/{fmt(p)} s, sum={sum(over):.0f} s")
- m, p, n = med_p90(killed)
- print(f"runs stopped by bench max_minutes: n={n}, overshoot past the ceiling med/p90 = {fmt(m)}/{fmt(p)} s; with ZERO completed rounds: {zero_rounds}")
- m, p, n = med_p90([float(d) for d in depths])
- if n:
- print(f"flash give-ups in fancy_v1 logs: n={n}; storm-streak depth at give-up med/p90 = {fmt(m)}/{fmt(p)} "
- f"=> 900 s / depth = {fmt(900 / m, 1)} s per failed attempt (sleep included, <= 5 s of it)")
- print(json.dumps({"overshoot_sum_s": round(sum(over)), "killed": len(killed), "zero_rounds": zero_rounds}))
-
-
-if __name__ == "__main__":
- main()
diff --git a/harness/bench/time_audit/judge_plan.py b/harness/bench/time_audit/judge_plan.py
deleted file mode 100644
index 2d417d1a..00000000
--- a/harness/bench/time_audit/judge_plan.py
+++ /dev/null
@@ -1,55 +0,0 @@
-"""Judge + planner stages: stage span vs recorded model latency (the difference is
-retry/timeout waiting), per corpus; judge samples that errored and what they cost."""
-from __future__ import annotations
-
-import sys
-from collections import defaultdict
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from corpus import cost_rows, discover, fmt, med_p90 # noqa: E402
-
-
-def main() -> None:
- J: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
- for ref in discover():
- rows = cost_rows(ref)
- by_round: dict[int, list[dict]] = defaultdict(list)
- for r in rows:
- if r["role"] == "judge":
- by_round[int(r.get("round") or 0)].append(r)
- b = J[ref.bucket]
- for e in ref.events:
- if e["event"] == "judge.done":
- d = float(e.get("duration_s") or 0)
- rs = by_round.get(int(e.get("round") or 0), [])
- ok = sum(r["latency_ms"] / 1000 for r in rs if r["outcome"] == "ok")
- err = [r["latency_ms"] / 1000 for r in rs if r["outcome"] != "ok"]
- b["judge_s"].append(d)
- b["judge_ok_lat"].append(ok)
- b["judge_wait"].append(max(0.0, d - ok))
- b["judge_n_ok"].append(float(sum(1 for r in rs if r["outcome"] == "ok")))
- if err:
- b["judge_err_rows"].extend(err)
- b["judge_rounds_with_err"].append(d)
- elif e["event"] == "stage.done" and e.get("stage") == "plan":
- d = float(e.get("duration_s") or 0)
- pl = [r for r in rows if r["role"] == "planner"]
- ok = sum(r["latency_ms"] / 1000 for r in pl if r["outcome"] == "ok")
- b["plan_s"].append(d)
- b["plan_ok_lat"].append(ok)
- b["plan_calls"].append(float(len(pl)))
- b["plan_wait"].append(max(0.0, d - ok))
- for bucket, b in sorted(J.items()):
- print(f"## {bucket}")
- for k in ("judge_s", "judge_ok_lat", "judge_wait", "judge_n_ok", "plan_s", "plan_ok_lat", "plan_calls", "plan_wait"):
- m, p, n = med_p90(b[k])
- print(f" {k:14s} n={n:3d} med={fmt(m, 1):>7s} p90={fmt(p, 1):>7s} sum={sum(b[k]):.0f}")
- er = b["judge_err_rows"]
- print(f" judge error rows: n={len(er)} sum={sum(er):.0f} s each: {[round(x) for x in sorted(er, reverse=True)[:8]]}")
- rw = b["judge_rounds_with_err"]
- print(f" judge rounds with an errored sample: {len(rw)} / {len(b['judge_s'])}; their judge_s: {[round(x) for x in sorted(rw, reverse=True)[:8]]}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/harness/bench/time_audit/keys.py b/harness/bench/time_audit/keys.py
deleted file mode 100644
index 64cc9f10..00000000
--- a/harness/bench/time_audit/keys.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""Task 4: does any persisted telemetry row say WHICH Gemini key served a call?
-Scans every field name in cost.jsonl / transcript.jsonl / result.json / rounds /
-record.json for something key-like, and greps the raw text for the '…xxxx' redaction
-that gemini.py puts into ChatResponse.raw."""
-from __future__ import annotations
-
-import json
-import re
-import sys
-from collections import Counter
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-import contextlib
-
-from corpus import discover, read_jsonl # noqa: E402
-
-KEYLIKE = re.compile(r"key|api_key|credential", re.I)
-REDACTED = re.compile(r'"key":\s*"…[^"]{0,8}"')
-
-
-def walk(obj: object, prefix: str, found: Counter[str]) -> None:
- if isinstance(obj, dict):
- for k, v in obj.items():
- if KEYLIKE.search(str(k)):
- found[f"{prefix}.{k}"] += 1
- walk(v, f"{prefix}.{k}", found)
- elif isinstance(obj, list):
- for v in obj[:50]:
- walk(v, prefix + "[]", found)
-
-
-def main() -> None:
- found: Counter[str] = Counter()
- redacted = 0
- files = 0
- for ref in discover():
- for p in [ref.run / "telemetry" / "cost.jsonl", ref.run / "record.json", *(ref.run / "rounds").glob("r*.json"),
- *(ref.run / "trajectories").glob("*/result.json"), *(ref.run / "trajectories").glob("*/transcript.jsonl")]:
- if not p.is_file():
- continue
- files += 1
- text = p.read_text(errors="replace")
- redacted += len(REDACTED.findall(text))
- if p.suffix == ".jsonl":
- for row in read_jsonl(p):
- walk(row, p.name, found)
- else:
- with contextlib.suppress(json.JSONDecodeError):
- walk(json.loads(text), p.name, found)
- print(f"scanned {files} files; '\"key\": \"…xxxx\"' occurrences: {redacted}")
- print("key-like field names found (name -> count):")
- for k, v in found.most_common(20):
- print(f" {v:7d} {k}")
- if not any("api" in k.lower() or k.endswith(".key") for k in found):
- print("=> no per-call key identity is persisted anywhere in the run telemetry.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/harness/bench/time_audit/sessions.py b/harness/bench/time_audit/sessions.py
deleted file mode 100644
index 55ffd861..00000000
--- a/harness/bench/time_audit/sessions.py
+++ /dev/null
@@ -1,58 +0,0 @@
-"""Task 3: per agent session (api-agent transcripts) — turns, tool calls by name
-with durations, and the model-thinking / tool-execution / waiting split, one table
-per track x corpus."""
-from __future__ import annotations
-
-import sys
-from collections import defaultdict
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from corpus import discover, fmt, med_p90, trajectories # noqa: E402
-from turns import parse_turns, split # noqa: E402
-
-
-def main() -> None:
- sess: dict[tuple[str, str], list[tuple[int, float, float, float, float]]] = defaultdict(list)
- tools: dict[tuple[str, str], dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
- for ref in discover():
- k = (ref.bucket, ref.track)
- for _, res, tr in trajectories(ref):
- turns = parse_turns(tr)
- if not turns:
- continue
- think, tool, wait = split(turns)
- sess[k].append((len([t for t in turns if t.latency_s > 0]), float(res.get("duration_s") or 0), think, tool, wait))
- for t in turns:
- for name, d in t.tools:
- tools[k][name].append(d)
- print("## per session: turns, duration, and the split (medians; shares are of summed session time)")
- print("| corpus | track | sessions | turns med/p90 | dur s med/p90 | think s | tool s | wait s | think% | tool% | wait% |")
- print("|---|---|---|---|---|---|---|---|---|---|---|")
- for k in sorted(sess):
- rows = sess[k]
- tm, tp, n = med_p90([float(r[0]) for r in rows])
- dm, dp, _ = med_p90([r[1] for r in rows])
- th, _, _ = med_p90([r[2] for r in rows])
- to, _, _ = med_p90([r[3] for r in rows])
- wa, _, _ = med_p90([r[4] for r in rows])
- T = sum(r[2] + r[3] + r[4] for r in rows) or 1.0
- print(f"| {k[0]} | {k[1]} | {n} | {fmt(tm)}/{fmt(tp)} | {fmt(dm)}/{fmt(dp)} | {fmt(th)} | {fmt(to)} | {fmt(wa)} | "
- f"{100 * sum(r[2] for r in rows) / T:.0f} | {100 * sum(r[3] for r in rows) / T:.0f} | {100 * sum(r[4] for r in rows) / T:.0f} |")
- print("\n## tool calls by name: count, median s, p90 s, total s per session (top tools per track; storm-day + baseline pooled by track)")
- pooled: dict[str, dict[str, list[float]]] = defaultdict(lambda: defaultdict(list))
- nsess: dict[str, int] = defaultdict(int)
- for k, d in tools.items():
- for name, xs in d.items():
- pooled[k[1]][name].extend(xs)
- nsess[k[1]] += len(sess[k])
- for track in sorted(pooled):
- items = sorted(pooled[track].items(), key=lambda kv: -sum(kv[1]))
- print(f" {track} ({nsess[track]} sessions):")
- for name, xs in items[:12]:
- m, p, n = med_p90(xs)
- print(f" {name:18s} n={n:5d} med={fmt(m, 2):>6s} p90={fmt(p, 1):>6s} total/session={sum(xs) / max(1, nsess[track]):6.1f} s")
-
-
-if __name__ == "__main__":
- main()
diff --git a/harness/bench/time_audit/stage_times.py b/harness/bench/time_audit/stage_times.py
deleted file mode 100644
index 3176f630..00000000
--- a/harness/bench/time_audit/stage_times.py
+++ /dev/null
@@ -1,156 +0,0 @@
-"""Task 1: per track x corpus, median / p90 of run wall and every stage, plus
-"waiting on the provider" (retry sleeps + failed attempts), derived from transcript
-timestamps (agent sessions) and from judge/planner stage span minus recorded latency.
-Writes stage_runs.jsonl (one row per run) and prints the table."""
-from __future__ import annotations
-
-import json
-import sys
-from collections import defaultdict
-from pathlib import Path
-from typing import Any
-
-sys.path.insert(0, str(Path(__file__).parent))
-from corpus import RunRef, cost_rows, discover, fmt, med_p90, rounds, trajectories # noqa: E402
-from turns import parse_turns, split # noqa: E402
-
-HERE = Path(__file__).parent
-
-
-def round_segments(ev: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """One dict per round.start..round.done segment (an abandoned segment ends at the
- next round.start / run.start and is flagged)."""
- segs: list[dict[str, Any]] = []
- cur: dict[str, Any] | None = None
- cand: dict[Any, float] = {} # best-of-N candidates are generated BEFORE round.start
- for e in ev:
- k = e["event"]
- if k == "candidates.start":
- cand[e.get("round")] = e["t"]
- if k in ("round.start", "run.start"):
- if cur is not None:
- cur["abandoned"] = True
- cur["end"] = e["t"]
- segs.append(cur)
- cur = None
- if k == "round.start":
- cur = {"round": e.get("round"), "start": min(e["t"], cand.pop(e.get("round"), e["t"])), "abandoned": False}
- continue
- if cur is None:
- continue
- if k == "build.done":
- cur.setdefault("first_build", e["t"])
- cur["last_build"] = e["t"]
- cur["build_ms"] = cur.get("build_ms", 0) + int(e.get("duration_ms") or 0)
- cur["n_builds"] = cur.get("n_builds", 0) + 1
- elif k == "gates.done":
- cur["gates_t"] = e["t"]
- elif k == "judge.done":
- cur["judge_s"] = float(e.get("duration_s") or 0)
- cur["judge_t"] = e["t"]
- elif k == "round.done":
- cur["end"] = e["t"]
- cur["duration_s"] = float(e.get("duration_s") or (e["t"] - cur["start"]))
- segs.append(cur)
- cur = None
- if cur is not None:
- cur["abandoned"] = True
- cur["end"] = ev[-1]["t"]
- segs.append(cur)
- return segs
-
-
-def analyse(ref: RunRef) -> dict[str, Any]:
- ev = ref.events
- done = [e for e in ev if e["event"] == "run.done"]
- row: dict[str, Any] = {
- "corpus": ref.corpus, "bucket": ref.bucket, "track": ref.track, "run": ref.name,
- "finished": bool(done), "wall_s": ev[-1]["t"] - ev[0]["t"],
- "status": done[-1].get("status") if done else "RUNNING",
- }
- plan_s = sum(float(e.get("duration_s") or 0) for e in ev if e["event"] == "stage.done" and e.get("stage") == "plan")
- row["plan_s"] = plan_s
- scene_gen = sum(float(e.get("duration_s") or 0) for e in ev
- if e["event"] == "stage.done" and e.get("stage") in ("assets", "env", "zones", "assemble"))
- row["plan_cached"] = any(e["event"] == "stage.cached" and e.get("stage") == "plan" for e in ev)
- rows = cost_rows(ref)
- lat = lambda role, ok=True: sum( # noqa: E731
- r["latency_ms"] / 1000 for r in rows if r["role"] == role and (r["outcome"] == "ok") == ok)
- row["plan_model_s"] = lat("planner")
- row["plan_wait_s"] = max(0.0, plan_s - lat("planner")) if plan_s else 0.0
- row["judge_err_s"] = lat("judge", ok=False) # error rows carry the whole failed span
- segs = [s for s in round_segments(ev) if not s["abandoned"]]
- row["abandoned_s"] = sum(s["end"] - s["start"] for s in round_segments(ev) if s["abandoned"])
- rd = {r.get("index"): r for r in rounds(ref)}
- gen, build, gates, render, judge, other = [], [], [], [], [], []
- for s in segs:
- if "gates_t" not in s or "last_build" not in s:
- continue
- b = s["build_ms"] / 1000
- gen.append(s["last_build"] - s["start"] - b)
- build.append(b)
- gates.append(s["gates_t"] - s["last_build"])
- r = rd.get(s["round"], {})
- rnd = float((r.get("renders") or {}).get("duration_ms") or 0) / 1000
- render.append(rnd)
- j = s.get("judge_s", 0.0)
- judge.append(j)
- if "judge_t" in s:
- other.append(max(0.0, s["judge_t"] - s["gates_t"] - j - rnd))
- judge_ok = lat("judge")
- row.update(gen_s=sum(gen) + scene_gen, build_s=sum(build), gates_s=sum(gates), render_s=sum(render),
- judge_s=sum(judge), other_s=sum(other), n_rounds=len(gen),
- judge_model_s=judge_ok, judge_wait_s=max(0.0, sum(judge) - judge_ok))
- think = tools = wait = sess = 0.0
- n_sess = 0
- for _, res, tr in trajectories(ref):
- a, b2, c = split(parse_turns(tr))
- think, tools, wait, sess, n_sess = think + a, tools + b2, wait + c, sess + float(res.get("duration_s") or 0), n_sess + 1
- row.update(sess_think_s=think, sess_tool_s=tools, sess_wait_s=wait, sess_total_s=sess, n_sessions=n_sess)
- row["wait_total_s"] = wait + row["plan_wait_s"] + row["judge_wait_s"]
- return row
-
-
-def main() -> None:
- refs = discover()
- out = [analyse(r) for r in refs]
- (HERE / "stage_runs.jsonl").write_text("".join(json.dumps(r) + "\n" for r in out))
- cols = ["wall_s", "plan_s", "gen_s", "build_s", "gates_s", "render_s", "judge_s", "wait_total_s", "sess_wait_s", "judge_wait_s", "judge_err_s", "abandoned_s"]
- groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
- for r in out:
- groups[(r["bucket"], r["track"])].append(r)
- print("| corpus | track | n(fin) | " + " | ".join(f"{c} med/p90" for c in cols) + " |")
- print("|" + "---|" * (len(cols) + 3))
- for (bucket, track), rs in sorted(groups.items()):
- fin = [r for r in rs if r["finished"]]
- cells = []
- for c in cols:
- src = fin if c == "wall_s" else rs
- m, p, _ = med_p90([float(r[c]) for r in src])
- cells.append(f"{fmt(m)}/{fmt(p)}")
- print(f"| {bucket} | {track} | {len(rs)}({len(fin)}) | " + " | ".join(cells) + " |")
- print("\nshare of run wall: waiting / thinking / tools (medians of per-run ratios, finished runs)")
- for (bucket, track), rs in sorted(groups.items()):
- fin = [r for r in rs if r["finished"] and r["wall_s"] > 0]
- if not fin:
- continue
- w = med_p90([100 * r["wait_total_s"] / r["wall_s"] for r in fin])[0]
- th = med_p90([100 * r["sess_think_s"] / r["wall_s"] for r in fin])[0]
- to = med_p90([100 * (r["sess_tool_s"] + r["build_s"] + r["gates_s"] + r["render_s"]) / r["wall_s"] for r in fin])[0]
- jm = med_p90([100 * r["judge_model_s"] / r["wall_s"] for r in fin])[0]
- print(f" {bucket:14s} {track:16s} wait {fmt(w)}% think {fmt(th)}% tools+build+gates+render {fmt(to)}% judge model {fmt(jm)}%")
- # per-round view (generate + judge per round) for the storm/baseline contrast
- print("\nper-round: gen_s and judge_s med/p90 (rounds with a build+gates)")
- per: dict[tuple[str, str], tuple[list[float], list[float]]] = defaultdict(lambda: ([], []))
- for r in out:
- n = max(1, r["n_rounds"])
- per[(r["bucket"], r["track"])][0].append(r["gen_s"] / n)
- per[(r["bucket"], r["track"])][1].append(r["judge_s"] / n)
- for k, (g, j) in sorted(per.items()):
- gm, gp, _ = med_p90(g)
- jm, jp, _ = med_p90(j)
- print(f" {k[0]:14s} {k[1]:16s} gen {fmt(gm)}/{fmt(gp)} judge {fmt(jm)}/{fmt(jp)}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/harness/bench/time_audit/storm_attempts.py b/harness/bench/time_audit/storm_attempts.py
deleted file mode 100644
index 601d947a..00000000
--- a/harness/bench/time_audit/storm_attempts.py
+++ /dev/null
@@ -1,78 +0,0 @@
-"""Price the pieces of "waiting": backoff sleeps (worker.log) vs failed 503 round
-trips (transcript wait minus sleep, per storm line) vs 900-s give-ups (model_error
-rows / turns with wait >= 850 s), plus cell wall vs run span for the fancy_v1 cells."""
-from __future__ import annotations
-
-import json
-import re
-import sys
-from collections import Counter
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from corpus import discover, fmt, med_p90, trajectories # noqa: E402
-from turns import parse_turns # noqa: E402
-
-STORM_RE = re.compile(r"gemini (\S+) capacity storm (\d+)/60 .*; waiting (\d+)s")
-GIVEUP_S = 850.0
-
-
-def hist(xs: list[float], edges: list[float]) -> str:
- c = Counter()
- for x in xs:
- c[next((e for e in edges if x < e), float("inf"))] += 1
- tot = max(1, len(xs))
- return " ".join(f"<{e:g}s:{100 * c[e] / tot:.0f}%" for e in [*edges, float("inf")])
-
-
-def main() -> None:
- refs = discover()
- for bucket in ("base(08-23/25)", "storm(08-26)"):
- waits, big, errs = [], [], []
- for ref in [r for r in refs if r.bucket == bucket]:
- for _, _, tr in trajectories(ref):
- for t in parse_turns(tr):
- if t.latency_s > 0:
- waits.append(t.wait_s)
- if t.error_s > 0:
- errs.append(t.error_s)
- tot = sum(waits) + sum(errs)
- big = [w for w in waits if w >= GIVEUP_S]
- print(f"## {bucket}: {len(waits)} generator turns; wait histogram: {hist(waits, [1, 5, 30, 120, 850])}")
- print(f" total wait {tot:.0f} s; turns with wait>=850 s: {len(big)} = {100 * sum(big) / max(1, tot):.0f}% of all wait; "
- f"model_error spans: n={len(errs)} sum={sum(errs):.0f} s ({100 * sum(errs) / max(1, tot):.0f}%) "
- f"med/p90={fmt(med_p90(errs)[0])}/{fmt(med_p90(errs)[1])} s")
- print(f" wait>=850s turn: {hist(big, [900, 1800, 2700])}")
- print("## fancy_v1 cells: transcript wait vs logged sleep -> cost of one failed 503 round trip")
- rt, per_cell, outside = [], [], []
- for ref in refs:
- cd = ref.cell_dir
- if cd is None:
- continue
- lines = (cd / "worker.log").read_text(errors="replace").splitlines()
- n = sleep = 0
- for m in filter(None, map(STORM_RE.match, lines)):
- if "flash" in m.group(1):
- n += 1
- sleep += int(m.group(3))
- turns = [t for _, _, tr in trajectories(ref) for t in parse_turns(tr)]
- w = sum(t.wait_s for t in turns) + sum(t.error_s for t in turns)
- w_nogiveup = sum(min(t.wait_s, 0) if t.wait_s >= GIVEUP_S else t.wait_s for t in turns)
- if n:
- rt.append((w_nogiveup - sleep) / n)
- per_cell.append((ref.name, n, sleep, round(w), round(w_nogiveup)))
- try:
- cell = json.loads((cd / "cell.json").read_text())
- outside.append(float(cell.get("wall_s") or 0) - (ref.events[-1]["t"] - ref.events[0]["t"]))
- except (OSError, json.JSONDecodeError):
- pass
- m, p, k = med_p90(rt)
- print(f" per-cell (wait_excluding_giveups - sleep) / storm_lines: med {fmt(m, 1)} s p90 {fmt(p, 1)} s (n={k} cells)")
- print(f" cell wall_s - run event span (pairwise + final pro scoring outside the run): med/p90 = "
- f"{fmt(med_p90(outside)[0])}/{fmt(med_p90(outside)[1])} s")
- for row in sorted(per_cell, key=lambda r: -r[3])[:5]:
- print(" worst:", row)
-
-
-if __name__ == "__main__":
- main()
diff --git a/harness/bench/time_audit/turns.py b/harness/bench/time_audit/turns.py
deleted file mode 100644
index e831f541..00000000
--- a/harness/bench/time_audit/turns.py
+++ /dev/null
@@ -1,67 +0,0 @@
-"""Per-turn reconstruction of an api-agent transcript.
-
-The ledger's ``latency_ms`` is the stopwatch of the SUCCESSFUL attempt only
-(``gemini.py:_once``); backoff sleeps, failed attempts and KeyPool waits are not
-recorded anywhere. Transcript rows are stamped when the model returns
-(``assistant``) and when a tool finishes (``tool_result``), so
-``model_wall = t(assistant) - t(previous row)`` and ``wait = model_wall - latency``.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Any
-
-
-@dataclass
-class Turn:
- turn: int
- model_wall_s: float
- latency_s: float
- wait_s: float
- tools: list[tuple[str, float]] = field(default_factory=list)
- error_s: float = 0.0 # time spent in a call that ended in a model_error row
-
-
-def is_gemini_api_session(rows: list[dict[str, Any]]) -> bool:
- """Only in-process api-agent sessions on Gemini stamp a per-turn latency."""
- return any(r.get("kind") == "assistant" and (r.get("usage") or {}).get("backend") == "gemini" for r in rows)
-
-
-def parse_turns(rows: list[dict[str, Any]]) -> list[Turn]:
- turns: list[Turn] = []
- if not rows or not is_gemini_api_session(rows):
- return turns
- prev_end = float(rows[0]["t"])
- cur: Turn | None = None
- tool_start = prev_end
- pending_err = 0.0
- for r in rows:
- t = float(r["t"])
- kind = r.get("kind")
- if kind == "assistant":
- wall = max(0.0, t - prev_end)
- lat = float((r.get("usage") or {}).get("latency_ms") or 0) / 1000.0
- cur = Turn(int(r.get("turn", len(turns))), wall, lat, max(0.0, wall - lat), error_s=pending_err)
- pending_err = 0.0
- turns.append(cur)
- prev_end = tool_start = t
- elif kind == "tool_result":
- if cur is not None:
- cur.tools.append((str(r.get("name", "?")), max(0.0, t - tool_start)))
- tool_start = prev_end = t
- elif kind == "model_error":
- pending_err += max(0.0, t - prev_end)
- prev_end = t
- elif kind in ("nudge", "compact", "system"):
- prev_end = t
- if pending_err > 0: # session died in a model_error after the last good turn
- turns.append(Turn(len(turns), 0.0, 0.0, 0.0, error_s=pending_err))
- return turns
-
-
-def split(turns: list[Turn]) -> tuple[float, float, float]:
- """(model thinking s, tool execution s, waiting s) over a session."""
- think = sum(t.latency_s for t in turns)
- tools = sum(d for t in turns for _, d in t.tools)
- wait = sum(t.wait_s + t.error_s for t in turns)
- return think, tools, wait
diff --git a/harness/codeverse/cli/flywheel_cmd.py b/harness/codeverse/cli/flywheel_cmd.py
index efb5cef9..ac4a180c 100644
--- a/harness/codeverse/cli/flywheel_cmd.py
+++ b/harness/codeverse/cli/flywheel_cmd.py
@@ -69,12 +69,11 @@ def pairs_cmd(
runs_dir: Annotated[Path, typer.Argument()],
out_jsonl: Annotated[Path, typer.Argument()],
min_delta: Annotated[float, typer.Option("--min-delta")] = 0.05,
- trajectories: Annotated[bool, typer.Option("--trajectories/--no-trajectories", help="also mine in-session repair pairs from agent transcripts")] = True,
) -> None:
"""Preference / repair (round + in-session) / cross-backend pairs → JSONL."""
from codeverse.flywheel.pairs import build_pairs
- n = build_pairs(runs_dir, out_jsonl, min_delta=min_delta, trajectories=trajectories)
+ n = build_pairs(runs_dir, out_jsonl, min_delta=min_delta)
ok(f"{n} pairs → {out_jsonl}")
diff --git a/harness/codeverse/cost/__init__.py b/harness/codeverse/cost/__init__.py
index 54c2b749..e66894dd 100644
--- a/harness/codeverse/cost/__init__.py
+++ b/harness/codeverse/cost/__init__.py
@@ -8,7 +8,7 @@ Public API::
from codeverse.cost import record_call, load_ledger, summarise
from codeverse.cost import audit_runs, markdown # the audit + its report
- from codeverse.cost import estimate_call, CostGuard # decide before you send
+ from codeverse.cost import estimate_call # estimate before you send
from codeverse.cost import Block, order_blocks # cache-friendly prompts
from codeverse.cost import default_route, pro_break_even # model routing
@@ -28,13 +28,7 @@ from codeverse.cost.caching import (
render_blocks,
)
from codeverse.cost.context import CallContext, bind_run, call_context
-from codeverse.cost.guard import (
- CostEstimate,
- CostGuard,
- cheapest_affordable,
- estimate_call,
- text_tokens,
-)
+from codeverse.cost.guard import CostEstimate, estimate_call, text_tokens
from codeverse.cost.instrument import (
MeteredAgent,
MeteredChatModel,
@@ -72,7 +66,6 @@ __all__ = [
"CallCost",
"CostBucket",
"CostEstimate",
- "CostGuard",
"CostLedger",
"MeteredAgent",
"MeteredChatModel",
@@ -90,7 +83,6 @@ __all__ = [
"bind_run",
"cache_efficiency",
"call_context",
- "cheapest_affordable",
"console",
"default_route",
"estimate_call",
diff --git a/harness/codeverse/cost/guard.py b/harness/codeverse/cost/guard.py
index 986a28d5..daf78477 100644
--- a/harness/codeverse/cost/guard.py
+++ b/harness/codeverse/cost/guard.py
@@ -1,9 +1,8 @@
"""Estimate what a call will cost **before** sending it, and pick a tier.
The budget guard in ``orchestrator/budget.py`` stops a run *after* the money is
-gone. This is the cheap check that belongs in front of a call: how many tokens
-am I about to send, what will they cost on this model, and is there a cheaper
-model that still fits the job?
+gone. This is the cheap estimate that belongs in front of a call: how many
+tokens am I about to send, and what will they cost on this model?
Token counts are estimates (≈4 characters per token for prose/code, a flat
per-image count for vision parts) — good to ±15% for prompt sizing, which is all
@@ -14,7 +13,7 @@ the provider returns (``codeverse.cost.record_call``).
from __future__ import annotations
from collections.abc import Iterable, Sequence
-from dataclasses import dataclass, field
+from dataclasses import dataclass
from codeverse.cost.types import normalise_ids
from codeverse.models.pricing import per_image_usd, price_provenance, unit_prices
@@ -97,66 +96,3 @@ def estimate_call(
usd=usd, price_source=row.match if row.price else "unknown",
approximate=row.approximate)
-
-@dataclass
-class CostGuard:
- """A per-run (or per-stage) allowance checked *before* each call.
-
- ``spent`` is advanced by :meth:`commit` with what the call really cost, so
- the guard converges on the truth even when the estimate was off."""
-
- budget_usd: float
- spent_usd: float = 0.0
- reserve_usd: float = 0.0 # keep this much back for the judge / finalise
- calls: int = 0
- rejected: list[str] = field(default_factory=list)
-
- @property
- def remaining_usd(self) -> float:
- return max(0.0, self.budget_usd - self.reserve_usd - self.spent_usd)
-
- def affordable(self, estimate: CostEstimate) -> bool:
- return estimate.usd <= self.remaining_usd
-
- def check(self, estimate: CostEstimate) -> tuple[bool, str]:
- """``(allow, reason)``. An unpriced model is allowed but flagged: refusing
- to run because we cannot price a model would be worse than running it."""
- if not estimate.known:
- return True, f"no price row for {estimate.model_id}: cost unknown, not blocked"
- if self.affordable(estimate):
- return True, ""
- msg = (f"{estimate.model_id} would cost ${estimate.usd:.4f} but only "
- f"${self.remaining_usd:.4f} is left of ${self.budget_usd:.2f}")
- self.rejected.append(msg)
- return False, msg
-
- def commit(self, usd: float) -> None:
- self.spent_usd += max(0.0, usd)
- self.calls += 1
-
-
-def cheapest_affordable(
- model_ids: Sequence[str],
- *,
- input_tokens: int = 0,
- output_tokens: int = 0,
- cached_tokens: int = 0,
- budget_usd: float | None = None,
-) -> tuple[str, CostEstimate] | None:
- """Pick the cheapest model in ``model_ids`` that fits ``budget_usd``.
-
- ``model_ids`` should be ordered best-quality-first; the caller decides how
- much quality it is prepared to trade (see :mod:`codeverse.cost.routing`)."""
- scored: list[tuple[float, str, CostEstimate]] = []
- for mid in model_ids:
- est = estimate_call(mid, input_tokens=input_tokens, output_tokens=output_tokens,
- cached_tokens=cached_tokens)
- if not est.known:
- continue
- if budget_usd is not None and est.usd > budget_usd:
- continue
- scored.append((est.usd, mid, est))
- if not scored:
- return None
- scored.sort(key=lambda s: s[0])
- return scored[0][1], scored[0][2]
diff --git a/harness/codeverse/flywheel/__init__.py b/harness/codeverse/flywheel/__init__.py
index 55f9d005..bb26cfd9 100644
--- a/harness/codeverse/flywheel/__init__.py
+++ b/harness/codeverse/flywheel/__init__.py
@@ -9,6 +9,5 @@ Public API (see docs/INTERFACES.md):
* ``captions`` ``caption_sample(ws, record, model_id)`` → ``Captions``
* ``dedupe`` ``code_fingerprint`` · ``mesh_fingerprint`` · ``near_duplicates``
* ``quality`` ``quality_tier`` (A/B/C/D) · ``prompt_hash`` · exact (code, prompt) duplicate groups
-* ``trajectories`` in-session repair pairs mined from api-agent transcripts
* ``index`` ``build_index(runs_dir, out_sqlite)`` + query helpers
"""
diff --git a/harness/codeverse/flywheel/deliverable.py b/harness/codeverse/flywheel/deliverable.py
index 323806f7..70d96161 100644
--- a/harness/codeverse/flywheel/deliverable.py
+++ b/harness/codeverse/flywheel/deliverable.py
@@ -49,10 +49,6 @@ _ARTIFACT_ROLES: tuple[tuple[str, str], ...] = (
)
-class DeliverableError(RuntimeError):
- """The run has nothing that can be handed over."""
-
-
def _sha256(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
diff --git a/harness/codeverse/flywheel/pairs.py b/harness/codeverse/flywheel/pairs.py
index 3a016969..b9fe3463 100644
--- a/harness/codeverse/flywheel/pairs.py
+++ b/harness/codeverse/flywheel/pairs.py
@@ -18,9 +18,6 @@ Output is JSONL, one object per pair. All kinds share::
(``notes`` say "repair attempts: N (fixed)"): the ``rNN <kind>: generated``
commit is the rejected state, the round's final commit the chosen one; the
error message comes from the round's failing ``build.done`` events.
-* ``repair`` (``source: "trajectory"``) — the same shape mined from an agent
- session's transcript: last failing ``build`` → next successful ``build``
- (see ``trajectories.py``); ``error`` is the build tool's message.
* ``cross_backend`` the same (prompt, track, language) run under ≥ 2 generators;
best vs each other candidate with Δ ≥ ``min_delta``.
"""
@@ -36,11 +33,10 @@ from codeverse.contracts.run import RoundRecord, RunRecord
from codeverse.flywheel import _git
from codeverse.flywheel.quality import prompt_hash
from codeverse.flywheel.record import effective_judgment, iter_runs
-from codeverse.flywheel.trajectories import mine_run
from codeverse.workspace import Workspace
__all__ = ["build_pairs", "preference_pairs", "repair_pairs", "in_round_repair_pairs",
- "trajectory_repair_pairs", "cross_backend_pairs", "prompt_hash"]
+ "cross_backend_pairs", "prompt_hash"]
MAX_INLINE_CODE = 200_000
@@ -207,28 +203,6 @@ def in_round_repair_pairs(ws: Workspace, rec: RunRecord, *, slug: str | None = N
return out
-def trajectory_repair_pairs(ws: Workspace, rec: RunRecord, *, slug: str | None = None) -> list[dict[str, Any]]:
- """In-session repairs (failing build → fixing build) mined from api-agent transcripts."""
- out = []
- for tr in mine_run(ws):
- pair = _base(ws, rec, "repair", slug=slug)
- pair.update(
- source="trajectory",
- trajectory=tr.trajectory,
- chosen={"round": tr.round_index, "kind": tr.stage, "commit": "", "turn": tr.fixed_turn, "score": None,
- "passed": None, "build_ok": True, "files": tr.chosen, "truncated_files": []},
- rejected={"round": tr.round_index, "kind": tr.stage, "commit": "", "turn": tr.broken_turn, "score": None,
- "passed": None, "build_ok": False, "files": tr.rejected, "truncated_files": []},
- delta=None,
- error={"type": "", "message": tr.error, "file": "", "line": None, "stderr_tail": "", "stdout_tail": "",
- "gate_errors": []},
- reason=[f"in-session fix of a failing build ({tr.n_failures} failing build(s) folded)"],
- changed_files=tr.changed_files,
- )
- out.append(pair)
- return out
-
-
def cross_backend_pairs(
groups: dict[tuple[str, str, str], list[tuple[Workspace, RunRecord, str]]], *, min_delta: float
) -> list[dict[str, Any]]:
@@ -269,12 +243,8 @@ def cross_backend_pairs(
return out
-def build_pairs(
- runs_dir: Path | str, out_jsonl: Path | str, *, min_delta: float = 0.05, trajectories: bool = True
-) -> int:
- """Write all pair kinds for the runs under ``runs_dir``; returns the number written.
-
- ``trajectories`` adds in-session repair pairs mined from agent transcripts."""
+def build_pairs(runs_dir: Path | str, out_jsonl: Path | str, *, min_delta: float = 0.05) -> int:
+ """Write all pair kinds for the runs under ``runs_dir``; returns the number written."""
out = Path(out_jsonl)
out.parent.mkdir(parents=True, exist_ok=True)
groups: dict[tuple[str, str, str], list[tuple[Workspace, RunRecord, str]]] = defaultdict(list)
@@ -286,8 +256,6 @@ def build_pairs(
groups[(prompt_hash(rec.spec.prompt), rec.spec.track.value, rec.spec.language.value)].append((ws, rec, slug))
pairs = (preference_pairs(ws, rec, min_delta=min_delta, slug=slug)
+ repair_pairs(ws, rec, slug=slug) + in_round_repair_pairs(ws, rec, slug=slug))
- if trajectories:
- pairs += trajectory_repair_pairs(ws, rec, slug=slug)
for pair in pairs:
fh.write(json.dumps(pair, ensure_ascii=False) + "\n")
n += 1
diff --git a/harness/codeverse/flywheel/trajectories.py b/harness/codeverse/flywheel/trajectories.py
deleted file mode 100644
index ba7fcfb9..00000000
--- a/harness/codeverse/flywheel/trajectories.py
+++ /dev/null
@@ -1,203 +0,0 @@
-"""Mine repair pairs from agent trajectories (``trajectories/<stage>_rNN/transcript.jsonl``).
-
-Round-level repair pairs (``pairs.repair_pairs``) need a round whose build
-failed. Most build failures are fixed *inside* one agent session instead —
-the ``api-agent`` transcript records every ``write_file`` / ``edit_file`` /
-``build`` tool call, so the file state can be replayed::
-
- write_file ... → build FAIL (error) → edit/write ... → build OK
- ^ rejected snapshot ^ chosen snapshot
-
-Only the ``api-agent`` transcript format (``assistant`` turns with
-``tool_calls`` + ``tool_result`` turns) is structured enough; CLI backends
-(gemini-cli / codex / claude-code) log raw stdout lines and yield no pairs.
-
-The base file state is the ``pre:<label>`` git commit taken by the harness
-right before the session (nearest one not after the first transcript event);
-``read_file`` results are not used (they may be truncated).
-"""
-
-from __future__ import annotations
-
-import subprocess
-from pathlib import Path
-from typing import Any
-
-from pydantic import BaseModel, Field
-
-from codeverse.flywheel import _git
-from codeverse.proc import read_jsonl_lenient
-from codeverse.workspace import Workspace
-
-BUILD_TOOLS = frozenset({"build", "run_build"})
-MAX_PAIR_CODE = 200_000
-
-
-class TrajectoryRepair(BaseModel):
- """One in-session repair: the last failing snapshot before a successful build."""
-
- trajectory: str = Field(description="trajectory dir name, e.g. refine_r00")
- stage: str
- round_index: int | None
- broken_turn: int
- fixed_turn: int
- error: str
- rejected: dict[str, str] = Field(description="path → code at the failing build")
- chosen: dict[str, str] = Field(description="path → code at the next successful build")
- changed_files: list[str]
- n_failures: int = Field(description="consecutive failing builds folded into this pair")
-
-
-def parse_trajectory_name(name: str) -> tuple[str, int | None]:
- """``refine_r00`` → ``("refine", 0)``; ``zone_koi_pond_r01`` → ``("zone_koi_pond", 1)``."""
- stem, _, tail = name.rpartition("_r")
- if stem and tail.isdigit():
- return stem, int(tail)
- return name, None
-
-
-def _base_commit(ws: Workspace, label: str, not_after: float | None) -> str | None:
- """Latest ``pre:<label>`` commit at or before ``not_after`` (epoch seconds)."""
- try:
- proc = subprocess.run(
- ["git", "log", "--format=%H %ct %s"], cwd=ws.root, capture_output=True, text=True, check=False, timeout=30
- )
- except (OSError, subprocess.TimeoutExpired):
- return None
- if proc.returncode != 0:
- return None
- for line in proc.stdout.splitlines(): # newest first
- sha, _, rest = line.partition(" ")
- ts, _, subject = rest.partition(" ")
- if subject.strip() != f"pre:{label}":
- continue
- if not_after is None or float(ts) <= not_after + 1.0:
- return sha
- return None
-
-
-def _tracked(path: str) -> bool:
- parts = Path(path).parts
- return bool(parts) and parts[0] in _git.CODE_ROOTS
-
-
-def _normalise_path(ws: Workspace, raw: str) -> str:
- p = Path(raw)
- if p.is_absolute():
- try:
- p = p.relative_to(ws.root)
- except ValueError:
- return raw
- return p.as_posix()
-
-
-class _Replay:
- """Applies write/edit tool calls to an in-memory tree; tracks whether it is exact."""
-
- def __init__(self, base: dict[str, str]):
- self.files = dict(base)
- self.exact = True # False once an edit touched a file we never saw in full
- self.changed: set[str] = set()
-
- def write(self, path: str, content: str) -> None:
- self.files[path] = content
- self.changed.add(path)
-
- def edit(self, path: str, old: str, new: str, all_: bool) -> None:
- if path not in self.files:
- self.exact = False
- return
- text = self.files[path]
- if not old or old not in text:
- self.exact = False
- return
- self.files[path] = text.replace(old, new) if all_ else text.replace(old, new, 1)
- self.changed.add(path)
-
-
-def read_events(transcript: Path) -> list[dict[str, Any]]:
- return read_jsonl_lenient(transcript)
-
-
-def mine_trajectory(ws: Workspace, traj_dir: Path) -> list[TrajectoryRepair]:
- """Repair pairs from one trajectory folder (empty for non api-agent transcripts)."""
- transcript = traj_dir / "transcript.jsonl"
- if not transcript.is_file():
- return []
- events = read_events(transcript)
- calls: dict[str, dict[str, Any]] = {}
- for ev in events:
- if ev.get("kind") == "assistant":
- for c in ev.get("tool_calls") or []:
- if isinstance(c, dict) and c.get("id"):
- calls[str(c["id"])] = c
- if not calls:
- return []
- stage, rnd = parse_trajectory_name(traj_dir.name)
- t0 = next((float(ev["t"]) for ev in events if "t" in ev), None)
- base_sha = _base_commit(ws, stage, t0)
- base: dict[str, str] = {}
- if base_sha:
- try:
- base, _ = _git.decode_text_files(_git.read_tree_at(ws, base_sha))
- except _git.GitReadError:
- base = {}
- rp = _Replay(base)
- broken: tuple[int, dict[str, str], str, set[str]] | None = None # turn, snapshot, error, changed-before
- n_fail = 0
- out: list[TrajectoryRepair] = []
- for ev in events:
- if ev.get("kind") != "tool_result":
- continue
- call = calls.get(str(ev.get("call_id")))
- if call is None:
- continue
- name, args = str(call.get("name")), dict(call.get("arguments") or {})
- ok = bool(ev.get("ok"))
- turn = int(ev.get("turn", -1))
- if name == "write_file" and ok:
- path = _normalise_path(ws, str(args.get("path", "")))
- if _tracked(path):
- rp.write(path, str(args.get("content", "")))
- elif name == "edit_file" and ok:
- path = _normalise_path(ws, str(args.get("path", "")))
- if _tracked(path):
- rp.edit(path, str(args.get("old", "")), str(args.get("new", "")), bool(args.get("all", False)))
- elif name in BUILD_TOOLS:
- if not ok:
- n_fail += 1
- broken = (turn, dict(rp.files), str(ev.get("content", ""))[:4000], set(rp.changed))
- continue
- if broken is not None and rp.exact:
- b_turn, b_files, b_err, _ = broken
- changed = sorted(p for p in set(b_files) | set(rp.files) if b_files.get(p) != rp.files.get(p))
- if changed:
- out.append(TrajectoryRepair(
- trajectory=traj_dir.name, stage=stage, round_index=rnd, broken_turn=b_turn,
- fixed_turn=turn, error=b_err, rejected=_limit(b_files), chosen=_limit(rp.files),
- changed_files=changed, n_failures=n_fail,
- ))
- broken, n_fail = None, 0
- return out
-
-
-def _limit(files: dict[str, str], max_total: int = MAX_PAIR_CODE) -> dict[str, str]:
- out: dict[str, str] = {}
- total = 0
- for p in sorted(files):
- if total + len(files[p]) > max_total:
- break
- out[p] = files[p]
- total += len(files[p])
- return out
-
-
-def mine_run(ws: Workspace) -> list[TrajectoryRepair]:
- """All in-session repair pairs of one run (every ``trajectories/*`` folder)."""
- root = ws.root / "trajectories"
- if not root.is_dir():
- return []
- out: list[TrajectoryRepair] = []
- for d in sorted(p for p in root.iterdir() if p.is_dir()):
- out.extend(mine_trajectory(ws, d))
- return out
diff --git a/harness/codeverse/gallery/model.py b/harness/codeverse/gallery/model.py
index 962b92fc..08f4b8b0 100644
--- a/harness/codeverse/gallery/model.py
+++ b/harness/codeverse/gallery/model.py
@@ -24,7 +24,6 @@ RunState = Literal["ok", "pending", "broken"]
#: filter names the index page and ``/api/runs`` both understand
FILTER_KEYS = ("q", "track", "lang", "tier", "backend", "pass", "verdict", "battery")
-SORT_KEYS = ("score", "cost", "time", "complexity", "name")
class RunLink(BaseModel):
diff --git a/harness/codeverse/gallery/static_site.py b/harness/codeverse/gallery/static_site.py
index a8c54550..67914626 100644
--- a/harness/codeverse/gallery/static_site.py
+++ b/harness/codeverse/gallery/static_site.py
@@ -30,11 +30,6 @@ def render_static(index: GalleryIndex, *, title: str = "3dcv gallery", embed: bo
title=title, sort=sort, view=view, note=note, extra_html=extra_html)
-def write_atomic(path: Path, text: str) -> Path:
- """tmp + rename, so a reader never sees a partial gallery."""
- return write_text_atomic(path, text)
-
-
def build_static(roots: list[Path] | list[str], out_html: Path | str, *, title: str | None = None,
embed: bool = False, thumb_px: int = THUMB_PX) -> tuple[Path, int, GalleryIndex]:
"""Scan ``roots`` and write one self-contained page; ``(path, n_runs, index)``."""
@@ -42,4 +37,4 @@ def build_static(roots: list[Path] | list[str], out_html: Path | str, *, title:
label = title or ("3dcv gallery — " + ", ".join(s.label for s in index.sections[:4])
+ ("…" if len(index.sections) > 4 else ""))
html = render_static(index, title=label, embed=embed, thumb_px=thumb_px)
- return write_atomic(Path(out_html), html), len(index.entries()), index
+ return write_text_atomic(Path(out_html), html), len(index.entries()), index
diff --git a/harness/codeverse/judges/metrics.py b/harness/codeverse/judges/metrics.py
deleted file mode 100644
index 73a5e2bf..00000000
--- a/harness/codeverse/judges/metrics.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""Small, pure helpers over judgments and round scores (no I/O, no models)."""
-
-from __future__ import annotations
-
-import statistics
-from collections.abc import Sequence
-
-from pydantic import BaseModel, Field
-
-from codeverse.contracts.judgment import Judgment
-
-
-class Agreement(BaseModel):
- mean: float
- std: float
- per_criterion_std: dict[str, float] = Field(default_factory=dict)
- n: int
-
-
-def judge_agreement(judgments: Sequence[Judgment]) -> Agreement:
- """Mean/std of ``overall`` and per-criterion std across repeated judgments of the same thing."""
- if not judgments:
- raise ValueError("judge_agreement needs at least one judgment")
- overalls = [j.overall for j in judgments]
- crit: dict[str, list[float]] = {}
- for j in judgments:
- for k, v in j.scores.items():
- crit.setdefault(k, []).append(float(v))
- return Agreement(
- mean=round(statistics.fmean(overalls), 4),
- std=round(statistics.pstdev(overalls), 4) if len(overalls) > 1 else 0.0,
- per_criterion_std={k: (round(statistics.pstdev(v), 4) if len(v) > 1 else 0.0) for k, v in crit.items()},
- n=len(judgments),
- )
-
-
-def plateau(scores: Sequence[float], window: int = 2, min_delta: float = 0.02) -> bool:
- """True when the best of the last ``window`` scores did not beat the best before them by ≥ ``min_delta``.
-
- Needs at least ``window + 1`` scores; fewer → False (keep going).
- """
- if window < 1:
- raise ValueError("window must be ≥ 1")
- if len(scores) < window + 1:
- return False
- before = max(scores[:-window])
- recent = max(scores[-window:])
- return (recent - before) < min_delta
-
-
-def best_index(rounds: Sequence[tuple[float, int]]) -> int:
- """Index of the best round: higher score, tie → fewer errors, tie → later round."""
- if not rounds:
- raise ValueError("best_index needs at least one round")
- best = 0
- for i, (score, n_err) in enumerate(rounds):
- bs, be = rounds[best]
- if score > bs or (score == bs and n_err < be) or (score == bs and n_err == be):
- best = i
- return best
diff --git a/harness/codeverse/models/health.py b/harness/codeverse/models/health.py
index 2ab3c295..b905298c 100644
--- a/harness/codeverse/models/health.py
+++ b/harness/codeverse/models/health.py
@@ -225,36 +225,6 @@ def _is_delegating_driver(args: list[str]) -> bool:
return args[i + 1 : i + 2] != ["cell"]
-def sibling_processes() -> int:
- """How many OTHER harness processes are running on this machine.
-
- ``shared_pool`` keeps one KeyPool per PROCESS, so each sibling believes it owns the
- whole key quota and gets its own ``max_in_flight``. N siblings multiply the real
- concurrency by N against a quota that is shared: on 2026-08-24 six concurrent
- batteries took gemini-3.7-flash from 25 % success to 0 % (``docs/COST.md`` §23).
-
- Best-effort — reads /proc and returns 0 when it cannot tell, because a wrong number
- here must never block a run.
- """
- import os
-
- me, found = os.getpid(), 0
- try:
- entries = list(Path("/proc").iterdir())
- except OSError:
- return 0
- for entry in entries:
- if not entry.name.isdigit() or int(entry.name) == me:
- continue
- try:
- raw = (entry / "cmdline").read_bytes()
- except OSError:
- continue # exited between listing and reading
- if _is_harness_argv([a for a in raw.decode(errors="replace").split("\0") if a]):
- found += 1
- return found
-
-
#: the measured per-process knee (docs/COST.md §20); the machine-wide budget is the same
#: number, because the provider sees one machine's worth of traffic, not one process's
POOL_KNEE = 64
@@ -321,8 +291,8 @@ def pool_budget() -> PoolBudget:
so the SUM of ``max_in_flight`` across processes must stay at the knee, not each
process alone. Counting *processes* (the first version) made every agent refuse to
launch while any sibling existed, and on 2026-08-24 that stalled a whole A/B wave
- behind two batteries that were themselves parked. Best-effort like
- :func:`sibling_processes`: on any /proc trouble it reports no siblings and no usage.
+ behind two batteries that were themselves parked. Best-effort: on any /proc
+ trouble it reports no siblings and no usage.
"""
import os
diff --git a/harness/codeverse/orchestrator/rounds.py b/harness/codeverse/orchestrator/rounds.py
index bf05dd18..11555b95 100644
--- a/harness/codeverse/orchestrator/rounds.py
+++ b/harness/codeverse/orchestrator/rounds.py
@@ -34,7 +34,6 @@ from dataclasses import dataclass, replace
from typing import Literal
from codeverse.contracts.run import RoundRecord
-from codeverse.judges.metrics import best_index
log = logging.getLogger(__name__)
@@ -351,10 +350,22 @@ def last_gain(history: Sequence[RoundRecord]) -> float | None:
return float(history[-1].score) - before
+def best_index(rounds: Sequence[tuple[float, int]]) -> int:
+ """Index of the best round: higher score, tie → fewer errors, tie → later round."""
+ if not rounds:
+ raise ValueError("best_index needs at least one round")
+ best = 0
+ for i, (score, n_err) in enumerate(rounds):
+ bs, be = rounds[best]
+ if score > bs or (score == bs and n_err < be) or (score == bs and n_err == be):
+ best = i
+ return best
+
+
class BestSelector:
"""Best round = highest score, then fewer gate errors, then later.
- ``codeverse.judges.metrics.best_index`` owns the ranking rule. A round that
+ ``best_index`` above owns the ranking rule. A round that
did not build is never picked (it has no score, and delivering code that does
not run is never an improvement); when NO round has a score the fallback is
the built round with the fewest gate errors — later on a tie — so a round that
@@ -385,7 +396,6 @@ def _build_failed(r: RoundRecord) -> bool:
# Moved to codeverse.orchestrator.refine_tasks; imported here so existing importers
# (tracks/*, reference/mismatch, tests) keep working unchanged.
from codeverse.orchestrator.refine_tasks import ( # noqa: E402
- FileForTarget,
RefineTask,
TaskGroup,
build_refine_instructions,
@@ -395,7 +405,7 @@ from codeverse.orchestrator.refine_tasks import ( # noqa: E402
)
__all__ = [
- "BestSelector", "DEFAULT_JUDGE_SIGMA", "DETAIL_KIND", "FileForTarget", "KIND_FOR_STRATEGY",
+ "BestSelector", "DEFAULT_JUDGE_SIGMA", "DETAIL_KIND", "KIND_FOR_STRATEGY",
"REWRITE_KIND", "RefineTask", "RoundPolicy", "StopDecision", "StopPolicy", "StopReason",
"Strategy", "TaskGroup", "best_score", "build_refine_instructions", "compact_instructions",
"detail_blocked", "gate_error_count", "judge_sigma", "kind_for_strategy", "last_gain",
diff --git a/harness/codeverse/orchestrator/runner.py b/harness/codeverse/orchestrator/runner.py
index a16c264e..a9bb241f 100644
--- a/harness/codeverse/orchestrator/runner.py
+++ b/harness/codeverse/orchestrator/runner.py
@@ -56,10 +56,6 @@ def _jsonable(obj: Any) -> Any:
return obj
-class StageError(RuntimeError):
- """A stage function raised; the original exception is ``__cause__``."""
-
-
class StageRunner:
"""Runs stages with resume semantics on top of ``RunState`` + ``EventLog``."""
diff --git a/harness/codeverse/proc.py b/harness/codeverse/proc.py
index 3020bdee..a7c958df 100644
--- a/harness/codeverse/proc.py
+++ b/harness/codeverse/proc.py
@@ -431,7 +431,7 @@ def read_json_or_none(path: Path | str, *, errors: str | None = None) -> dict[st
replaced identical ``try: json.loads(read_text()) except (OSError, ValueError)``
copies in ``flywheel/telemetry.read_json``, ``bench/complexity_report._read_json``,
``gallery/index._measurement_complexity`` / ``_spec_fields``,
- ``cli/main._best_round_of_record``, ``runlock._holder``,
+ ``cli/inspect_cmd._best_round_of_record``, ``runlock._holder``,
``languages/threejs/lint._load_plan``, ``cost/reconstruct._read_json``,
``spatial/_render_common.read_json`` and ``cli/_judge.plan_summary_for`` — all
the same algorithm, differing only in whether they also checked ``isinstance(dict)``
diff --git a/harness/codeverse/tracks/plan_features.py b/harness/codeverse/tracks/plan_features.py
index c961efc7..c7a05cad 100644
--- a/harness/codeverse/tracks/plan_features.py
+++ b/harness/codeverse/tracks/plan_features.py
@@ -1,56 +1,37 @@
-"""Feature switches for the plan-loop-engineering wave, one env variable for all of them.
-
-Every change in that wave (the plan fit check, the rendered contact graph, the brief
-attachment graph, …) ships OFF and is A/B'd one at a time with ``bench/ab_plan.py``,
-whose arms differ only by environment. So the switch has to be an env variable, it
-has to be read at call time (never cached in ``Settings``), and every change has to
-read the SAME variable — a second spelling per change is how a control arm quietly
-turns into a variant.
-
-``CV3D_PLAN_FEATURES="fit,contacts"`` turns two features on; ``all`` turns every known
-one on and ``all,-graph`` all but one. Unknown names are ignored (and logged once)
-rather than fatal: a typo in a bench command must produce a control run, not a crash
-half-way through a battery.
-
-STATUS (CQ-5): none of the six features below is implemented, and nothing outside this
-module reads ``CV3D_PLAN_FEATURES`` — the plan-loop changes that DID land each invented
-their own spelling instead (see :data:`LIVE_SWITCHES`). The switch is therefore DEAD,
-and an A/B whose arms differ only by it is a no-op by construction: one such run is on
-record printing "keep, mean delta +0.344" for two byte-identical arms. :data:`DEAD_SWITCHES`
-exists so ``bench/ab_plan.py`` refuses that A/B instead of producing a verdict. Delete
-this module, or implement a feature and move its variable into ``LIVE_SWITCHES``.
+"""The A/B switch registry: which env switches any code path ACTUALLY reads.
+
+Born as the switchboard for the plan-loop-engineering wave (CQ-5): six features were
+declared under ``CV3D_PLAN_FEATURES``, none was ever implemented, and one A/B run is on
+record printing "keep, mean delta +0.344" for two byte-identical arms. The feature
+runtime (``parse_features`` / ``plan_feature_on``) was deleted 2026-08-28 per its own
+instruction; the six names stay only so ``pin_plan_blockers`` can classify a pinned
+A/B, and what else remains is the guard rail — :data:`LIVE_SWITCHES` / :data:`DEAD_SWITCHES` (kept honest by
+``tests/orchestrator_tracks/test_plan_features.py``, which greps the tree) and the
+``--pin-plan`` blocker rule ``bench/ab_plan.py`` consults so a paired A/B cannot
+silently pin away the very thing it is testing.
"""
from __future__ import annotations
-import logging
-import os
-
-log = logging.getLogger(__name__)
-
PLAN_FEATURES_ENV = "CV3D_PLAN_FEATURES"
-#: the switches the wave defines, in the order the design doc lists them
-#: (``scratchpad/planloop/design/DESIGN.md``). Add a name here in the same commit as
-#: the code that reads it; a feature that is not listed cannot be turned on.
-FIT = "fit" # plan-level fit check: undeclared deep overlaps, declared gaps → one repair re-ask
-CONTACTS = "contacts" # every declared contact rendered to the builder with its mating faces and overlap rule
-GRAPH = "graph" # brief sub-assemblies carry `mounts_on` edges; parts are grouped by node
-GRAPH_BUDGET = "graph_budget" # plan budget derived from the brief graph instead of the flat floor
-GRAPH_EXAMPLE = "graph_example" # the planner's worked example written as a graph (touches + assembly filled)
-CONSISTENCY = "consistency" # plan overall bbox checked against the brief's dimension rows
+#: the six switch names the wave defined. Nothing reads them at runtime any more —
+#: they remain because ``pin_plan_blockers`` still classifies a CV3D_PLAN_FEATURES
+#: value inside a MIXED A/B env, and that classification must not silently widen.
+FIT = "fit"
+CONTACTS = "contacts"
+GRAPH = "graph"
+GRAPH_BUDGET = "graph_budget"
+GRAPH_EXAMPLE = "graph_example"
+CONSISTENCY = "consistency"
KNOWN_FEATURES: tuple[str, ...] = (FIT, CONTACTS, GRAPH, GRAPH_BUDGET, GRAPH_EXAMPLE, CONSISTENCY)
-
-#: Which stage a switch changes. This is not documentation — ``bench/ab_plan.py --pin-plan``
-#: reads it, and pinning a PLAN-side switch would silently disable the very thing under test
-#: (both arms would share one plan, so a change that only alters planning becomes a no-op the
-#: rig would then report as "no effect"). A switch missing from here is treated as plan-side:
-#: refusing to pin costs one noisy A/B, pinning wrongly costs a confident wrong answer.
-#: docs/EVAL.md §8.1 measured why pinning matters — the A/A's worst pair differed 1 part vs 10.
+#: which stage each switch would have changed — read by ``pin_plan_blockers``
PLAN_SIDE: frozenset[str] = frozenset({FIT, GRAPH, GRAPH_BUDGET, GRAPH_EXAMPLE, CONSISTENCY})
GENERATION_SIDE: frozenset[str] = frozenset({CONTACTS})
-#: env switches outside CV3D_PLAN_FEATURES, same rule
-PLAN_SIDE_ENV: frozenset[str] = frozenset({"CV3D_PLAN_BRIEF", "CV3D_SCOPED_PARTS"})
+#: env switches that act AFTER planning — ``--pin-plan`` may share one plan across arms
+#: that differ only by these. Anything not listed is treated as plan-side: refusing to
+#: pin costs one noisy A/B, pinning wrongly costs a confident wrong answer
+#: (docs/EVAL.md §8.1 measured it — the A/A's worst pair differed 1 part vs 10).
GENERATION_SIDE_ENV: frozenset[str] = frozenset({"CV3D_SKILLS", "CV3D_SKILLS_MAX", "CV3D_SKILLS_UNVERIFIED",
"CV3D_SKILLS_ONLY",
"CV3D_DETAIL_ROUNDS", "CV3D_REFERENCE_DIFF",
@@ -111,49 +92,6 @@ def dead_env_keys(env: dict[str, str]) -> list[str]:
return sorted(k for k in env if k in DEAD_SWITCHES)
-_warned: set[str] = set()
-
-
-def parse_features(raw: str | None) -> frozenset[str]:
- """``"fit, contacts"`` → ``{fit, contacts}``; ``"all,-graph"`` → everything but graph.
-
- Pure, so it is testable without touching the environment. Unknown names are
- dropped and logged once per process (see the module docstring for why not fatal)."""
- if not raw or not raw.strip():
- return frozenset()
- on: set[str] = set()
- for tok in (t.strip().lower() for t in raw.split(",")):
- if not tok:
- continue
- negate = tok.startswith("-")
- name = tok[1:] if negate else tok
- names = set(KNOWN_FEATURES) if name == "all" else {name}
- unknown = names - set(KNOWN_FEATURES)
- for u in unknown - _warned:
- _warned.add(u)
- log.warning("%s: unknown plan feature %r ignored (known: %s)", PLAN_FEATURES_ENV, u, ", ".join(KNOWN_FEATURES))
- names -= unknown
- if negate:
- on -= names
- else:
- on |= names
- return frozenset(on)
-
-
-def plan_features() -> frozenset[str]:
- """The features switched on for this process, read from the environment NOW."""
- return parse_features(os.environ.get(PLAN_FEATURES_ENV))
-
-
-def plan_feature_on(name: str) -> bool:
- """Is feature ``name`` on? Raises on a name the wave does not define, because a
- caller asking about a feature that cannot be switched on is a bug, not a config."""
- if name not in KNOWN_FEATURES:
- raise ValueError(f"unknown plan feature {name!r}; known: {', '.join(KNOWN_FEATURES)}")
- return name in plan_features()
-
-
-__all__ = ["CONSISTENCY", "CONTACTS", "DEAD_SWITCHES", "FIT", "GENERATION_SIDE", "GENERATION_SIDE_ENV",
- "GRAPH", "GRAPH_BUDGET", "GRAPH_EXAMPLE", "PLAN_SIDE", "PLAN_SIDE_ENV", "pin_plan_blockers",
- "KNOWN_FEATURES", "LIVE_SWITCHES", "PLAN_FEATURES_ENV", "dead_env_keys", "parse_features",
- "plan_feature_on", "plan_features"]
+__all__ = ["CONSISTENCY", "CONTACTS", "DEAD_SWITCHES", "FIT", "GENERATION_SIDE",
+ "GENERATION_SIDE_ENV", "GRAPH", "GRAPH_BUDGET", "GRAPH_EXAMPLE", "KNOWN_FEATURES",
+ "LIVE_SWITCHES", "PLAN_FEATURES_ENV", "PLAN_SIDE", "dead_env_keys", "pin_plan_blockers"]
diff --git a/harness/docs/ARCHITECTURE.md b/harness/docs/ARCHITECTURE.md
index d974fea5..e2f0a101 100644
--- a/harness/docs/ARCHITECTURE.md
+++ b/harness/docs/ARCHITECTURE.md
@@ -24,11 +24,11 @@ layered on top of the language — for four **tracks**:
…driven by **any** of these backends behind two protocols:
* `ChatModel` (API): `gemini:*`, `anthropic:*`, `openai:*` — planner, judge,
- captioner, single-shot generation, and the in-process `api-agent`.
-* `CodingAgent` (agentic session on a workspace): `gemini-cli:*`, `claude-code:*`,
- `codex:*`, `agy:*` (Antigravity), `api-agent:<chat-model>` (our own tool loop so
- API-only users get parity). Plus `single-shot:<chat-model>` (one envelope of
- files, no tools) handled inside `tracks/generation.py`.
+ captioner, single-shot generation.
+* `CodingAgent` (agentic session on a workspace): always a vendor CLI —
+ `gemini-cli:*`, `claude-code:*`, `codex:*`, `agy:*` (Antigravity); the in-process
+ `api-agent` was deleted 2026-08-28. Plus `single-shot:<chat-model>` (one envelope
+ of files, no tools) handled inside `tracks/generation.py`.
Every run is: **spec → plan → skeleton → [scene stages] → baseline round
(optionally best-of-N candidates) → refine rounds → finalise → record**, with an
@@ -100,10 +100,9 @@ codeverse/
fallback), retry.py (MAX_WAIT_S: no single wait > 3 s), storm.py (shared 503 gate —
measured, ships OFF, see docs/COST.md §21), health.py (preflight probe: is the model
serving? no retries, no backoff), schema_utils.py (strict schema), registry.py
- agents/ CodingAgent; gemini_cli.py claude_code.py codex.py antigravity.py api_agent.py
- (+ api_tools.py run_shell policy, api_skills.py: the read_skill tool every
- backend WITHOUT a native loader gets — measured, prose in a system prompt is
- not an affordance), materialize.py, cli_common.py (sessions, retry
+ agents/ CodingAgent; gemini_cli.py claude_code.py codex.py antigravity.py
+ (api_agent.py + api_tools.py + api_skills.py deleted 2026-08-28 —
+ the vendors already ship the loop), materialize.py, cli_common.py (sessions, retry
trajectory naming, files_changed attribution), watchdog.py, transcript.py, registry.py
languages/ LanguageRuntime; blender/ (multi-file: layout.py, model.py + parts/*.py) cadquery/
threejs/ (+ templates.py) urdf/ scene_threejs/ glsl_shader/ (wrap.py header+line-map)
@@ -178,8 +177,7 @@ codeverse/
skills_hook.py (the round's view of codeverse/skills: attach before generating,
probe reads after — a no-op unless CV3D_SKILLS is on)
flywheel/ record.py, export.py, pack.py, sample.py, pairs.py, migrate.py (schema moves),
- deliverable.py, telemetry.py, trajectories.py (repair-pair
- mining), captions.py, quality.py (tiers + dedupe), dedupe.py, index.py,
+ deliverable.py, telemetry.py, captions.py, quality.py (tiers + dedupe), dedupe.py, index.py,
code_quality.py (the delivered CODE's own vector — magic numbers per 100 LOC,
function length, dead functions, duplication, docstrings → record.extra
["code_quality"].index, a flywheel filter beside score and complexity)
@@ -295,8 +293,8 @@ summary:
`@tool(name, ArgsModel, description, *, tracks=(), languages=(), cost_hint)` registers
`fn(ctx, args) -> Observation` → (a) direct call from tracks, (b) the stdio MCP
-server (name `3dcv`) for gemini-cli / claude-code / codex, (c) native tool schema
-for `api-agent`, (d) a prompt card. Tools: `build`, `measure`, `render_views`,
+server (name `3dcv`) for the vendor CLIs, (c) a native tool schema for any embedder
+(`ToolDef.schema()`), (d) a prompt card. Tools: `build`, `measure`, `render_views`,
`render_sheet`, `isolate`, `cross_section`, `check_connectivity`, `check_contract`,
`compare_silhouette`, `joint_sweep` (articulated), `shader_probe`, `scene_probe`,
`scene_views` (scene), `gl_probe`, `gl_frames` (graphics), `texture_pass`,
@@ -437,8 +435,7 @@ $0.02–0.03 (flash) / ~$0.2 (pro); api-agent generation 3–6 min per object ro
* Scenes: fps is a relative cost; camera-in-geometry can miss open-back enclosures.
* threejs: textures are stripped on GLB export (the texture pass re-adds them as a
derived pack); `userData.tick` cannot survive export.
-* Trajectory repair mining only understands api-agent transcripts; CLI backends
- produce raw stdout. agy exposes no per-workspace MCP, cost or served model.
+* agy exposes no per-workspace MCP, cost or served model.
* Anthropic / OpenAI backends are mock-tested only (no keys on this box).
* Budget checks run between steps: a refine round that finishes its judge and then
trips the budget is not promoted to best — give scenes `--max-minutes 60 --max-usd 4`.
@@ -456,9 +453,9 @@ passed, C best ≥ 0.6, D else), acceptance checklists, gate summaries and
`(code fingerprint, prompt)` dedupe (`--drop-duplicates`; side-car captions via
`--captions-dir`); `metadata.parquet`/`.jsonl` + sqlite index carry tier, gate
errors, cost, fingerprints, `duplicate_of`; `--pack` tars with byte-range locators.
-`flywheel pairs` emits preference pairs (round i < j by judge Δ ≥ τ), round-level
-repair pairs and **in-session repair pairs mined from api-agent transcripts**
-(replay-verified against the git snapshots); `flywheel caption` adds
+`flywheel pairs` emits preference pairs (round i < j by judge Δ ≥ τ) and round-level
+repair pairs (in-session trajectory mining died with the api-agent, 2026-08-28 —
+vendor CLIs log raw stdout, not structured tool turns); `flywheel caption` adds
{detailed, instruction, factory} captions (image-grounded, brand-free, `--out` for
side-car mode); `flywheel gallery` is an alias of `3dcv gallery build --embed`
(the flywheel package has no renderer of its own).
diff --git a/harness/docs/COST.md b/harness/docs/COST.md
index 171fd3c0..783f1f47 100644
--- a/harness/docs/COST.md
+++ b/harness/docs/COST.md
@@ -1233,7 +1233,7 @@ guard is now `len(failed_keys) < len(pool)`.
-**Follow-up, same day, from the time audit (51 storm-day runs vs 52 baseline; `time_audit/REPORT.md`).**
+**Follow-up, same day, from the time audit (51 storm-day runs vs 52 baseline; scripts retired 2026-08-28 — findings preserved in `docs/TIME_AUDIT_2026-08-26.md`).**
Three accelerations, all additive and on by default:
*A caller clips the retry budget to what it can afford* (`ChatRequest.max_wait_s`, None = the
diff --git a/harness/docs/INTERFACES.md b/harness/docs/INTERFACES.md
index 54140eab..55f4f0a8 100644
--- a/harness/docs/INTERFACES.md
+++ b/harness/docs/INTERFACES.md
@@ -12,7 +12,7 @@ file was reconciled against it on 2026-08-23 (waves 2–3 + fix batch 1).
* Chat models: `<provider>:<model>` — `gemini:*` · `anthropic:*` · `openai:*`
(`models.registry.parse_model_id`).
* Coding agents: `<kind>:<model>` — `gemini-cli:*` · `claude-code:*` · `codex:*` ·
- `agy:*` · `api-agent:<provider>:<model>` (`agents.registry.parse_agent_id`).
+ `agy:*` (`agents.registry.parse_agent_id`).
* **Δ** Single-shot generation ids: `single-shot:<provider>:<model>` — handled only
inside `tracks/generation.py` (`is_single_shot`, `single_shot_model_id`); never
registered in `agents/`. `Spec.backends.generator` accepts any of the three.
@@ -107,7 +107,7 @@ with run_ledger(ws.root, run=slug): # binds the run, points record_call a
# models.get_chat_model returns a MeteredChatModel → ONE row per ChatModel.generate
# agents.get_coding_agent returns a MeteredAgent → ambient round/stage for the session, the
# profile's turn cap (only ever LOWERS job.max_turns), and — iff NOT meters_own_calls(agent),
- # i.e. the backend is not in IN_PROCESS_AGENT_KINDS = {"api-agent"} — one session row.
+ # a declaration no shipped backend makes — one session row.
# The rule is the BACKEND, never "did a row get written while it ran": a CLI session with one
# in-process tool call used to be dropped entirely, an in-process session whose turns ran in
# another thread used to be counted twice. A backend may declare `meters_own_calls`.
@@ -118,7 +118,7 @@ from codeverse.cost.context import CallContext, call_context, bind_run, context_
# texture pass that bills a model INSIDE an agent session is filed under its own stage, not the
# session's; a generation label yields to the session (best-of-N: kind="candidate" beats
# label="baseline"); Stage.OTHER means "the label said nothing" and displaces nothing.
- # labels understood: api-agent:<job label>:t<turn> · judge:<rubric>:r<NN>:s<k> · planner · pairwise:… · texture… · caption…
+ # labels understood: judge:<rubric>:r<NN>:s<k> · planner · pairwise:… · texture… · caption… (api-agent:<label>:t<turn> in historical ledgers only)
from codeverse.cost.caching import Block, order_blocks, prefix_report, session_cache, session_key
session_cache(rows) -> [SessionCache] # per session: cold first call, cached share, saved_usd, cold_usd
from codeverse.cost.profiles import get_profile, PROFILES # economy | balanced | quality
@@ -167,10 +167,9 @@ one workspace; harness-owned paths and sibling sessions' hinted files are droppe
the entry file), `read_only: list[str]` (harness-owned files INSIDE the write roots the
session may read but never write — `contracts.common.HARNESS_OWNED_SRC[language]`, i.e.
`src/recipes.glsl` for glsl_shader; `tracks/generation.py` sets it on every job of the
-run and `FileTools(read_only=…)` answers a `write_file` / `edit_file` on one with
-`PathDenied("… is harness-owned and read-only … call them from your own files")`)
-and `job.api: ApiAgentOptions(max_usd, temperature, thinking, allow_shell)` (api-agent
-only). **Δ legacy lift**: the same keys passed inside `extra={...}` are lifted into
+run and `agents/cli_common._enforce_scope` reverts a post-session write to one and
+fails that session). **Δ legacy lift**: the job keys (`round`, `kind`, `language`,
+`track`, `files_hint`, `mcp_command`) passed inside `extra={...}` are lifted into
the typed fields at validation (extra itself is left untouched), so old constructors
and serialized jobs keep working; `extra` stays for one-off backend hints.
Trajectories: `ws/trajectories/<label>_rNN/{prompt.md, transcript.jsonl, stdout.json,
@@ -185,14 +184,6 @@ applied LAST and `mcp.allowed` replaces, so an agent-planted server in ws/.gemin
cached; `tokens.input` is the uncached count) and each served model is priced at its
own rate. KeyPoolExhausted never escapes `run()` (→ `exit_reason=budget`); retries
prefer a different key (10 s wait) before re-using the same one.
-api-agent tools: `read_file`, `write_file`, `edit_file`, `list_files`, `run_shell`
-+ every spatial tool (`build` keeps its real name; `run_build` is an alias).
-**Δ** `run_shell` is a *policy filter*, not an OS sandbox: allow-listed programs only,
-no inline-code/REPL/loader flags (`-c/-e/-p/-i/--eval/--require/--import/…`),
-`python -m` restricted to a small allow-list, every path-like argument
-realpath-confined to the workspace (and never `.git/`). `FileTools.call` never
-raises — bad arguments / non-UTF-8 files come back as `ToolOutcome(is_error=True)`.
-
## languages/
```python
from codeverse.languages import get_runtime # (Language | str) -> LanguageRuntime
@@ -436,8 +427,7 @@ from codeverse.flywheel.export import export_samples # (runs_dir, out_dir, *,
# overwrite=True, include_unbuilt=False, captions_dir=None, drop_duplicates=False) -> ExportReport{…, n_duplicates, duplicates, tiers}
from codeverse.flywheel.quality import quality_tier, prompt_hash, find_duplicates # tiers: A passed & 0 gate errors, B passed,
# C best ≥ 0.6, D else; dedupe = (code fingerprint, prompt)
-from codeverse.flywheel.pairs import build_pairs # (runs_dir, out_jsonl, *, min_delta=0.05, trajectories=True) -> n
-from codeverse.flywheel.trajectories import mine_run # in-session repair pairs from api-agent transcripts (replay-verified)
+from codeverse.flywheel.pairs import build_pairs # (runs_dir, out_jsonl, *, min_delta=0.05) -> n
from codeverse.flywheel.captions import caption_sample # Δ (ws, record, model_id, *, model=None, out_dir=None) -> Captions;
# out_dir → side-car <out_dir>/<slug>.json, run untouched
from codeverse.gallery import build_index, default_roots, build_static, serve, GalleryApp # THE local gallery
diff --git a/harness/docs/RUNBOOK.md b/harness/docs/RUNBOOK.md
index ca20d366..0c45b9cd 100644
--- a/harness/docs/RUNBOOK.md
+++ b/harness/docs/RUNBOOK.md
@@ -155,7 +155,7 @@ unless you type `--host` yourself; it never serves a path outside the declared r
3dcv tools list [--cards] · 3dcv tools measure --workspace runs/<slug> · 3dcv tools gl_frames --workspace … --json '{"times":[0,1,2.5]}'
3dcv flywheel export runs/ dataset/ [--min-score 0.7] [--only-passed] [--pack] [--include-unbuilt]
[--captions-dir caps/] [--drop-duplicates]
-3dcv flywheel pairs runs/ pairs.jsonl [--min-delta 0.05] [--no-trajectories]
+3dcv flywheel pairs runs/ pairs.jsonl [--min-delta 0.05]
3dcv flywheel caption <slug> [--model …] [--out caps/] # --out = side-car mode, run untouched
3dcv flywheel gallery runs/ gallery.html [--title …] # alias of `3dcv gallery build --embed` (§4)
3dcv flywheel index runs/ runs_index.sqlite · 3dcv flywheel dedupe dataset/
diff --git a/harness/docs/SKILLS.md b/harness/docs/SKILLS.md
index ce390384..24d2fa04 100644
--- a/harness/docs/SKILLS.md
+++ b/harness/docs/SKILLS.md
@@ -10,7 +10,9 @@ names the cookbook section to fetch. Two libraries of truth would be worse than
`SKILL.md` is an open standard (agentskills.io, Dec 2025), which is why the format below is
not ours to bend: **claude-code, codex, gemini-cli and agy discover these bundles
-themselves** — proven live, see §7. Only `api-agent` needs harness-side injection.
+themselves** — proven live, see §7. (The in-process `api-agent`, the one backend that
+needed harness-side injection, was deleted 2026-08-28; an unclassified backend still
+gets the routed index as the safe default.)
Everything is behind `CV3D_SKILLS`. **It ships OFF**, because `api-agent` read 0 of 5
routed bundles in the measured A/B while the three subscription CLIs read all five.
@@ -32,7 +34,6 @@ one file, `skills/delivery.py`:
|---|---|---|
| `claude-code` | yes | `.claude/skills` |
| `codex`, `gemini-cli`, `agy` | yes | `.agents/skills` |
-| `api-agent` | no | `.agents/skills` |
| *anything unclassified* | **no** (safe default) | `.agents/skills` |
Both bits were read out of the shipped binaries, not assumed. Everything else is derived:
@@ -203,7 +204,7 @@ answerable. Per session, `telemetry/skills.jsonl` gets one line.
3dcv doctor --skills # library + discovery wiring
```
-Targets: ≥ 60% deep-read for CLI backends, ≥ 80% for api-agent, counted only over sessions
+Targets: ≥ 60% deep-read for CLI backends (the api-agent's 80% target died with it, 2026-08-28), counted only over sessions
the control (§1) says were measurable. **A skill under 20% over 20 such sessions is merged
or deleted.** A library that only ever grows is how this ends as bloat.
The read rate is the FIRST readout, before any score: `read_cookbook` was called by 0 of 16
diff --git a/harness/tests/cost/test_guard_caching_routing.py b/harness/tests/cost/test_guard_caching_routing.py
index 8f7c7306..4782eceb 100644
--- a/harness/tests/cost/test_guard_caching_routing.py
+++ b/harness/tests/cost/test_guard_caching_routing.py
@@ -8,9 +8,7 @@ import pytest
from codeverse.contracts.common import Usage
from codeverse.cost import (
Block,
- CostGuard,
cache_efficiency,
- cheapest_affordable,
estimate_call,
order_blocks,
prefix_report,
@@ -43,28 +41,6 @@ def test_estimate_of_an_image_model_uses_the_per_image_price():
assert est.usd == pytest.approx(500 * 0.5 / 1e6 + 2 * 0.067)
-def test_guard_blocks_only_what_it_can_price():
- guard = CostGuard(budget_usd=0.10, reserve_usd=0.02)
- cheap = estimate_call("gemini:gemini-3.7-flash", input_tokens=10_000, output_tokens=500)
- allow, why = guard.check(cheap)
- assert allow and not why
- guard.commit(cheap.usd)
- dear = estimate_call("gemini:gemini-3.1-pro-preview", input_tokens=2_000_000, output_tokens=50_000)
- allow, why = guard.check(dear)
- assert not allow and "left of" in why and guard.rejected
- unknown = estimate_call("gemini:no-such-model", input_tokens=10_000_000, output_tokens=1)
- assert guard.check(unknown)[0] is True # unpriceable ≠ blocked
-
-
-def test_cheapest_affordable_picks_a_tier():
- pick = cheapest_affordable(
- ["gemini:gemini-3.1-pro-preview", "gemini:gemini-3.7-flash", "gemini:gemini-3.1-flash-lite"],
- input_tokens=50_000, output_tokens=2_000)
- assert pick is not None and pick[0] == "gemini:gemini-3.1-flash-lite"
- assert cheapest_affordable(["gemini:gemini-3.1-pro-preview"], input_tokens=10_000_000,
- output_tokens=0, budget_usd=0.001) is None
-
-
# ------------------------------------------------------------------ caching
def test_order_blocks_puts_the_stable_prefix_first():
blocks = [Block("round", "round 3 of 4", order=1),
diff --git a/harness/tests/flywheel_cli/test_quality_gallery.py b/harness/tests/flywheel_cli/test_quality_gallery.py
index 291ea64f..f2efe3b8 100644
--- a/harness/tests/flywheel_cli/test_quality_gallery.py
+++ b/harness/tests/flywheel_cli/test_quality_gallery.py
@@ -1,4 +1,4 @@
-"""Quality tiers, (code, prompt) dedupe, richer meta.json, captions side-car, gallery, trajectory repair mining."""
+"""Quality tiers, (code, prompt) dedupe, richer meta.json, captions side-car, gallery."""
from __future__ import annotations
@@ -8,10 +8,8 @@ from pathlib import Path
from codeverse.contracts.common import Language
from codeverse.flywheel.captions import caption_sample
from codeverse.flywheel.export import export_samples, load_captions
-from codeverse.flywheel.pairs import build_pairs
from codeverse.flywheel.quality import find_duplicates, mark_duplicates, prompt_hash, quality_tier
from codeverse.flywheel.record import load_record
-from codeverse.flywheel.trajectories import mine_run, parse_trajectory_name
from codeverse.gallery import (
GalleryIndex,
RootSection,
@@ -20,7 +18,6 @@ from codeverse.gallery import (
build_static,
render_static,
)
-from codeverse.workspace import Workspace
from tests.flywheel_cli.conftest import make_fake_run
from tests.flywheel_cli.test_captions import GOOD, FakeModel
@@ -172,75 +169,3 @@ def test_gallery_from_runs(runs_dir: Path, tmp_path: Path):
RunEntry(battery="t", slug="y", path="", state="broken", error="boom <b>")])
html = render_static(GalleryIndex(sections=[section]), title="t")
assert "no render" in html and "boom &lt;b&gt;" in html
-
-
-# --------------------------------------------------------------------------- trajectory repair pairs
-
-
-def _write_transcript(ws: Workspace, label: str, rnd: int, events: list[dict]) -> Path:
- d = ws.root / "trajectories" / f"{label}_r{rnd:02d}"
- d.mkdir(parents=True, exist_ok=True)
- (d / "transcript.jsonl").write_text("\n".join(json.dumps(e) for e in events) + "\n")
- return d
-
-
-def _call(turn: int, t: float, cid: str, name: str, **args) -> dict:
- return {"t": t, "kind": "assistant", "turn": turn, "text": "", "tool_calls": [{"id": cid, "name": name, "arguments": args}]}
-
-
-def _result(turn: int, t: float, cid: str, name: str, ok: bool, content: str) -> dict:
- return {"t": t, "kind": "tool_result", "turn": turn, "name": name, "call_id": cid, "ok": ok, "content": content}
-
-
-def test_trajectory_repair_mining(fake_run, tmp_path: Path):
- ws, rec = fake_run
- assert parse_trajectory_name("zone_koi_pond_r01") == ("zone_koi_pond", 1)
- # the base state = the `pre:baseline` snapshot the harness takes before the session
- (ws.src / "model.py").write_text("import bpy\nA = 1\n")
- ws.commit("pre:baseline")
- import time
-
- t0 = time.time() + 5 # transcript starts after the snapshot
- ev = [
- {"t": t0, "kind": "system", "text": "rules"},
- _call(0, t0 + 1, "c1", "edit_file", path="src/model.py", old="A = 1", new="A = 2)"),
- _result(0, t0 + 1, "c1", "edit_file", True, "edited"),
- _call(1, t0 + 2, "c2", "build"), _result(1, t0 + 2, "c2", "build", False, "build failed: SyntaxError"),
- _call(2, t0 + 3, "c3", "write_file", path="src/model.py", content="import bpy\nA = 3\nB = 4\n"),
- _result(2, t0 + 3, "c3", "write_file", True, "overwrote"),
- _call(3, t0 + 4, "c4", "build"), _result(3, t0 + 4, "c4", "build", False, "build failed: NameError"),
- _call(4, t0 + 5, "c5", "edit_file", path="src/model.py", old="B = 4", new="B = A"),
- _result(4, t0 + 5, "c5", "edit_file", True, "edited"),
- _call(5, t0 + 6, "c6", "build"), _result(5, t0 + 6, "c6", "build", True, "BUILD OK"),
- _call(6, t0 + 7, "c7", "build"), _result(6, t0 + 7, "c7", "build", True, "BUILD OK"), # no new pair
- ]
- _write_transcript(ws, "baseline", 0, ev)
- pairs = mine_run(ws)
- assert len(pairs) == 1
- p = pairs[0]
- assert p.trajectory == "baseline_r00" and p.stage == "baseline" and p.round_index == 0
- assert p.broken_turn == 3 and p.fixed_turn == 5 and p.n_failures == 2 and "NameError" in p.error
- assert p.rejected["src/model.py"] == "import bpy\nA = 3\nB = 4\n" and p.chosen["src/model.py"] == "import bpy\nA = 3\nB = A\n"
- assert "src/parts/leg.py" in p.chosen and p.changed_files == ["src/model.py"] # untouched files ride along
- # a CLI-backend transcript (raw lines) yields nothing
- _write_transcript(ws, "refine", 0, [{"t": t0, "kind": "invoke", "argv": ["gemini"]}, {"t": t0, "kind": "line", "text": "x"}])
- assert len(mine_run(ws)) == 1
- out = tmp_path / "pairs.jsonl"
- n = build_pairs(ws.root.parent, out)
- kinds = [(json.loads(line)["kind"], json.loads(line).get("source", "")) for line in out.read_text().splitlines()]
- assert n == 2 and ("repair", "trajectory") in kinds and ("preference", "") in kinds
- assert build_pairs(ws.root.parent, tmp_path / "p2.jsonl", trajectories=False) == 1
-
-
-def test_trajectory_inexact_base_is_skipped(fake_run):
- """An edit on a file we never saw in full (no pre: commit, no write_file) must not produce a pair."""
- ws, rec = fake_run
- t0 = 1.0
- ev = [
- _call(0, t0, "c1", "edit_file", path="src/model.py", old="zzz", new="y"), _result(0, t0, "c1", "edit_file", True, "ok"),
- _call(1, t0, "c2", "build"), _result(1, t0, "c2", "build", False, "fail"),
- _call(2, t0, "c3", "edit_file", path="src/model.py", old="y", new="w"), _result(2, t0, "c3", "edit_file", True, "ok"),
- _call(3, t0, "c4", "build"), _result(3, t0, "c4", "build", True, "ok"),
- ]
- _write_transcript(ws, "baseline", 0, ev)
- assert mine_run(ws) == []
diff --git a/harness/tests/judges/test_metrics.py b/harness/tests/judges/test_metrics.py
deleted file mode 100644
index cf0dc47a..00000000
--- a/harness/tests/judges/test_metrics.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import pytest
-
-from codeverse.contracts.judgment import Judgment
-from codeverse.judges.metrics import best_index, judge_agreement, plateau
-
-
-def _j(o, **sc):
- return Judgment(rubric="r", scores=sc or {"a": o}, overall=o, passed=o > 0.7)
-
-
-def test_agreement():
- a = judge_agreement([_j(0.7, a=0.6, b=0.8), _j(0.9, a=0.8, b=1.0)])
- assert a.mean == 0.8 and a.std == pytest.approx(0.1) and a.per_criterion_std == {"a": 0.1, "b": 0.1} and a.n == 2
- assert judge_agreement([_j(0.5)]).std == 0.0
- with pytest.raises(ValueError):
- judge_agreement([])
-
-
-def test_plateau():
- assert plateau([0.5, 0.6]) is False # too few
- assert plateau([0.5, 0.6, 0.61, 0.6]) is True
- assert plateau([0.5, 0.6, 0.61, 0.7]) is False
- assert plateau([0.5, 0.52, 0.53], window=2, min_delta=0.02) is False # 0.53-0.5 = 0.03 ≥ 0.02
- assert plateau([0.5, 0.51, 0.515], window=2, min_delta=0.02) is True
-
-
-def test_best_index():
- assert best_index([(0.5, 0), (0.7, 2), (0.7, 1), (0.6, 0)]) == 2
- assert best_index([(0.7, 1), (0.7, 1)]) == 1 # later wins ties
- assert best_index([(0.9, 3), (0.8, 0)]) == 0
- with pytest.raises(ValueError):
- best_index([])
diff --git a/harness/tests/models/test_health.py b/harness/tests/models/test_health.py
index 0dcb3bf0..84daccb6 100644
--- a/harness/tests/models/test_health.py
+++ b/harness/tests/models/test_health.py
@@ -102,19 +102,6 @@ def test_sibling_detection_matches_arguments_not_the_repo_path():
assert not _is_harness_argv(["python", "-c", "print(1)", "/home/u/.local/bin/3dcv"])
-def test_sibling_count_never_raises(monkeypatch):
- """A wrong number here must never block a run, so /proc trouble returns 0."""
- from pathlib import Path
-
- import codeverse.models.health as health
-
- def boom(_self):
- raise OSError("no /proc on this platform")
-
- monkeypatch.setattr(Path, "iterdir", boom)
- assert health.sibling_processes() == 0
-
-
def test_probe_model_treats_a_503_as_final(monkeypatch):
""" "No retries" must include the storm branch. rotate_with_retries' capacity-storm
branch does NOT consume max_attempts, so max_attempts=1 alone still retried a 503
diff --git a/harness/tests/orchestrator_tracks/test_best_index.py b/harness/tests/orchestrator_tracks/test_best_index.py
new file mode 100644
index 00000000..c9077957
--- /dev/null
+++ b/harness/tests/orchestrator_tracks/test_best_index.py
@@ -0,0 +1,16 @@
+"""``orchestrator.rounds.best_index`` — the ranking rule BestSelector delegates to
+(moved here from tests/judges/test_metrics.py when judges/metrics.py was deleted)."""
+
+from __future__ import annotations
+
+import pytest
+
+from codeverse.orchestrator.rounds import best_index
+
+
+def test_best_index():
+ assert best_index([(0.5, 0), (0.7, 2), (0.7, 1), (0.6, 0)]) == 2
+ assert best_index([(0.7, 1), (0.7, 1)]) == 1 # later wins ties
+ assert best_index([(0.9, 3), (0.8, 0)]) == 0
+ with pytest.raises(ValueError):
+ best_index([])
diff --git a/harness/tests/orchestrator_tracks/test_plan_features.py b/harness/tests/orchestrator_tracks/test_plan_features.py
index 64bca37a..bdccd73e 100644
--- a/harness/tests/orchestrator_tracks/test_plan_features.py
+++ b/harness/tests/orchestrator_tracks/test_plan_features.py
@@ -1,5 +1,5 @@
-"""``tracks/plan_features.py``: the plan-loop switch registry, and the guard that keeps
-it honest about which switches any code actually reads (CQ-5)."""
+"""``tracks/plan_features.py``: the switch registry that keeps A/B rigs honest about
+which env switches any code actually reads (CQ-5)."""
from __future__ import annotations
@@ -9,44 +9,6 @@ import pytest
from codeverse.tracks import plan_features as F
-
-def test_empty_means_nothing_on(monkeypatch):
- monkeypatch.delenv(F.PLAN_FEATURES_ENV, raising=False)
- assert F.plan_features() == frozenset()
- assert not F.plan_feature_on(F.FIT)
- monkeypatch.setenv(F.PLAN_FEATURES_ENV, " ")
- assert F.plan_features() == frozenset()
-
-
-def test_parse_names_case_and_spaces():
- assert F.parse_features("fit, Contacts ,") == {F.FIT, F.CONTACTS}
-
-
-def test_all_and_negation():
- assert F.parse_features("all") == frozenset(F.KNOWN_FEATURES)
- assert F.parse_features("all,-graph") == frozenset(F.KNOWN_FEATURES) - {F.GRAPH}
- # order matters: a later positive re-enables
- assert F.GRAPH in F.parse_features("-graph,graph")
-
-
-def test_unknown_names_are_dropped_not_fatal(caplog):
- with caplog.at_level("WARNING"):
- assert F.parse_features("fit,typo") == {F.FIT}
- assert any("typo" in r.message for r in caplog.records)
-
-
-def test_reads_env_at_call_time(monkeypatch):
- monkeypatch.setenv(F.PLAN_FEATURES_ENV, "fit")
- assert F.plan_feature_on(F.FIT) and not F.plan_feature_on(F.CONTACTS)
- monkeypatch.setenv(F.PLAN_FEATURES_ENV, "contacts")
- assert F.plan_feature_on(F.CONTACTS) and not F.plan_feature_on(F.FIT)
-
-
-def test_asking_about_an_undefined_feature_is_a_bug():
- with pytest.raises(ValueError):
- F.plan_feature_on("not_a_feature")
-
-
# --------------------------------------------------------------------------- CQ-5
HARNESS = Path(__file__).resolve().parents[2]