Skip to content

Official Codebase of 3DCodeVerse

Python

cost/tracks/prompts: three subsystems with no production caller

gaoypeng458ac709fee3
diff --git a/harness/codeverse/cost/__init__.py b/harness/codeverse/cost/__init__.py
index e66894dd..a22a5c6b 100644
--- a/harness/codeverse/cost/__init__.py
+++ b/harness/codeverse/cost/__init__.py
@@ -1,112 +1,32 @@
-"""Cost accounting: a typed ledger of every model call, a price-provenance layer
-on top of ``models/pricing.py``, an audit over recorded runs, a pre-send cost
-guard, and the prompt-prefix measurement helpers (``Block`` / ``order_blocks`` /
-``prefix_report`` — measurement only: reordering the generation prompts for the
-cache was measured and reverted, see ``docs/COST.md`` §13).
+"""Cost accounting: a typed ledger of every model call, a price-provenance layer on
+top of ``models/pricing.py``, an audit over recorded runs, and a pre-send cost guard.
-Public API::
-
- from codeverse.cost import record_call, load_ledger, summarise
- from codeverse.cost import audit_runs, markdown # the audit + its report
- from codeverse.cost import estimate_call # estimate before you send
- from codeverse.cost import Block, order_blocks # cache-friendly prompts
- from codeverse.cost import default_route, pro_break_even # model routing
+This package re-exports only the names other packages actually import through it.
+Everything else lives in — and is imported from — its own module: ``cost.report``
+(console/markdown), ``cost.profiles`` (get_profile/PROFILES), ``cost.routing``
+(ROUTES/Route), ``cost.instrument`` (metered_chat_model/MeteredAgent),
+``cost.types`` (CallCost/CostBucket/Summary), ``cost.reconstruct`` (RunLedger).
+Keeping the shim narrow also keeps report/profiles/routing out of the eager import
+graph of ``import codeverse.cost``.
Nothing here imports tracks/ or orchestrator/, so any layer may use it.
"""
from __future__ import annotations
-from codeverse.cost.audit import Audit, WasteItem, audit_runs
-from codeverse.cost.caching import (
- Block,
- PrefixReport,
- cache_efficiency,
- order_blocks,
- prefix_report,
- prefix_signature,
- render_blocks,
-)
-from codeverse.cost.context import CallContext, bind_run, call_context
-from codeverse.cost.guard import CostEstimate, estimate_call, text_tokens
-from codeverse.cost.instrument import (
- MeteredAgent,
- MeteredChatModel,
- metered_agent,
- metered_chat_model,
- per_call_metering,
- run_ledger,
-)
-from codeverse.cost.ledger import (
- CostLedger,
- ledger_path,
- load_ledger,
- open_run_ledger,
- price_call,
- record_call,
- set_default_ledger,
- summarise,
-)
-from codeverse.cost.profiles import PROFILES, Profile, get_profile
-from codeverse.cost.reconstruct import RunLedger, find_runs, reconstruct, reconstruct_run
-from codeverse.cost.report import console, markdown
-from codeverse.cost.routing import (
- ROUTES,
- Route,
- default_route,
- pro_break_even,
- samples_for_precision,
-)
-from codeverse.cost.types import CallCost, CostBucket, Role, Stage, Summary
+from codeverse.cost.audit import audit_runs
+from codeverse.cost.context import call_context
+from codeverse.cost.guard import estimate_call, text_tokens
+from codeverse.cost.instrument import per_call_metering, run_ledger
+from codeverse.cost.ledger import CostLedger, load_ledger, open_run_ledger, record_call, summarise
+from codeverse.cost.reconstruct import find_runs, reconstruct
+from codeverse.cost.report import markdown
+from codeverse.cost.routing import default_route, pro_break_even
+from codeverse.cost.types import Role, Stage
__all__ = [
- "Audit",
- "Block",
- "CallContext",
- "CallCost",
- "CostBucket",
- "CostEstimate",
- "CostLedger",
- "MeteredAgent",
- "MeteredChatModel",
- "PROFILES",
- "PrefixReport",
- "Profile",
- "ROUTES",
- "Role",
- "Route",
- "RunLedger",
- "Stage",
- "Summary",
- "WasteItem",
- "audit_runs",
- "bind_run",
- "cache_efficiency",
- "call_context",
- "console",
- "default_route",
- "estimate_call",
- "find_runs",
- "get_profile",
- "ledger_path",
- "load_ledger",
- "markdown",
- "metered_agent",
- "metered_chat_model",
- "open_run_ledger",
- "order_blocks",
- "per_call_metering",
- "prefix_report",
- "prefix_signature",
- "price_call",
- "pro_break_even",
- "reconstruct",
- "reconstruct_run",
- "record_call",
- "render_blocks",
- "run_ledger",
- "samples_for_precision",
- "set_default_ledger",
- "summarise",
- "text_tokens",
+ "CostLedger", "Role", "Stage", "audit_runs", "call_context", "default_route",
+ "estimate_call", "find_runs", "load_ledger", "markdown", "open_run_ledger",
+ "per_call_metering", "pro_break_even", "reconstruct", "record_call", "run_ledger",
+ "summarise", "text_tokens",
]
diff --git a/harness/codeverse/cost/audit.py b/harness/codeverse/cost/audit.py
index b2cc47b8..d200c1cd 100644
--- a/harness/codeverse/cost/audit.py
+++ b/harness/codeverse/cost/audit.py
@@ -43,9 +43,6 @@ class WasteItem:
detail: str = ""
round: int | None = None
- def as_row(self) -> tuple[str, str, str, float]:
- return (self.kind, self.run, self.detail, self.usd)
-
@dataclass
class RoundCost:
@@ -74,7 +71,6 @@ class Audit:
summary: Summary = field(default_factory=Summary)
rounds: list[RoundCost] = field(default_factory=list)
waste: list[WasteItem] = field(default_factory=list)
- recheck: bool = False
# ---------------------------------------------------------------- totals
@property
@@ -200,7 +196,7 @@ def audit_runs(paths: Iterable[str | Path], *, recheck: bool = False) -> Audit:
dirs: list[Path] = []
for p in paths:
dirs += find_runs(p)
- audit = Audit(recheck=recheck)
+ audit = Audit()
for d in dirs:
led = reconstruct(d, recheck=recheck)
if not led.rows:
diff --git a/harness/codeverse/cost/caching.py b/harness/codeverse/cost/caching.py
index 19deaa5d..d80d21ae 100644
--- a/harness/codeverse/cost/caching.py
+++ b/harness/codeverse/cost/caching.py
@@ -1,154 +1,23 @@
-"""Cache-friendly prompt assembly + a measurement for whether it worked.
+"""Per-session cache behaviour, read back out of the ledger.
Gemini's implicit cache (and Anthropic's explicit one) only ever matches a
-**prefix**: the shared part must be byte-identical *from token 0*. One
-variable line near the top — a timestamp, a round number, a shuffled view list —
-moves everything after it out of the cache. Cache reads cost 10% of input, so
-the whole difference between a $0.75/M prompt and a $0.075/M prompt is where the
-volatile blocks sit.
-
-Usage::
-
- from codeverse.cost import order_blocks, Block
- blocks = [Block("system rules", SYSTEM, stable=True),
- Block("rubric", rubric_md, stable=True),
- Block("round", f"round {i}", stable=False)]
- prompt = render_blocks(order_blocks(blocks))
-
-:func:`prefix_report` measures a real sequence of prompts: how many tokens they
-share, and how many tokens each call pays full price for because of divergence.
-
-**Measured, and NOT applied to the generation prompts (2026-08-23).** Wave 2
-reordered ``generate_static`` / ``generate_articulated`` / ``refine_object``
-stable-prefix-first, by hoisting a shared head into every one of them. Measured
-on the real templates (``prefix_report`` over one baseline + three refines):
-
-=============================== ============================== ===============
- per-prompt tokens mean/call
-=============================== ============================== ===============
-task-first (shipped, restored) 5,072 / 1,325 / 1,325 / 1,325 2,261
-stable-head-first (reverted) 5,110 / 5,211 / 5,211 / 5,211 5,186
-=============================== ============================== ===============
-
-The head was **duplicated** into every prompt rather than shared with anything
-already being sent, so the run-level change is +2,925 tokens on the average call
-(+3,886 on every refine) — and the prompt is re-sent on every turn of the
-session, so it is that figure times the turn count. What it bought was a 4,970
-token shared prefix, below the ~12k floor §4 of ``docs/COST.md`` measured for
-Gemini's implicit cache, so there is no cache read to set against it. The one
-prefix that is genuinely shared without duplication — the api-agent's system
-prompt (``AGENTS.md``, 4,879 tokens) plus its tool declarations (1,691) — already
-sits first in every request and totals **6,570 tokens**, also below the floor.
-
-So: keep these helpers for measuring and for any future prompt family that is
-big enough, but do not reorder a prompt for the cache without measuring the
-prefix against the floor first.
+**prefix**: the shared part must be byte-identical from token 0, and cache reads
+cost 10 % of input. This module answers the after-the-fact question — did a
+session's later turns actually read the cache, and what did that save?
+
+It used to also carry a prompt-ASSEMBLY half (``Block``/``order_blocks``/
+``prefix_report``/``cache_efficiency``) for building cache-friendly prompts. That
+experiment was measured and reverted (docs/COST.md §13: every prompt family here
+sits below the provider's minimum cacheable prefix, so reordering bought nothing),
+and the code sat with no production caller until it was deleted on 2026-08-28.
+
+``3dcv cost cache <slug>`` is the reader.
"""
from __future__ import annotations
-import hashlib
from collections.abc import Sequence
-from dataclasses import dataclass, field
-
-from codeverse.cost.guard import text_tokens
-
-
-@dataclass(frozen=True)
-class Block:
- """One labelled chunk of a prompt. ``stable`` blocks are identical across
- every call in a family (system prompt, contract, cookbook, rubric, defect
- checklist); everything else is volatile."""
-
- name: str
- text: str
- stable: bool = False
- order: int = 0 # tie-break inside the stable / volatile group
-
- @property
- def tokens(self) -> int:
- return text_tokens(self.text)
-
-
-def order_blocks(blocks: Sequence[Block]) -> list[Block]:
- """Stable blocks first (input order preserved), volatile blocks after.
-
- This is the whole optimisation: the longest possible identical prefix, then
- everything that changes per call."""
- stable = [b for b in blocks if b.stable]
- volatile = [b for b in blocks if not b.stable]
- stable.sort(key=lambda b: b.order)
- volatile.sort(key=lambda b: b.order)
- return stable + volatile
-
-
-def render_blocks(blocks: Sequence[Block], *, sep: str = "\n\n") -> str:
- return sep.join(b.text for b in blocks if b.text)
-
-
-def prefix_signature(blocks: Sequence[Block]) -> str:
- """Hash of the stable prefix — log it per call; if it ever changes between
- two calls that should share a cache, the cache was silently lost."""
- h = hashlib.sha256()
- for b in blocks:
- if not b.stable:
- break
- h.update(b.text.encode("utf-8", "replace"))
- return h.hexdigest()[:12]
-
-
-def common_prefix_chars(a: str, b: str) -> int:
- n = min(len(a), len(b))
- i = 0
- while i < n and a[i] == b[i]:
- i += 1
- return i
-
-
-@dataclass
-class PrefixReport:
- """What a family of prompts shares."""
-
- n_prompts: int = 0
- prefix_chars: int = 0
- prefix_tokens: int = 0
- mean_tokens: int = 0
- divergent_tokens: int = 0 # mean tokens after the shared prefix
- per_prompt: list[int] = field(default_factory=list)
-
- @property
- def shared_fraction(self) -> float:
- return self.prefix_tokens / self.mean_tokens if self.mean_tokens else 0.0
-
- def savings_usd(self, price_input: float, price_cached: float, *, calls: int | None = None) -> float:
- """USD saved per ``calls`` calls by the shared prefix being a cache hit
- instead of fresh input (the first call still pays full price)."""
- n = (calls if calls is not None else self.n_prompts) - 1
- return max(0, n) * self.prefix_tokens * max(0.0, price_input - price_cached) / 1e6
-
-
-def prefix_report(prompts: Sequence[str]) -> PrefixReport:
- """Measure the common prefix of a real prompt sequence."""
- if not prompts:
- return PrefixReport()
- shared = prompts[0]
- for p in prompts[1:]:
- shared = shared[: common_prefix_chars(shared, p)]
- sizes = [text_tokens(p) for p in prompts]
- prefix_tokens = text_tokens(shared)
- return PrefixReport(
- n_prompts=len(prompts), prefix_chars=len(shared), prefix_tokens=prefix_tokens,
- mean_tokens=int(sum(sizes) / len(sizes)), per_prompt=sizes,
- divergent_tokens=int(sum(sizes) / len(sizes)) - prefix_tokens,
- )
-
-
-def cache_efficiency(usages: Sequence[object]) -> tuple[int, int, float]:
- """``(cached, input, hit rate)`` over a sequence of ``Usage``-like objects —
- the after-the-fact check that the ordering actually paid off."""
- cached = sum(int(getattr(u, "cached_tokens", 0)) for u in usages)
- total = sum(int(getattr(u, "input_tokens", 0)) for u in usages)
- return cached, total, (cached / total if total else 0.0)
+from dataclasses import dataclass
# --------------------------------------------------------------------------- ledger view
@@ -199,17 +68,16 @@ def session_key(label: str) -> str:
return ":".join(parts)
-def session_cache(rows: Sequence[object], *, key: str = "label") -> list[SessionCache]:
+def session_cache(rows: Sequence[object]) -> list[SessionCache]:
"""Per-session cache behaviour from ledger rows (``codeverse.cost.CallCost``).
Sessions are ``(round, session_key(<key>))`` groups in ledger order, so the first row of a
group is that session's cold turn — the one the audit shows pays full price
(docs/COST.md §4). Use it to check whether a prompt-ordering change actually
moved ``cached_tokens``, not just the prompt."""
- out: dict[tuple[object, str], SessionCache] = {}
- order: list[tuple[object, str]] = []
+ out: dict[tuple[object, str], SessionCache] = {} # insertion order IS ledger order
for row in rows:
- k = (getattr(row, "round", None), session_key(str(getattr(row, key, "") or "")))
+ k = (getattr(row, "round", None), session_key(str(getattr(row, "label", "") or "")))
s = out.get(k)
if s is None:
s = out[k] = SessionCache(key=f"r{k[0] if k[0] is not None else '-'}:{k[1]}",
@@ -217,9 +85,8 @@ def session_cache(rows: Sequence[object], *, key: str = "label") -> list[Session
first_cached=int(getattr(row, "cached_tokens", 0)),
price_input=float(getattr(row, "price_input", 0.0)),
price_cached=float(getattr(row, "price_cached", 0.0)))
- order.append(k)
s.n_calls += 1
s.input_tokens += int(getattr(row, "input_tokens", 0))
s.cached_tokens += min(int(getattr(row, "cached_tokens", 0)), int(getattr(row, "input_tokens", 0)))
s.cost_usd += float(getattr(row, "cost_usd", 0.0))
- return [out[k] for k in order]
+ return list(out.values())
diff --git a/harness/codeverse/cost/guard.py b/harness/codeverse/cost/guard.py
index daf78477..7cd0f435 100644
--- a/harness/codeverse/cost/guard.py
+++ b/harness/codeverse/cost/guard.py
@@ -54,10 +54,6 @@ class CostEstimate:
price_source: str = "unknown"
approximate: bool = True
- @property
- def known(self) -> bool:
- return self.price_source != "unknown"
-
def line(self) -> str:
flag = " (approx)" if self.approximate else ""
return (f"{self.model_id}: ~{self.input_tokens:,} in ({self.cached_tokens:,} cached) "
diff --git a/harness/codeverse/cost/ledger.py b/harness/codeverse/cost/ledger.py
index 08c291a1..ddf16ba5 100644
--- a/harness/codeverse/cost/ledger.py
+++ b/harness/codeverse/cost/ledger.py
@@ -25,7 +25,7 @@ from pathlib import Path
from typing import Any
from codeverse.contracts.common import Usage
-from codeverse.cost.context import CallContext, attribute
+from codeverse.cost.context import CallContext, _stage, attribute
from codeverse.cost.types import (
CallCost,
CostBucket,
@@ -34,7 +34,6 @@ from codeverse.cost.types import (
Summary,
normalise_ids,
role_for_stage,
- stage_for_label,
)
from codeverse.models.pricing import (
cache_write_surcharge,
@@ -205,13 +204,10 @@ def open_run_ledger(workspace: str | Path) -> CostLedger:
return CostLedger(path)
-def _as_stage(value: Stage | str | None) -> Stage | None:
- if value is None or isinstance(value, Stage):
- return value
- try:
- return Stage(value)
- except ValueError:
- return stage_for_label(str(value))
+#: context._stage is the same function; ledger already imports from context. NOT true
+#: of the neighbouring _as_role — that one returns Role.OTHER where context._role returns
+#: None, and Role.OTHER is truthy, so it survives CallContext.resolved().
+_as_stage = _stage
def _as_role(value: Role | str | None) -> Role | None:
diff --git a/harness/codeverse/cost/reconstruct.py b/harness/codeverse/cost/reconstruct.py
index ba8ee82f..2e77b39f 100644
--- a/harness/codeverse/cost/reconstruct.py
+++ b/harness/codeverse/cost/reconstruct.py
@@ -81,7 +81,6 @@ class RunLedger:
track: str = ""
language: str = ""
generator: str = ""
- judge_model: str = ""
status: str = ""
stop_reason: str = ""
passed: bool = False
@@ -95,8 +94,6 @@ class RunLedger:
span_s: float = 0.0 # first to last event — includes time queued behind other runs
model_s: float = 0.0
budget_usd: float = 0.0
- parallel: bool = False
- resumed: bool = False
selected_candidate: str = "" # "c1" when the run ran best-of-N and c1 won
source: str = "reconstructed" # live (the run wrote its own ledger) | reconstructed
rows: list[CallCost] = field(default_factory=list)
@@ -272,7 +269,7 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
led = RunLedger(
run=run, path=root,
track=str(spec.get("track") or ""), language=str(spec.get("language") or ""),
- generator=str(backends.get("generator") or ""), judge_model=str(backends.get("judge") or ""),
+ generator=str(backends.get("generator") or ""),
status=str(record.get("status") or ""), stop_reason=str(extra.get("stop_reason") or ""),
baseline_score=record.get("baseline_score"), final_score=record.get("final_score"),
recorded_usd=float(total.cost_usd),
@@ -325,13 +322,13 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
# ------------------------------------------------------------------ events
first_t = last_t = 0.0
active_s = 0.0
- n_finishes = 0
for ev in _read_jsonl(root / "events.jsonl"):
name = str(ev.get("event") or "")
t = float(ev.get("t") or 0.0)
first_t = first_t or t
- if name in ("run.done", "run.failed"):
- n_finishes += 1
+ # folded in from a SECOND full pass over the same file, which added only this
+ if name == "stage.done" and str(ev.get("stage")) == "plan":
+ model_s += float(ev.get("duration_s") or 0.0)
if name in ("run.done", "run.failed", "stop", "budget.exceeded"):
last_t = max(last_t, t) # a post-hoc texture pass is not part of the run's wall clock
elif not last_t:
@@ -361,11 +358,6 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
role=role, ts=t, model=model)
row.price_source = "event-cost" # cost known, tokens are not recorded by the event
led.rows.append(row)
- for ev in _read_jsonl(root / "events.jsonl"):
- if str(ev.get("event")) == "stage.done" and str(ev.get("stage")) == "plan":
- model_s += float(ev.get("duration_s") or 0.0)
-
- led.resumed = n_finishes > 1
led.span_s = max(0.0, last_t - first_t)
elapsed_min = float(((extra.get("budget") or {}).get("elapsed_min")) or 0.0)
# events are emitted from run creation, so the span of a queued bench run counts
@@ -373,7 +365,6 @@ def reconstruct_run(run_dir: str | Path, *, recheck: bool = False) -> RunLedger:
# ones (the budget's own elapsed restarts on ``3dcv resume``).
led.wall_s = max(active_s, elapsed_min * 60.0) or led.span_s
led.model_s = model_s
- led.parallel = model_s > led.wall_s > 0
# ------------------------------------------------------------------ residual
if total.cost_usd:
@@ -407,7 +398,7 @@ def reconstruct_cell(cell_dir: str | Path, *, recheck: bool = False) -> RunLedge
cell = _read_json(root / "cell.json")
run = RunId(battery="", rel=f"{root.parent.name}/{root.name}").slug # <prompt>__<arm>
led = RunLedger(run=run, path=root, track="static_object", language="blender",
- generator=str(cell.get("arm") or root.name), judge_model=str(cell.get("judge") or ""),
+ generator=str(cell.get("arm") or root.name),
status=str(cell.get("status") or ""), passed=bool(cell.get("passed")),
final_score=cell.get("score"), n_rounds=1,
wall_s=float(cell.get("wall_s") or 0.0), model_s=float(cell.get("gen_seconds") or 0.0))
diff --git a/harness/codeverse/cost/types.py b/harness/codeverse/cost/types.py
index 30af53e5..84ebfb09 100644
--- a/harness/codeverse/cost/types.py
+++ b/harness/codeverse/cost/types.py
@@ -204,10 +204,6 @@ class CallCost(BaseModel):
#: and counts; a free one is ``source="attempt"`` and is filtered out
discarded: bool = False
- @property
- def uncached_tokens(self) -> int:
- return max(0, self.input_tokens - min(self.cached_tokens, self.input_tokens))
-
@property
def total_tokens(self) -> int:
return self.input_tokens + self.output_tokens + self.thoughts_tokens
@@ -250,10 +246,6 @@ class CostBucket(BaseModel):
if row.price_approximate or row.price_source in ("unknown", "provider-reported"):
self.approximate_usd += row.cost_usd
- @property
- def uncached_tokens(self) -> int:
- return max(0, self.input_tokens - self.cached_tokens)
-
@property
def cached_fraction(self) -> float:
return self.cached_tokens / self.input_tokens if self.input_tokens else 0.0
diff --git a/harness/codeverse/prompts/system/tools_usage.md b/harness/codeverse/prompts/system/tools_usage.md
deleted file mode 100644
index 8b7ecae7..00000000
--- a/harness/codeverse/prompts/system/tools_usage.md
+++ /dev/null
@@ -1,68 +0,0 @@
-# Using the 3dcv spatial tools (look before you leap)
-
-The tools are the harness's instruments. They are cheap, deterministic and honest —
-trust their numbers over your mental model. Observations come back as
-`{text, numbers, images[]}`; images are small labelled PNGs / contact sheets.
-
-## The standard loop
-
-```
-1. build → errors first (file:line + traceback tail) or key numbers
-2. render_sheet → 4 labelled views (front_right_34, back_left_34, front, top)
-3. check_connectivity → islands, floating parts, ground gap, per-part touch graph
-4. fix the worst finding → rebuild (repeat 1–3 until clean)
-5. isolate <part> → the one part you doubt, alone, 4 views + its bbox
-6. cross_section → axis=y at the cavity height: is the mug actually hollow?
-7. compare_silhouette → only when a reference image exists (IoU per view)
-8. joint_sweep / scene_probe / shader_probe → articulated / scene tracks
-9. check_contract → plan bbox vs measured, per part (last, before done)
-```
-Always `build` after an edit; never reason about stale renders.
-
-## Tool cards (arguments are validated; read the error if you get one)
-
-* `build()` — lint + execute + export + measure. On error: fix that file:line first.
- On success: overall bbox/extents, tri count, n_meshes, n_islands, part table.
-* `measure(parts=[...])` — bbox/extents/islands/volume per part; use it to check a
- dimension numerically instead of eyeballing a render.
-* `render_views(views=[...], mode="shaded"|"wire"|"normals"|"clay", isolate=[...],
- explode=0.0)` — named views from the preset list (front, right, back, left, top,
- front_right_34, back_left_34, low_front_left). `explode=0.3` separates parts to see
- what is where. `wire` shows density; `clay` removes colour so shape is judged alone.
-* `render_sheet()` — the quick 4-view contact sheet. One image, always look at it.
-* `isolate(part="SeatCushion")` — that part alone, framed; everything else hidden.
-* `cross_section(axis="y", at=0.05)` — slice at height `at` (meters, in the GLB Y-up
- frame); shows walls, cavities, interpenetration. Slice where the detail is.
-* `check_connectivity(gap_m=0.002)` — hard facts: islands, floating pieces (with the
- nearest neighbour and the gap in mm), ground gap. A positive gap = you must weld.
-* `check_contract(tol_m=0.01)` — plan bbox vs measured per part; lists the offenders
- with the translation / scale that would fix them.
-* `compare_silhouette(view="front")` — IoU of your render vs the reference image.
-* `joint_sweep(joint="DoorHinge")` — FK over [lower, mid, upper]; renders + collision
- pairs per pose; "child does not move" or "penetrates parent at q=1.2" is the finding.
-* `scene_probe()` — census (draw calls, tris, instanced meshes), cameras inside geometry,
- console errors, fps, black/blown frames.
-* `check_placement()` — (scene) every placed asset's gap to what is under it, burial depth,
- water, contacts and 3-D interpenetrations; "floating / sunken / unsupported /
- interpenetration" findings with the exact move to make. Run it after every `build`.
-* `shader_probe()` — compiles every ShaderMaterial / onBeforeCompile patch with the real
- renderer; reports GLSL errors with the author's line number.
-* `read_cookbook(section="Booleans")` — fetch one cookbook chapter by its heading.
-
-## Reading the outputs
-
-* `n_islands > n_parts` → something is split or floating; `check_connectivity` names it.
-* `ground_gap_m < 0` → buried; `> 0.002` → hovering. Translate the whole object.
-* `extents` far from plan → wrong unit (mm vs m) or a scale you forgot to apply.
-* `footprint_offset_m > 0.01` → not centred on the up-axis.
-* In renders: identical grey everywhere = materials missing; a part present in `top`
- but absent in `front` = it is inside something else; thin white lines at seams = gaps.
-
-## Image budget etiquette
-
-* One `render_sheet` per build cycle (not per edit). Use `isolate` / `cross_section`
- only for the part you are about to change. Prefer `measure` (numbers, no image) for
- dimension questions.
-* Do not request all 8 views + wire + normals at once; the sheet is enough to decide.
-* Re-render after a fix, and compare with the previous sheet before moving on.
-* Stop when the definition-of-done list is green, not when you run out of ideas.
diff --git a/harness/codeverse/tracks/common.py b/harness/codeverse/tracks/common.py
index b04676eb..093b261c 100644
--- a/harness/codeverse/tracks/common.py
+++ b/harness/codeverse/tracks/common.py
@@ -9,6 +9,7 @@ missing; tests subclass it with fakes. Nothing here touches a network.
from __future__ import annotations
import logging
+import os
from collections.abc import Sequence
from dataclasses import dataclass, field
from pathlib import Path
@@ -20,6 +21,7 @@ from codeverse.contracts.common import Language, Track
from codeverse.contracts.plan import CameraPlan, Plan
from codeverse.contracts.run import RunRecord
from codeverse.conventions import ViewPreset
+from codeverse.languages._docs import prompt_dir_for
from codeverse.orchestrator import BudgetGuard, RoundPolicy, RunState
from codeverse.proc import EventLog
from codeverse.prompts import load_text, prompt_hash
@@ -211,8 +213,8 @@ def load_prompt_or(rel: str, fallback: str) -> str:
def language_contract(ctx_language: Language, runtime: Any) -> str:
- """The language authoring contract: prompts/<lang>/contract.md → runtime.contract_doc() → minimal."""
- text = load_prompt_or(f"{ctx_language.value}/contract.md", "")
+ """The language authoring contract: prompts/<dir>/contract.md → runtime.contract_doc() → minimal."""
+ text = load_prompt_or(f"{prompt_dir_for(ctx_language)}/contract.md", "")
if text.strip():
return text
doc = ""
@@ -242,5 +244,12 @@ _MINIMAL_CONTRACT: dict[Language, str] = {
def cookbook_rel_for(language: Language) -> str:
- return f"{language.value}/cookbook.md"
+ """prompts/<dir>/cookbook.md. ``language.value`` is NOT always the directory:
+ urdf_blender's prompts live in prompts/urdf/, so this returned a path that does not
+ exist and the articulated agent was told "No cookbook is available in this session"
+ while its 24 063-character cookbook sat on disk. A/B'd behind CV3D_URDF_COOKBOOK
+ before being made unconditional, because it changes the prompt."""
+ if language is Language.URDF_BLENDER and os.environ.get("CV3D_URDF_COOKBOOK", "0") == "0":
+ return f"{language.value}/cookbook.md" # arm A: today's behaviour (misses)
+ return f"{prompt_dir_for(language)}/cookbook.md"
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index 00fab06c..928943c9 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -809,8 +809,6 @@ def generate(
settings: Any | None = None,
budget: Any | None = None,
events: Any | None = None,
- model_factory: Callable[[str], Any] | None = None,
- agent_factory: Callable[[str], Any] | None = None,
max_turns: int = 0,
wrapup_turns: int = DEFAULT_WRAPUP_TURNS,
) -> GenerationResult:
@@ -821,8 +819,9 @@ def generate(
"""
if is_single_shot(agent_id):
if model is None:
- factory = model_factory or _default_model_factory
- model = factory(single_shot_model_id(agent_id))
+ from codeverse.models import get_chat_model
+
+ model = get_chat_model(single_shot_model_id(agent_id))
return generate_files(
ws,
model=model,
@@ -832,8 +831,9 @@ def generate(
allowed_roots=tuple(r.rstrip("/") + "/" for r in task.write_roots),
)
if agent is None:
- factory = agent_factory or _default_agent_factory
- agent = factory(agent_id)
+ from codeverse.agents import get_coding_agent
+
+ agent = get_coding_agent(agent_id)
return run_agent_task(
ws,
agent=agent,
@@ -844,15 +844,3 @@ def generate(
max_turns=max_turns,
wrapup_turns=wrapup_turns,
)
-
-
-def _default_model_factory(model_id: str) -> Any:
- from codeverse.models import get_chat_model
-
- return get_chat_model(model_id)
-
-
-def _default_agent_factory(agent_id: str) -> Any:
- from codeverse.agents import get_coding_agent
-
- return get_coding_agent(agent_id)
\ No newline at end of file
diff --git a/harness/codeverse/tracks/graphics.py b/harness/codeverse/tracks/graphics.py
index 64c96544..122a17c5 100644
--- a/harness/codeverse/tracks/graphics.py
+++ b/harness/codeverse/tracks/graphics.py
@@ -39,8 +39,7 @@ from codeverse.spatial.cookbook_tool import Section, split_sections
from codeverse.tracks.common import RunContext
from codeverse.tracks.generation import SINGLE_SHOT_FORMAT, GenerationTask
from codeverse.tracks.lifecycle import BaseTrack, StageRunner
-from codeverse.tracks.planner import add_acceptance_item, build_system_prompt
-from codeverse.tracks.planner import plan as run_planner
+from codeverse.tracks.planner import add_acceptance_item
from codeverse.tracks.prompting import (
AGENT_OUTPUT_RULES,
acceptance_lines,
@@ -325,12 +324,6 @@ def plan_example() -> dict[str, Any]: # the graphics worked example (planner ho
}
-def build_plan_system_prompt(spec: Spec, *, runtime: Any | None = None) -> str:
- """The graphics plan system prompt = the shared builder with the graphics
- template + worked example (``plan_graphics.j2`` references no 3D frame)."""
- return build_system_prompt(spec, GraphicsPlan, runtime=runtime, template=PLAN_TEMPLATE, example=plan_example())
-
-
def ensure_graphics_acceptance(plan: GraphicsPlan, spec: Spec) -> GraphicsPlan:
"""Spec must_have / must_not → visual items; planned motion → a probe item (never 'ground contact')."""
items: list[AcceptanceItem] = list(plan.acceptance)
@@ -349,16 +342,6 @@ def graphics_event_stats(plan: GraphicsPlan) -> dict[str, Any]:
return {"n_passes": len(plan.passes), "n_acceptance": len(plan.acceptance)}
-def plan_graphics(spec: Spec, model_id: str, ws: Workspace, *, model: Any | None = None, events: Any | None = None,
- budget: Any | None = None, runtime: Any | None = None) -> GraphicsPlan:
- """Structured planner call → validated GraphicsPlan: the ONE planner loop
- (``tracks/planner.plan``) parameterised with the graphics hooks."""
- return run_planner(spec, model_id, GraphicsPlan, ws, model=model, events=events, budget=budget, runtime=runtime,
- template=PLAN_TEMPLATE, example=plan_example(), temperature=PLAN_TEMPERATURE,
- max_output_tokens=PLAN_MAX_OUTPUT_TOKENS,
- finalise=lambda p: ensure_graphics_acceptance(p, spec), event_stats=graphics_event_stats)
-
-
# ----------------------------------------------------------------------------- prompt context
def graphics_expected_files(ctx: RunContext) -> list[str]:
return list(EXPECTED_FILES.get(ctx.language, ["src/shader.frag"]))
diff --git a/harness/codeverse/tracks/plan_features.py b/harness/codeverse/tracks/plan_features.py
index 40964ab5..9c553a8a 100644
--- a/harness/codeverse/tracks/plan_features.py
+++ b/harness/codeverse/tracks/plan_features.py
@@ -80,6 +80,10 @@ LIVE_SWITCHES: dict[str, str] = {
# inter-chunk stall detection, and the IPv4-only transport. Read at call time
# by every gemini request, so a control arm can set either to 0.
"CV3D_STREAM": "codeverse/models/gemini.py",
+ #: urdf_blender resolved its cookbook path from language.value ("urdf_blender")
+ #: while its prompts live in prompts/urdf/, so the articulated agent got NO
+ #: cookbook at all. On = the fixed mapping; A/B before making it unconditional.
+ "CV3D_URDF_COOKBOOK": "codeverse/tracks/common.py",
"CV3D_IPV4": "codeverse/models/gemini.py",
}
diff --git a/harness/tests/cost/test_guard_caching_routing.py b/harness/tests/cost/test_guard_caching_routing.py
index 4782eceb..0d0010e8 100644
--- a/harness/tests/cost/test_guard_caching_routing.py
+++ b/harness/tests/cost/test_guard_caching_routing.py
@@ -5,15 +5,8 @@ from __future__ import annotations
import pytest
-from codeverse.contracts.common import Usage
from codeverse.cost import (
- Block,
- cache_efficiency,
estimate_call,
- order_blocks,
- prefix_report,
- prefix_signature,
- render_blocks,
text_tokens,
)
from codeverse.cost.guard import image_tokens
@@ -42,43 +35,6 @@ def test_estimate_of_an_image_model_uses_the_per_image_price():
# ------------------------------------------------------------------ caching
-def test_order_blocks_puts_the_stable_prefix_first():
- blocks = [Block("round", "round 3 of 4", order=1),
- Block("system", "SYSTEM RULES", stable=True, order=0),
- Block("rubric", "RUBRIC", stable=True, order=1),
- Block("renders", "<images>", order=0)]
- ordered = order_blocks(blocks)
- assert [b.name for b in ordered] == ["system", "rubric", "renders", "round"]
- assert render_blocks(ordered).startswith("SYSTEM RULES\n\nRUBRIC")
-
-
-def test_prefix_signature_only_covers_the_stable_head():
- a = order_blocks([Block("s", "SYS", stable=True), Block("v", "round 1")])
- b = order_blocks([Block("s", "SYS", stable=True), Block("v", "round 2")])
- assert prefix_signature(a) == prefix_signature(b)
- c = order_blocks([Block("s", "SYS!", stable=True), Block("v", "round 1")])
- assert prefix_signature(c) != prefix_signature(a)
-
-
-def test_prefix_report_measures_the_shared_head_and_its_value():
- stable = "S" * 40_000 # 10k tokens
- prompts = [stable + f"\nround {i}" for i in range(4)]
- rep = prefix_report(prompts)
- assert rep.prefix_tokens == pytest.approx(10_000, rel=0.01)
- assert rep.shared_fraction > 0.99
- # 3 later calls read 10k tokens from cache instead of paying full input price
- assert rep.savings_usd(0.75, 0.075) == pytest.approx(3 * 10_000 * 0.675 / 1e6, rel=0.01)
- volatile_first = [f"round {i}\n" + stable for i in range(4)]
- assert prefix_report(volatile_first).prefix_tokens < 10 # the cache is gone
-
-
-def test_cache_efficiency_reads_usages():
- us = [Usage(input_tokens=100, cached_tokens=0), Usage(input_tokens=100, cached_tokens=80)]
- cached, total, rate = cache_efficiency(us)
- assert (cached, total) == (80, 200) and rate == 0.4
-
-
-# ------------------------------------------------------------------ routing
def test_routing_defaults_exist_for_every_role():
for role in (Role.PLANNER, Role.GENERATOR, Role.JUDGE, Role.IMAGE, Role.CAPTIONER):
route = default_route(role)
diff --git a/harness/tests/cost/test_live.py b/harness/tests/cost/test_live.py
index e242d6d7..51442ab3 100644
--- a/harness/tests/cost/test_live.py
+++ b/harness/tests/cost/test_live.py
@@ -10,8 +10,7 @@ from pathlib import Path
import pytest
from codeverse.contracts.chat import ChatMessage, ChatRequest
-from codeverse.cost import Block, cache_efficiency, order_blocks, render_blocks
-from codeverse.cost.guard import estimate_call, text_tokens
+from codeverse.cost.guard import estimate_call
pytestmark = pytest.mark.live
@@ -34,37 +33,6 @@ def _send(model, text: str):
temperature=0.0, thinking="low", label="cost-test")).usage
[email protected]("arm", ["stable_first", "volatile_first"])
-def test_cache_friendly_ordering_is_measurably_cheaper(arm: str):
- """A ≥12k-token stable prefix must cache when it comes FIRST and must not
- when the volatile block precedes it (measured 2026-08-23: 69% vs 0%)."""
- import uuid
-
- from codeverse.config import get_settings
- from codeverse.models.gemini import GeminiModel
-
- keys = list(get_settings().gemini_api_keys)
- if not keys:
- pytest.skip("no gemini keys")
- # a corpus no call has seen before: the two arms must not inherit each
- # other's cache (the implicit cache is shared across the whole key pool).
- stable = f"[corpus {uuid.uuid4().hex}]\n" + _stable_text() * 2
- assert text_tokens(stable) > 12_000, "the experiment needs a prefix above the cache floor"
- model = GeminiModel("gemini-3.7-flash", keys=[keys[0 if arm == "stable_first" else 1]])
- usages = []
- for i in range(4):
- blocks = [Block("stable", stable, stable=(arm == "stable_first")),
- Block("volatile", f"ROUND {i}\nReply with ONE JSON object {{\"n\": {i}}}.")]
- text = render_blocks(order_blocks(blocks)) if arm == "stable_first" \
- else f"ROUND {i}\nReply with ONE JSON object {{\"n\": {i}}}.\n\n{stable}"
- usages.append(_send(model, text))
- cached, total, rate = cache_efficiency(usages[1:]) # the first call warms the cache
- if arm == "stable_first":
- assert rate > 0.3, f"stable-first should cache; got {rate:.2f} (implicit cache is best-effort)"
- else:
- assert rate < 0.2, f"volatile-first must not cache; got {rate:.2f}"
-
-
def test_estimate_is_within_15_percent_of_the_bill():
"""The pre-send estimate has to be good enough to route on."""
from codeverse.models import get_chat_model
diff --git a/harness/tests/graphics/test_live.py b/harness/tests/graphics/test_live.py
index 74156a64..aa39079c 100644
--- a/harness/tests/graphics/test_live.py
+++ b/harness/tests/graphics/test_live.py
@@ -16,7 +16,6 @@ from codeverse.tracks.graphics import (
GraphicsTrack,
frames_render_set,
graphics_prompt_context,
- plan_graphics,
)
from codeverse.workspace import Workspace
@@ -41,7 +40,8 @@ def test_live_plan_generate_build_judge(tmp_path, keys_ok):
budget=Budget(max_rounds=1, max_usd=0.5), backends=Backends(planner=MODEL, generator=f"single-shot:{MODEL}", judge=MODEL))
ws = Workspace(tmp_path / "live_gfx").create()
model = get_chat_model(MODEL)
- plan = plan_graphics(spec, MODEL, ws, model=model)
+ from codeverse.tracks.planner import plan as run_planner
+ plan = run_planner(spec, MODEL, GraphicsPlan, ws, model=model, **GraphicsTrack()._plan_kwargs(spec))
assert isinstance(plan, GraphicsPlan) and plan.passes and plan.key_visuals and plan.motion
rt = GlslShaderRuntime()
rt.skeleton(ws, plan)
diff --git a/harness/tests/graphics/test_track.py b/harness/tests/graphics/test_track.py
index 7c0b993e..ec3c2893 100644
--- a/harness/tests/graphics/test_track.py
+++ b/harness/tests/graphics/test_track.py
@@ -26,11 +26,9 @@ from codeverse.tracks import get_track
from codeverse.tracks.graphics import (
GraphicsPipeline,
GraphicsTrack,
- build_plan_system_prompt,
ensure_graphics_acceptance,
graphics_prompt_context,
plan_example,
- plan_graphics,
)
from codeverse.tracks.planner import PlanningError
from codeverse.workspace import Workspace
@@ -107,6 +105,16 @@ def _services(**kw):
return FakeServices(runtime_factory=lambda lang: FakeGlRuntime(), **kw)
+def _plan(spec, ws, model, budget=None):
+ """What plan_graphics() used to be: run_planner with GraphicsTrack's own hooks.
+ The wrapper was a second spelling of _plan_kwargs and had no production caller,
+ so the tests go through the hooks the live path actually uses."""
+ from codeverse.tracks.graphics import GraphicsTrack
+ from codeverse.tracks.planner import plan as run_planner
+ return run_planner(spec, "fake:planner", GraphicsPlan, ws, model=model, budget=budget,
+ **GraphicsTrack()._plan_kwargs(spec))
+
+
def test_graphics_track_end_to_end(tmp_path, settings):
spec = make_spec(max_rounds=3)
ws = Workspace(tmp_path / "runs" / "rain")
@@ -209,7 +217,7 @@ def test_planner_reask_and_acceptance(tmp_ws):
return plan_example()
model = FakeChatModel(responder)
- plan = plan_graphics(spec, "fake:planner", tmp_ws, model=model)
+ plan = _plan(spec, tmp_ws, model)
assert attempts["n"] == 2 and plan.title == "Neon rain on a window" and tmp_ws.plan_path.is_file()
assert "Fix EXACTLY these problems" in model.requests[1].messages[-1].text
ids = [a.id for a in plan.acceptance]
@@ -219,8 +227,10 @@ def test_planner_reask_and_acceptance(tmp_ws):
assert any(a.id.startswith("motion") for a in p2.acceptance) and not any("ground" in a.text.lower() for a in p2.acceptance)
always_bad = FakeChatModel(lambda req: {"title": "x"})
with pytest.raises(PlanningError):
- plan_graphics(spec, "fake:planner", tmp_ws, model=always_bad)
- sys_prompt = build_plan_system_prompt(spec)
+ _plan(spec, tmp_ws, always_bad)
+ from codeverse.tracks.graphics import PLAN_TEMPLATE
+ from codeverse.tracks.planner import build_system_prompt
+ sys_prompt = build_system_prompt(spec, GraphicsPlan, template=PLAN_TEMPLATE, example=plan_example())
assert "ART-DIRECTOR" in sys_prompt and "NeonRainWindow" in sys_prompt
diff --git a/harness/tests/orchestrator_tracks/test_refactor_batch2.py b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
index 63c0fc31..8c3d28a8 100644
--- a/harness/tests/orchestrator_tracks/test_refactor_batch2.py
+++ b/harness/tests/orchestrator_tracks/test_refactor_batch2.py
@@ -151,16 +151,19 @@ def test_judge_context_needs_no_run_context(tmp_path):
def test_graphics_planner_hooks_charge_budget_on_planning_error(tmp_ws):
import pytest
+ from codeverse.contracts.plan import GraphicsPlan
from codeverse.contracts.spec import Budget
from codeverse.orchestrator import BudgetGuard
- from codeverse.tracks.graphics import plan_graphics
+ from codeverse.tracks.graphics import GraphicsTrack
from codeverse.tracks.planner import PlanningError
+ from codeverse.tracks.planner import plan as run_planner
spec = make_spec(Track.GRAPHICS, Language.GLSL_SHADER)
budget = BudgetGuard(Budget(max_usd=5.0, max_minutes=10))
always_bad = FakeChatModel(lambda req: {"title": "x"})
with pytest.raises(PlanningError):
- plan_graphics(spec, "fake:planner", tmp_ws, model=always_bad, budget=budget)
+ run_planner(spec, "fake:planner", GraphicsPlan, tmp_ws, model=always_bad, budget=budget,
+ **GraphicsTrack()._plan_kwargs(spec))
assert budget.spent.cost_usd > 0, "a failed re-ask is still paid for"
diff --git a/harness/tests/prompts/conftest.py b/harness/tests/prompts/conftest.py
index fe2e84dc..6f9d6cee 100644
--- a/harness/tests/prompts/conftest.py
+++ b/harness/tests/prompts/conftest.py
@@ -24,7 +24,6 @@ HELPERS = Path(__file__).resolve().parent / "helpers"
PROMPT_FILES = [
"system/harness_contract.md",
"system/singleshot_format.md",
- "system/tools_usage.md",
"blender/contract.md",
"blender/cookbook.md",
"cadquery/contract.md",
diff --git a/harness/tests/prompts/test_files.py b/harness/tests/prompts/test_files.py
index 18ce595e..bdb7d178 100644
--- a/harness/tests/prompts/test_files.py
+++ b/harness/tests/prompts/test_files.py
@@ -81,8 +81,7 @@ def test_system_prompts_cover_the_laws() -> None:
for needle in ("artifacts/", "render_sheet", "check_connectivity", "2 mm",
"Definition of done", "ground", "seed"):
assert needle.lower() in hc.lower(), f"harness_contract.md must mention {needle!r}"
- tu = read_prompt("system/tools_usage.md")
- for tool in ("build", "measure", "render_views", "render_sheet", "isolate", "cross_section",
- "check_connectivity", "check_contract", "compare_silhouette", "joint_sweep",
- "shader_probe", "scene_probe", "read_cookbook"):
- assert f"`{tool}" in tu, f"tools_usage.md must document {tool!r}"
+ # system/tools_usage.md was a hand-written tool list with no production reader
+ # (deleted 2026-08-28): the agent's tool section is GENERATED from the @tool
+ # registrations by agents/materialize._tool_section, so "every tool is documented"
+ # is true by construction and this assertion was checking a file nobody read.