Skip to content

Official Codebase of 3DCodeVerse

Python

sweep the api-agent's ashes: dead seams, dead knobs, stale prose

gaoypeng4db6c199fcc0
diff --git a/harness/bench/compare_backends.py b/harness/bench/compare_backends.py
index 39f051cf..582c61dd 100644
--- a/harness/bench/compare_backends.py
+++ b/harness/bench/compare_backends.py
@@ -453,9 +453,8 @@ def _preflight(judge: str, arms: Sequence[Arm], opts: CompareOptions, *, wait_mi
from codeverse.models.health import probe
def api_model(target: str) -> str:
- """'api-agent:gemini:x' -> 'gemini:x'; a subscription CLI target -> ''."""
- inner = target.split(":", 1)[1] if target.startswith("api-agent:") else target
- return inner if inner.startswith(("gemini:", "anthropic:", "openai:")) else ""
+ """An API-billed oneshot target ('gemini:x') as-is; a subscription CLI target -> ''."""
+ return target if target.startswith(("gemini:", "anthropic:", "openai:")) else ""
models = {judge, *(api_model(a.target) for a in arms)}
if any(a.kind.startswith("harness") for a in arms):
diff --git a/harness/codeverse/__init__.py b/harness/codeverse/__init__.py
index f7c7b547..4548d322 100644
--- a/harness/codeverse/__init__.py
+++ b/harness/codeverse/__init__.py
@@ -13,15 +13,15 @@ Package map (each sub-package has its own docstring):
- ``contracts`` pydantic models shared by everything (Spec, Plan, Gate,
Judgment, Usage, RunRecord ...). No logic lives here.
- ``models`` ChatModel backends: Gemini / Anthropic / OpenAI (+ key pool).
-- ``agents`` CodingAgent backends: gemini-cli / claude-code / codex /
- antigravity / api-agent (tool loop on any ChatModel).
+- ``agents`` CodingAgent backends: the vendor CLIs — gemini-cli /
+ claude-code / codex / antigravity.
- ``languages`` LanguageRuntime per raw language: lint, build, export.
- ``spatial`` language-agnostic 3D tools on GLB/scene: measure, render,
slice, isolate, silhouette, joints, probes — also served as
an MCP server to agentic harnesses.
- ``judges`` VLM judges + rubrics + deterministic metrics.
- ``tracks`` static_object / articulated_object / scene / graphics pipelines.
-- ``orchestrator`` stage runner, round loop, fan-out, budget, events.
+- ``orchestrator`` stage runner, round loop, budget, candidate selection.
- ``flywheel`` run records → dataset samples / preference pairs.
- ``cli`` ``3dcv`` command line.
"""
diff --git a/harness/codeverse/agents/__init__.py b/harness/codeverse/agents/__init__.py
index 4a36bb0b..2f3987bf 100644
--- a/harness/codeverse/agents/__init__.py
+++ b/harness/codeverse/agents/__init__.py
@@ -1,7 +1,7 @@
"""CodingAgent backends (agentic sessions on a workspace).
Ids: ``gemini-cli:<model>`` · ``claude-code:<model>`` · ``codex:<model>`` ·
-``agy:<model>`` · ``api-agent:<provider>:<model>``.
+``agy:<model>``.
"""
from codeverse.agents.base import CodingAgent
diff --git a/harness/codeverse/agents/base.py b/harness/codeverse/agents/base.py
index 2f77336f..a4589669 100644
--- a/harness/codeverse/agents/base.py
+++ b/harness/codeverse/agents/base.py
@@ -25,7 +25,7 @@ class CodingAgent(Protocol):
process group on timeout).
"""
- kind: str # gemini-cli | claude-code | codex | agy | api-agent
+ kind: str # gemini-cli | claude-code | codex | agy
model: str
@property
diff --git a/harness/codeverse/agents/cli_common.py b/harness/codeverse/agents/cli_common.py
index cb81d8fb..7fd392b5 100644
--- a/harness/codeverse/agents/cli_common.py
+++ b/harness/codeverse/agents/cli_common.py
@@ -179,13 +179,12 @@ class Session:
# --------------------------------------------------------------------------- serialisation
-#: agent kinds whose sessions are EXCLUSIVE per workspace. CLI agents have no
-#: write-time enforcement (FileTools guards only the in-process api-agent), and both
-#: session snapshots run ``git add -A`` — a late-starting session's ``pre:`` commit
-#: absorbs a sibling's in-flight writes, which makes concurrent provenance (and any
-#: per-path rollback, see :func:`_enforce_scope`) unfixable after the fact. So CLI
-#: sessions on one workspace are serialised; the main generator (api-agent) keeps its
-#: full parallelism because its writes are scope-checked as they happen.
+#: agent kinds whose sessions are EXCLUSIVE per workspace — today, every kind. No
+#: backend has write-time enforcement (scope is checked post-session by
+#: :func:`_enforce_scope`), and both session snapshots run ``git add -A`` — a
+#: late-starting session's ``pre:`` commit absorbs a sibling's in-flight writes, which
+#: makes concurrent provenance (and any per-path rollback) unfixable after the fact.
+#: So agent sessions on one workspace are serialised.
EXCLUSIVE_KINDS = frozenset({"claude-code", "codex", "gemini-cli", "agy"})
_WS_SESSION_LOCKS: dict[str, threading.RLock] = {}
diff --git a/harness/codeverse/cli/main.py b/harness/codeverse/cli/main.py
index e1e1f868..fc7d6e59 100644
--- a/harness/codeverse/cli/main.py
+++ b/harness/codeverse/cli/main.py
@@ -104,7 +104,7 @@ def make(
generator: Annotated[
str | None,
typer.Option(
- help="api-agent:gemini:gemini-3.7-flash | gemini-cli:... | claude-code:... | codex:... | agy:..."
+ help="gemini-cli:gemini-3.6-flash | claude-code:... | codex:... | agy:... | single-shot:gemini:..."
),
] = None,
planner: Annotated[str | None, typer.Option()] = None,
diff --git a/harness/codeverse/cli/skills_cmd.py b/harness/codeverse/cli/skills_cmd.py
index 1e3f6f97..c6cad7f2 100644
--- a/harness/codeverse/cli/skills_cmd.py
+++ b/harness/codeverse/cli/skills_cmd.py
@@ -119,7 +119,7 @@ def report(
) -> None:
"""Deep-read rate per skill and per backend — the number that replaces "0 of 16".
- Targets from the design: >= 60% for CLI backends, >= 80% for api-agent. A skill
+ Target from the design: >= 60% deep-read rate for a CLI backend. A skill
under 20% over 20 sessions is merged or deleted.
Sessions whose CONTROL bundle was also "read" are counted separately and excluded from
diff --git a/harness/codeverse/contracts/__init__.py b/harness/codeverse/contracts/__init__.py
index 889a2d08..8b39e5ed 100644
--- a/harness/codeverse/contracts/__init__.py
+++ b/harness/codeverse/contracts/__init__.py
@@ -11,7 +11,7 @@ Modules:
- ``agent`` CodingAgent job/result shapes.
"""
-from codeverse.contracts.agent import AgentJob, AgentResult, ApiAgentOptions, FileChange
+from codeverse.contracts.agent import AgentJob, AgentResult, FileChange
from codeverse.contracts.artifacts import (
BuildResult,
GateFinding,
@@ -70,7 +70,6 @@ __all__ = [
"AcceptanceItem",
"AgentJob",
"AgentResult",
- "ApiAgentOptions",
"ArticulatedPlan",
"AssetPlan",
"BBox",
diff --git a/harness/codeverse/contracts/agent.py b/harness/codeverse/contracts/agent.py
index d67d510d..55f00f4b 100644
--- a/harness/codeverse/contracts/agent.py
+++ b/harness/codeverse/contracts/agent.py
@@ -23,20 +23,8 @@ class FileChange(BaseModel):
lines_removed: int = 0
-class ApiAgentOptions(BaseModel):
- """Knobs honoured only by the in-process ``api-agent`` backend."""
-
- max_usd: float = Field(
- default=0.0, description="stop the session when its own cost exceeds this (0 = no cap)"
- )
- temperature: float = 0.3
- thinking: str = "low"
- allow_shell: bool = True
-
-
#: legacy ``AgentJob.extra`` keys lifted into the typed fields (values stay in extra too)
_LEGACY_JOB_KEYS = ("round", "kind", "language", "track", "files_hint", "mcp_command")
-_LEGACY_API_KEYS = ("max_usd", "temperature", "thinking", "allow_shell")
class AgentJob(BaseModel):
@@ -85,13 +73,12 @@ class AgentJob(BaseModel):
read_only: list[str] = Field(
default_factory=list,
description="harness-owned files inside write_roots the agent may read but never write "
- "(src/recipes.glsl for glsl_shader: contracts.common.HARNESS_OWNED_SRC); a write_file / "
- "edit_file on one is refused with 'harness-owned — call its functions instead'",
+ "(src/recipes.glsl for glsl_shader: contracts.common.HARNESS_OWNED_SRC); enforced "
+ "post-session: a write to one is reverted and the session failed",
)
mcp_command: list[str] | None = Field(
default=None, description="override for the 3dcv MCP server command"
)
- api: ApiAgentOptions = Field(default_factory=ApiAgentOptions)
extra: dict[str, Any] = Field(
default_factory=dict, description="one-off backend hints (legacy keys are lifted)"
)
@@ -111,10 +98,6 @@ class AgentJob(BaseModel):
for key in _LEGACY_JOB_KEYS:
if key in extra and key not in data:
data[key] = extra[key]
- if "api" not in data:
- lifted = {k: extra[k] for k in _LEGACY_API_KEYS if k in extra}
- if lifted:
- data["api"] = lifted
return data
diff --git a/harness/codeverse/contracts/skills.py b/harness/codeverse/contracts/skills.py
index 5bc36b95..8fa944b6 100644
--- a/harness/codeverse/contracts/skills.py
+++ b/harness/codeverse/contracts/skills.py
@@ -18,7 +18,7 @@ class SkillRead(BaseModel):
deep: bool = Field(default=False, description="a references/*.md atime > mtime: the body was read and followed")
deep_measurable: bool = Field(default=True, description="False when the bundle ships no references/ file to probe")
body_tokens: int = 0
- first_seen_turn: int | None = Field(default=None, description="api-agent only: exact turn of the first read")
+ first_seen_turn: int | None = Field(default=None, description="historical (pre-2026-08-28) api-agent runs only: exact turn of the first read")
reason: str = Field(default="", description="which route attached it, and which finding")
diff --git a/harness/codeverse/cost/instrument.py b/harness/codeverse/cost/instrument.py
index e53810e2..0f9b1aa6 100644
--- a/harness/codeverse/cost/instrument.py
+++ b/harness/codeverse/cost/instrument.py
@@ -6,13 +6,13 @@ Two thin proxies do the whole job:
Every ``generate`` appends one :class:`~codeverse.cost.types.CallCost` row
(tokens, the unit prices actually used + their provenance, $, latency,
cache hit, outcome). Because ``models.registry.get_chat_model`` returns the
- proxy, this covers the planner, the judges, the captioner, single-shot
- generation **and every turn of the in-process api-agent**.
+ proxy, this covers the planner, the judges, the captioner and single-shot
+ generation.
* :class:`MeteredAgent` wraps a :class:`~codeverse.agents.base.CodingAgent`.
It sets the ambient attribution (round / stage / label) for the session so
the model rows above land in the right bucket, applies the profile's turn cap
- and, for backends whose calls we cannot see (the subscription CLIs — see
- :data:`IN_PROCESS_AGENT_KINDS`), records one session row from
+ and, for backends whose calls we cannot see (every subscription CLI that does
+ not declare ``meters_own_calls``), records one session row from
``AgentResult.usage``.
Accounting is never allowed to fail a run: every hook is wrapped, and an
@@ -80,36 +80,20 @@ def per_call_metering() -> bool:
return _active_ledgers > 0
-#: Agent kinds whose model calls go through ``models.get_chat_model`` and are
-#: therefore ALREADY one ledger row each (``MeteredChatModel`` wraps that
-#: factory). Every other kind is an opaque subscription CLI: we never see its
-#: turns, it hands back one ``AgentResult.usage`` for the whole session, and that
-#: session row is the only record of the money.
-#:
-#: This used to be decided by a thread-local "did anybody write a row while the
-#: session ran?" counter, which is wrong in both directions and was reproduced by
-#: the verifier (``adversarial.py``): a CLI session during which any in-process
-#: tool billed a model (a texture pass, a summariser) looked metered and its whole
-#: session — $1.23 in the reproduction — was silently dropped; and an in-process
-#: session whose turns ran in another thread looked unmetered and was counted
-#: twice. The backend either meters itself or it does not; that is a property of
-#: the backend, not of what happened to run alongside it.
-#: empty since 2026-08-28: the in-process ``api-agent`` was the only backend whose
-#: individual model calls landed on the ledger by themselves. A vendor CLI bills as one
-#: session row. Kept (rather than deleted) because ``meters_own_calls`` is the seam a
-#: future self-metering backend would declare itself through.
-IN_PROCESS_AGENT_KINDS: frozenset[str] = frozenset()
-
-
def meters_own_calls(agent: Any) -> bool:
"""True when this backend's individual model calls are already on the ledger.
- A backend may state it with a ``meters_own_calls`` attribute; otherwise the
- kind decides (:data:`IN_PROCESS_AGENT_KINDS`)."""
- declared = getattr(agent, "meters_own_calls", None)
- if declared is not None:
- return bool(declared)
- return str(getattr(agent, "kind", "")) in IN_PROCESS_AGENT_KINDS
+ Declared by the backend itself (a ``meters_own_calls`` attribute) — a property
+ of the backend, not of what happened to run alongside it. The old thread-local
+ "did anybody write a row while the session ran?" counter was wrong in both
+ directions (reproduced by ``adversarial.py``): a CLI session during which an
+ in-process tool billed a model (a texture pass, a summariser) looked metered
+ and its whole session row was silently dropped, and a session whose turns ran
+ in another thread was counted twice. Every shipped backend is a vendor CLI
+ billing as ONE session row, so the default is False; the deleted in-process
+ api-agent (2026-08-28) was the last one whose turns landed on the ledger by
+ themselves."""
+ return bool(getattr(agent, "meters_own_calls", False))
class MeteredChatModel:
diff --git a/harness/codeverse/cost/profiles.py b/harness/codeverse/cost/profiles.py
index 3e143385..977ce17c 100644
--- a/harness/codeverse/cost/profiles.py
+++ b/harness/codeverse/cost/profiles.py
@@ -23,10 +23,10 @@ Every number is taken from the measurements in ``docs/COST.md``:
$4.93, r02 for $6.80, r03 for $33.49). It does **not** shrink the judge
payload and does **not** carry its own turn cap — both were measured and
neither paid (see the field comments below).
-* **balanced** — exactly today's defaults (api-agent generator, pro judge at
+* **balanced** — exactly today's defaults (gemini-cli generator, pro judge at
n=1, 4 rounds, no best-of-N, no texture): the arm every number in the audit
was measured on.
-* **quality** — api-agent + the pro judge at ``n=3`` (σ 0.030/√3 ≈ 0.017),
+* **quality** — gemini-cli + the pro judge at ``n=3`` (σ 0.030/√3 ≈ 0.017),
best-of-2 baselines and the texture pass behind its before/after gate.
The expected $ per profile comes from the same 61 runs; see
@@ -141,7 +141,7 @@ PROFILES: dict[str, Profile] = {
expected_usd=3.20,
expected_score="best-of-2 lifted the stool baseline 0.563 → 0.612 and the texture pass "
"0.686 → 0.701; judge σ 0.017 at n=3",
- note="api-agent + pro judge n=3 + 4 rounds + best-of-2 + texture pass",
+ note="gemini-cli + pro judge n=3 + 4 rounds + best-of-2 + texture pass",
),
}
diff --git a/harness/codeverse/cost/routing.py b/harness/codeverse/cost/routing.py
index d91623bd..b620972b 100644
--- a/harness/codeverse/cost/routing.py
+++ b/harness/codeverse/cost/routing.py
@@ -55,8 +55,8 @@ ROUTES: tuple[Route, ...] = (
"graphics track: 5/6 passed, median 0.810, $0.099 per run",
"glsl_shader / opengl_python — one file, compiler feedback, no tool loop needed"),
Route(Role.GENERATOR, "gemini-cli:gemini-3.7-flash", 0.73,
- "compare_v1: 0.827 — statistically the same as api-agent at 2x the price and 2x the wall clock",
- "only when you need the CLI's own tooling; the api-agent is cheaper for identical quality"),
+ "compare_v1: 0.827 — statistically the same as the since-deleted api-agent arm at 2x the price",
+ "costs ~2x the 3.6-flash default for the same measured quality — prefer the default"),
Route(Role.GENERATOR, "codex:gpt-5.6-sol", 1.93,
"one-shot 0.786 for $0.20; inside the harness loop $1.93/run (long cached context at $0.40/M)",
"strong one-shot baseline; expensive as a loop generator"),
diff --git a/harness/codeverse/flywheel/telemetry.py b/harness/codeverse/flywheel/telemetry.py
index 24625af5..fb7497fc 100644
--- a/harness/codeverse/flywheel/telemetry.py
+++ b/harness/codeverse/flywheel/telemetry.py
@@ -92,15 +92,12 @@ def _planner_defaults() -> dict[str, Any]:
def _generator_defaults(model_id: str) -> dict[str, Any]:
- """Only the in-process ``api-agent`` exposes sampling knobs; CLI agents own theirs."""
- if backend_kind(model_id) != "api-agent": # historical runs only (backend deleted 2026-08-28)
+ """CLI agents own their sampling knobs. Only the deleted in-process ``api-agent``
+ exposed ours — the constants below reproduce its shipped defaults so a HISTORICAL
+ run record (pre-2026-08-28) still reports what it actually ran with."""
+ if backend_kind(model_id) != "api-agent":
return {}
- try:
- from codeverse.contracts.agent import ApiAgentOptions
- except Exception: # pragma: no cover
- return {}
- o = ApiAgentOptions()
- return {"temperature": o.temperature, "thinking": o.thinking}
+ return {"temperature": 0.3, "thinking": "low"}
def _rubric_name(record: RunRecord) -> str:
diff --git a/harness/codeverse/judges/vlm_judge.py b/harness/codeverse/judges/vlm_judge.py
index afc5c4c0..dc36e8fc 100644
--- a/harness/codeverse/judges/vlm_judge.py
+++ b/harness/codeverse/judges/vlm_judge.py
@@ -43,12 +43,12 @@ from codeverse.models.base import ChatModel, ModelError
log = logging.getLogger(__name__)
-#: retry budget for ONE judge sample, every attempt included. The verdict call is 42 s
-#: p50 / 73 s p90 on a normal day and 50 / 103 s under a storm (audit 2026-08-26 §2), so
-#: 240 s is ~2.3x the storm p90. It replaces 3 x (a 300 s read timeout + the model's 900 s
-#: retry deadline): two storm-day rounds lost 1 162 s and 927 s that way (three 300 s
-#: timeouts each) before a second sample answered in 128 s.
-SAMPLE_BUDGET_S = 900.0 # owner 2026-08-27: generous time, zero timeouts (was 240)
+#: retry budget for ONE judge sample, every attempt included — the owner's rule
+#: (2026-08-27): generous time, zero timeouts. For scale, the verdict call is 42 s p50 /
+#: 73 s p90 on a normal day and 50 / 103 s under a storm (audit 2026-08-26 §2); the old
+#: 240 s cap, and before it 3 x (a 300 s read timeout + the model's 900 s retry deadline),
+#: lost two storm-day rounds 1 162 s and 927 s before a second sample answered in 128 s.
+SAMPLE_BUDGET_S = 900.0
#: the floor of one attempt's ``max_wait_s``: a last attempt still gets a real try
SAMPLE_MIN_WAIT_S = 20.0
diff --git a/harness/codeverse/models/gemini.py b/harness/codeverse/models/gemini.py
index 554a2287..7e53347a 100644
--- a/harness/codeverse/models/gemini.py
+++ b/harness/codeverse/models/gemini.py
@@ -54,10 +54,10 @@ _RETRY_DELAY_RE = re.compile(r"retryDelay['\"]?\s*:\s*['\"]?(\d+(?:\.\d+)?)s")
#: floor of the per-attempt HTTP read timeout derived from the remaining retry
#: budget (``ChatRequest.max_wait_s``): a call that starts near its deadline still
#: gets ONE real attempt instead of an instant socket timeout. 20 s matches the
-#: callers' own floors (``agents.api_agent.TURN_WAIT_MIN_S`` = 20 s,
-#: ``judges.vlm_judge.SAMPLE_MIN_WAIT_S`` = 20 s), so an in-flight attempt may
-#: overshoot the deadline by at most this floor — never by the full ``timeout_s``
-#: (300 s) read timeout, which was the dominant overshoot (audit 2026-08-27).
+#: judge's own floor (``judges.vlm_judge.SAMPLE_MIN_WAIT_S`` = 20 s), so an in-flight
+#: attempt may overshoot the deadline by at most this floor — never by the full
+#: ``timeout_s`` read timeout, which was the dominant overshoot (audit 2026-08-27,
+#: when that ceiling was 300 s).
HTTP_TIMEOUT_FLOOR_S = 20.0
_pools: dict[tuple[str, ...], KeyPool] = {}
diff --git a/harness/codeverse/skills/delivery.py b/harness/codeverse/skills/delivery.py
index dbc26e5c..f107b236 100644
--- a/harness/codeverse/skills/delivery.py
+++ b/harness/codeverse/skills/delivery.py
@@ -53,11 +53,6 @@ class Delivery:
"""An explicit one-line-per-skill index in the agent body file."""
return not self.native_loader
- @property
- def needs_tool(self) -> bool:
- """``read_skill``: the routed set as an affordance rather than as prose."""
- return not self.native_loader
-
#: The whole per-backend policy. One row per backend; anything absent is loaderless.
_BACKENDS: dict[str, Delivery] = {
diff --git a/harness/codeverse/skills/prompting.py b/harness/codeverse/skills/prompting.py
index 98a55a9f..bd3e4072 100644
--- a/harness/codeverse/skills/prompting.py
+++ b/harness/codeverse/skills/prompting.py
@@ -8,8 +8,8 @@ already being sent, and each backend gets the least text that reaches it:
* claude-code / codex / gemini-cli / agy discover ``SKILL.md`` themselves. Writing our
own index would double-index the same bundles, so they get ONE sentence (~35 tokens).
-* api-agent has no loader, so it gets a one-line-per-skill index inside the AGENTS.md
- body, which is already message 0 of the session.
+* a backend nobody has classified yet (``delivery.py``'s safe answer) gets a
+ one-line-per-skill index inside the AGENTS.md body, which is already message 0.
* a single-shot call has no read loop at all: it gets ONE body inlined, so the cost is
explicit and bounded instead of a pointer nobody can follow.
* on a repair round the gate-fired skills are named beside the findings they answer, in
@@ -42,20 +42,19 @@ NATIVE_LOADERS = tuple(k for k in known_backends() if delivery_for(k).native_loa
#: the user installed globally, so "the ones that match" is the right instruction there.
MANDATE = ("Skills for this task are installed in this workspace. Any whose description matches "
"your task are MANDATORY — activate and read them before writing code.")
-#: what api-agent is told. The list under it was chosen by OUR router from the track, the
-#: language, the plan and the previous round's gate findings — inviting the agent to filter
-#: it again would only lose reads, and it cannot see what we used to pick them.
+#: what an unclassified (loaderless) backend is told. The list under it was chosen by OUR
+#: router from the track, the language, the plan and the previous round's gate findings —
+#: inviting the agent to filter it again would only lose reads.
MANDATE_ROUTED = ("The skills below were selected for THIS task by the harness, from your track, "
"your language, your plan and the gate findings of the previous round. Reading "
"them is MANDATORY — open each one before writing code.")
_HEADING = "## Skills"
-#: how much of a description the api-agent index repeats. The spec lets a description run to
-#: 1024 chars and ours use it — they are what a CLI's own matcher reads. api-agent's index is
-#: not a matcher: OUR router already decided, so the line only has to be recognisable enough
-#: for the agent to know which file to open. Measured on the real library, quoting the full
-#: 14 descriptions costs 780 tokens for five skills; the first clause costs 401.
+#: how much of a description the routed index repeats. Descriptions run to 1024 chars for
+#: the CLIs' own matchers; our index is not a matcher — the router already decided — so a
+#: line only has to be recognisable enough to know which file to open. Measured on the
+#: real library: five full descriptions cost 780 tokens; the first clause costs 401.
INDEX_SUMMARY_CHARS = 200
_SENTENCE_END = re.compile(r"(?<=[.!?])\s+(?=[A-Z`])")
@@ -77,8 +76,9 @@ def skill_path(name: str, *, agent_kind: str = "") -> str:
def index_block(skills: Sequence[Skill | Selection], agent_kind: str) -> str:
"""The text appended to the AGENTS.md body for ``agent_kind`` ('' when nothing is added).
- Native-loader backends get the mandate only; api-agent (and anything unknown, which
- is the safe assumption) also gets the one-line-per-skill index it cannot discover.
+ Native-loader backends get the mandate only; an unclassified backend (the safe
+ assumption for anything unknown) also gets the one-line-per-skill index it cannot
+ discover itself.
"""
items = [s.skill if isinstance(s, Selection) else s for s in skills]
if not items:
diff --git a/harness/codeverse/skills/telemetry.py b/harness/codeverse/skills/telemetry.py
index a5b5fc64..2670f704 100644
--- a/harness/codeverse/skills/telemetry.py
+++ b/harness/codeverse/skills/telemetry.py
@@ -19,14 +19,10 @@ Three signals, cheapest first:
and git reads every untracked file to diff it; and all four CLIs open ``references/``
while activating a skill, even one whose description does not match the task. So the
depth signal is kept — it is still the difference between "listed" and "opened" — but
- it is no longer trusted on its own, which is what signal 4 is for. (A bundle that
+ it is no longer trusted on its own, which is what signal 3 is for. (A bundle that
ships no ``references/`` cannot be probed for depth and says so via
``deep_measurable`` instead of quietly scoring 0.)
-3. **exact reads for api-agent** — our own tool loop owns ``read_file``, so it logs every
- read of a skill path with its turn into ``telemetry/skill_reads.jsonl``. api-agent is
- therefore the CALIBRATION arm: compare signals 1/2 against ground truth and report the
- probe's false-positive / false-negative rate rather than assuming it is exact.
-4. **the control bundle** — ``materialize.write_control`` puts one never-routed,
+3. **the control bundle** — ``materialize.write_control`` puts one never-routed,
never-indexed bundle beside the real ones. Nothing should ever open it. When it comes
back opened, this session's atime evidence proves nothing, ``control_read`` is set and
``deep_read_rate`` returns ``None`` rather than a confident 100%. It costs two small
@@ -48,62 +44,11 @@ from codeverse.skills.prompting import AGENTS_SKILL_ROOT, CLAUDE_SKILL_ROOT
log = logging.getLogger(__name__)
TELEMETRY_DIR = "telemetry"
-READS_FILE = "skill_reads.jsonl" # signal 3, written during the session
SKILLS_FILE = "skills.jsonl" # the per-session roll-up, written after it
#: seconds of slack: some filesystems round atime/mtime differently on the two writes
ATIME_EPSILON_S = 1.0
-def is_skill_path(rel: str) -> str:
- """The skill name a workspace-relative path belongs to, or '' — the one place the
- two discovery roots are recognised, so signal 3 cannot drift from what we wrote."""
- p = str(rel).replace("\\", "/")
- while p.startswith("./"):
- p = p[2:]
- for root in (AGENTS_SKILL_ROOT, CLAUDE_SKILL_ROOT):
- if p.startswith(root + "/"):
- rest = p[len(root) + 1:].split("/")
- return rest[0] if rest and rest[0] else ""
- return ""
-
-
-def record_exact_read(ws_root: Path, rel_path: str, *, turn: int, label: str = "") -> None:
- """Signal 3: log one api-agent read of a skill file. Never raises into the loop."""
- name = is_skill_path(rel_path)
- if not name:
- return
- try:
- d = Path(ws_root) / TELEMETRY_DIR
- d.mkdir(parents=True, exist_ok=True)
- with (d / READS_FILE).open("a") as fh:
- fh.write(json.dumps({"skill": name, "path": str(rel_path), "turn": int(turn), "label": label}) + "\n")
- except OSError as e: # telemetry must never cost a run
- log.debug("could not record a skill read: %s", e)
-
-
-def _exact_reads(ws_root: Path) -> dict[str, dict[str, Any]]:
- """name → {first_turn, deep} from ``skill_reads.jsonl`` (empty when absent)."""
- p = Path(ws_root) / TELEMETRY_DIR / READS_FILE
- if not p.is_file():
- return {}
- out: dict[str, dict[str, Any]] = {}
- for line in p.read_text().splitlines():
- try:
- row = json.loads(line)
- except json.JSONDecodeError:
- continue
- name = str(row.get("skill") or "")
- if not name:
- continue
- rec = out.setdefault(name, {"first_turn": None, "deep": False})
- turn = row.get("turn")
- if isinstance(turn, int) and (rec["first_turn"] is None or turn < rec["first_turn"]):
- rec["first_turn"] = turn
- if "/references/" in str(row.get("path", "")):
- rec["deep"] = True
- return out
-
-
def _opened(path: Path) -> bool:
try:
st = path.stat()
@@ -115,7 +60,6 @@ def _opened(path: Path) -> bool:
def probe_reads(ws_root: Path, materialized: SkillsMaterialized) -> SkillsUsage:
"""What the session did with the bundles we attached, per skill."""
root = Path(ws_root)
- exact = _exact_reads(root)
control_present, control_read = _control_state(root)
reads: list[SkillRead] = []
for sel in materialized.selections:
@@ -124,10 +68,6 @@ def probe_reads(ws_root: Path, materialized: SkillsMaterialized) -> SkillsUsage:
refs = [f for b in bundles for f in sorted((b / "references").glob("*.md"))]
surfaced = any(_opened(b / "SKILL.md") for b in bundles)
deep = any(_opened(f) for f in refs)
- got = exact.get(name)
- if got:
- surfaced = True
- deep = deep or bool(got["deep"])
# reading a reference means the bundle was reached, whatever SKILL.md's atime says
# (relatime can miss the second access within 24 h, and some CLIs stream the body
# out of their own index rather than re-opening the file)
@@ -135,7 +75,6 @@ def probe_reads(ws_root: Path, materialized: SkillsMaterialized) -> SkillsUsage:
reads.append(SkillRead(
name=name, surfaced=surfaced, deep=deep, deep_measurable=bool(refs),
body_tokens=sel.skill.body_tokens, reason=sel.reason,
- first_seen_turn=got["first_turn"] if got else None,
))
return SkillsUsage(
listed=list(materialized.listed), reads=reads, index_tokens=materialized.index_tokens,
@@ -190,5 +129,5 @@ def deep_read_rate(usages: Sequence[SkillsUsage]) -> dict[str, tuple[int, int]]:
return {k: (v[0], v[1]) for k, v in sorted(out.items())}
-__all__ = ["ATIME_EPSILON_S", "READS_FILE", "SKILLS_FILE", "TELEMETRY_DIR", "append_usage",
- "deep_read_rate", "is_skill_path", "probe_reads", "record_exact_read"]
+__all__ = ["ATIME_EPSILON_S", "SKILLS_FILE", "TELEMETRY_DIR", "append_usage",
+ "deep_read_rate", "probe_reads"]
diff --git a/harness/codeverse/tracks/generation.py b/harness/codeverse/tracks/generation.py
index 78d7da40..04a0b083 100644
--- a/harness/codeverse/tracks/generation.py
+++ b/harness/codeverse/tracks/generation.py
@@ -1,7 +1,7 @@
"""Code generation: one entry point, two strategies.
* **CodingAgent path** — ``agent_id`` is any id ``codeverse.agents`` knows
- (``gemini-cli:*``, ``claude-code:*``, ``codex:*``, ``agy:*``, ``api-agent:*``):
+ (``gemini-cli:*``, ``claude-code:*``, ``codex:*``, ``agy:*``):
the workspace is materialised once per run (AGENTS.md + MCP config), then
``agent.run(AgentJob)``. Truth is on disk: ``ok`` means files changed.
* **Single-shot path** — ``agent_id = "single-shot:<provider>:<model>"``: one
@@ -101,8 +101,8 @@ def agent_max_turns(default: int = DEFAULT_AGENT_MAX_TURNS) -> int:
def turn_capped(res: Any) -> bool:
"""True when the agent stopped because it ran out of turns (not money/time).
- api-agent says ``max_turns (N) reached``; claude-code reports the subtype
- ``error_max_turns`` — both surface as ``exit_reason="budget"``."""
+ claude-code reports the subtype ``error_max_turns``; it surfaces as
+ ``exit_reason="budget"``."""
if getattr(res, "exit_reason", "") != "budget":
return False
return any("max_turns" in str(e) for e in (getattr(res, "errors", None) or []))
@@ -550,8 +550,7 @@ def session_turns(res: Any) -> int:
``agents/cli_common.finish_session`` writes into ``result.json`` (0 when the
backend does not report one).
- Backends count slightly differently (the api-agent reports transcript
- messages, roughly two per model turn), so this is a size signal for the
+ Backends count turns slightly differently, so this is a size signal for the
``cost.round`` event, not the number the turn cap is compared against —
that one is enforced inside the session by ``job.max_turns``."""
path = getattr(res, "transcript_path", "") or ""
diff --git a/harness/codeverse/tracks/planner.py b/harness/codeverse/tracks/planner.py
index a28c1246..8c2e619c 100644
--- a/harness/codeverse/tracks/planner.py
+++ b/harness/codeverse/tracks/planner.py
@@ -61,11 +61,11 @@ PLAN_TOKENS_PER_LEAF = 400
PLAN_TOKENS_MAX = 65_536 # the model's declared output ceiling
#: one more attempt when the model truncates anyway, with half again as much room
TRUNCATION_GROWTH = 1.5
-#: retry budget (``ChatRequest.max_wait_s``) for one planner call. Audit 2026-08-26 §2: the
-#: call itself is 13.7 s p50 / 32 s p90 / 76 s max (23.4 / 43 / 68 under the storm), yet the
-#: storm-day plan stage waited 492 s median per run for 39 s of model time; 300 s is ~4x the
-#: worst observed call and replaces the model's 900 s default. The re-asks are on top.
-PLAN_MAX_WAIT_S = 900.0 # owner's floor 2026-08-27: 15 minutes before a plan may be cut off
+#: retry budget (``ChatRequest.max_wait_s``) for one planner call — the owner's floor
+#: (2026-08-27): 15 minutes before a plan may be cut off. For scale, the call itself is
+#: 13.7 s p50 / 32 s p90 / 76 s max (audit 2026-08-26 §2); the flat 300 s that once stood
+#: here is what killed grown re-asks (see the deadline-vs-answer block below).
+PLAN_MAX_WAIT_S = 900.0
#: ...but a deadline must fit the ANSWER it asked for, or the tokens are billed and then
#: thrown away on a timeout. Measured over 395 real planner calls (2026-08-27): output
#: runs at 145 tok/s p50 and 60 tok/s p10, and the largest answers seen were 32 k tokens
diff --git a/harness/tests/core/test_contract_additions.py b/harness/tests/core/test_contract_additions.py
index 467e2f69..bb552e71 100644
--- a/harness/tests/core/test_contract_additions.py
+++ b/harness/tests/core/test_contract_additions.py
@@ -11,7 +11,6 @@ from codeverse.contracts import (
TRACK_INFO,
TRACK_LANGUAGES,
AgentJob,
- ApiAgentOptions,
Language,
RunOptions,
Spec,
@@ -102,26 +101,22 @@ def test_render_view_judge_flag_defaults_none():
def test_agent_job_legacy_extra_is_lifted_and_kept():
extra = {"round": 2, "kind": "refine", "language": "blender", "track": "static_object",
"files_hint": ["src/a.py"], "mcp_command": ["python", "-m", "x"],
- "max_usd": 1.5, "temperature": 0.7, "thinking": "high", "allow_shell": False}
+ "temperature": 0.7} # unknown keys (like the old api-agent knobs) just stay put
j = AgentJob(workspace="w", prompt="p", extra=dict(extra))
assert (j.round, j.kind, j.language, j.track) == (2, "refine", "blender", "static_object")
assert j.files_hint == ["src/a.py"]
assert j.mcp_command == ["python", "-m", "x"]
- assert j.api == ApiAgentOptions(max_usd=1.5, temperature=0.7, thinking="high", allow_shell=False)
assert j.extra == extra # untouched: legacy readers see what they were given
def test_agent_job_explicit_fields_beat_extra():
- j = AgentJob(workspace="w", prompt="p", round=5, api=ApiAgentOptions(max_usd=9.0),
- extra={"round": 2, "max_usd": 1.0})
+ j = AgentJob(workspace="w", prompt="p", round=5, extra={"round": 2})
assert j.round == 5
- assert j.api.max_usd == 9.0
def test_agent_job_defaults_and_roundtrip():
j = AgentJob(workspace="w", prompt="p")
assert j.round == 0 and j.kind == "" and j.files_hint == [] and j.mcp_command is None
- assert j.api == ApiAgentOptions()
j2 = AgentJob(workspace="w", prompt="p", extra={"round": 3, "temperature": 0.9})
assert AgentJob(**j2.model_dump()) == j2 # dump/construct stable
diff --git a/harness/tests/skills/test_delivery.py b/harness/tests/skills/test_delivery.py
index 26476816..d6eab11c 100644
--- a/harness/tests/skills/test_delivery.py
+++ b/harness/tests/skills/test_delivery.py
@@ -22,24 +22,24 @@ def test_a_native_loader_is_never_handed_a_second_index():
"""It lists the bundles itself; ours on top would double-index the same skills."""
for kind in ("claude-code", "codex", "gemini-cli", "agy"):
d = delivery_for(kind)
- assert d.native_loader and not d.needs_index and not d.needs_tool, kind
+ assert d.native_loader and not d.needs_index, kind
-def test_a_loaderless_backend_gets_both_the_index_and_the_tool():
+def test_a_loaderless_backend_gets_the_index():
"""Measured 2026-08-25: the native loaders read 5 of 5 routed bundles while a
loaderless backend read 0 of 5 from a MANDATORY paragraph, making 52 read_file calls
instead. Prose is not an affordance. Every SHIPPED backend has a native loader
since the in-process one was deleted (2026-08-28), so this is the policy for a
backend nobody has classified."""
d = delivery_for("some-future-cli")
- assert not d.native_loader and d.needs_index and d.needs_tool
+ assert not d.native_loader and d.needs_index
def test_an_unclassified_backend_gets_the_safe_answer():
"""Over-delivering costs tokens; under-delivering costs the skill entirely."""
for kind in ("some-future-cli", "", "typo-agent"):
d = delivery_for(kind)
- assert d.needs_index and d.needs_tool, kind
+ assert d.needs_index, kind
assert d.root == AGENTS_SKILL_ROOT
@@ -83,11 +83,3 @@ def test_adding_a_backend_is_one_row():
offenders.append(py.name)
assert not offenders, f"per-backend knowledge leaked out of delivery.py into {offenders}"
-
-# ------------------------------------------------------------------ the read_skill tool
-
-
-# The in-process skill-reading TOOL (agents/api_skills.py) went with the api-agent on
-# 2026-08-28: every remaining backend is a vendor CLI with a native loader, so a bundle
-# is delivered as files and read by the vendor's own mechanism. The four tests that
-# drove that tool are gone with it; delivery_for() above still pins the routing policy.
diff --git a/harness/tests/skills/test_telemetry.py b/harness/tests/skills/test_telemetry.py
index d52ae466..a6077c21 100644
--- a/harness/tests/skills/test_telemetry.py
+++ b/harness/tests/skills/test_telemetry.py
@@ -1,8 +1,8 @@
"""The read probe — the measurement that replaces "0 of 16 read_cookbook calls".
If these tests are wrong, every later conclusion about whether skills work is wrong, so
-they pin the semantics explicitly: surfaced != deep, a bundle with no references cannot
-be probed for depth and says so, and api-agent's exact log wins over the probe.
+they pin the semantics explicitly: surfaced != deep, and a bundle with no references
+cannot be probed for depth and says so.
"""
from __future__ import annotations
@@ -15,14 +15,11 @@ import pytest
from codeverse.skills.materialize import attach_skills
from codeverse.skills.telemetry import (
- READS_FILE,
SKILLS_FILE,
TELEMETRY_DIR,
append_usage,
deep_read_rate,
- is_skill_path,
probe_reads,
- record_exact_read,
)
from tests.skills.conftest import write_bundle
@@ -96,39 +93,11 @@ def test_a_bundle_with_no_references_says_depth_is_unmeasurable(tmp_path, ws):
assert read.deep_measurable is False and read.deep is False
-def test_api_agent_exact_reads_override_the_probe_and_carry_the_turn(ws, library):
- got = _attach(ws, library)
- name = got.listed[0]
- record_exact_read(ws, f".agents/skills/{name}/SKILL.md", turn=3, label="baseline")
- record_exact_read(ws, f".agents/skills/{name}/references/worked_example.md", turn=5, label="baseline")
- record_exact_read(ws, "src/model.py", turn=1) # not a skill: ignored
- usage = probe_reads(ws, got)
- row = next(r for r in usage.reads if r.name == name)
- assert row.surfaced and row.deep and row.first_seen_turn == 3
- lines = (ws / TELEMETRY_DIR / READS_FILE).read_text().splitlines()
- assert len(lines) == 2 and json.loads(lines[0])["skill"] == name
-
-
-def test_is_skill_path_recognises_both_roots_and_nothing_else():
- assert is_skill_path(".agents/skills/cv3d-x/SKILL.md") == "cv3d-x"
- assert is_skill_path(".claude/skills/cv3d-x/references/a.md") == "cv3d-x"
- assert is_skill_path("./.agents/skills/cv3d-x/SKILL.md") == "cv3d-x"
- assert is_skill_path("src/model.py") == ""
- assert is_skill_path(".agents/settings.json") == ""
-
-
-def test_a_damaged_reads_file_does_not_break_the_probe(ws, library):
- got = _attach(ws, library)
- (ws / TELEMETRY_DIR).mkdir(exist_ok=True)
- (ws / TELEMETRY_DIR / READS_FILE).write_text("not json\n{}\n")
- assert probe_reads(ws, got).surfaced == []
-
-
def test_usage_is_appended_as_one_json_line_per_session(ws, library):
got = _attach(ws, library)
usage = probe_reads(ws, got)
- append_usage(ws, usage, round=0, kind="baseline", agent="api-agent:gemini-3.7-flash")
- append_usage(ws, usage, round=1, kind="refine", agent="api-agent:gemini-3.7-flash")
+ append_usage(ws, usage, round=0, kind="baseline", agent="gemini-cli:gemini-3.6-flash")
+ append_usage(ws, usage, round=1, kind="refine", agent="gemini-cli:gemini-3.6-flash")
rows = [json.loads(x) for x in (ws / TELEMETRY_DIR / SKILLS_FILE).read_text().splitlines()]
assert [r["round"] for r in rows] == [0, 1]
assert rows[0]["listed"] == usage.listed and "index_tokens" in rows[0]
@@ -147,7 +116,6 @@ def test_deep_read_rate_aggregates_across_sessions(ws, library):
def test_telemetry_never_raises_into_a_run(tmp_path):
from codeverse.skills.model import SkillsMaterialized
- record_exact_read(tmp_path / "does" / "not" / "exist", ".agents/skills/x/SKILL.md", turn=1)
assert append_usage(tmp_path / "nope" / "nope", probe_reads(tmp_path, SkillsMaterialized())) is None or True