Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
"""The canonical index must describe the whole tree, or refuse to be written.
`aggregate.py --arms` limits the REWRITE, not just the reading: both CSVs are written
whole on every invocation. Naming a subset while writing to `<root>/index` would therefore
replace an index describing the tree with one describing a subset, at the path
`compare_arms.py` reads, under a name that says nothing about being partial -- with every
cell still on disk. The Codex lane runs one arm per invocation, which makes an `--arms`
an easy thing to pass, so the subset write to the canonical path is refused.
These tests are offline: they build a tiny tree of synthetic `row.json` files and run
`main()` against it.
"""
import importlib.util
import json
import os
import sys
import pytest
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_AGGREGATE = os.path.join(_REPO, "scripts", "aggregate.py")
def _agg():
spec = importlib.util.spec_from_file_location("_aggregate", _AGGREGATE)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
def _row(root, arm, cell, stamp, instance, **kw):
d = os.path.join(root, arm, cell, f"{stamp}__{instance}__{arm}__standalone")
os.makedirs(d, exist_ok=True)
row = {"instance": instance, "repo": "sympy/sympy", "seed": 0, "arm": arm,
"resolved": True, "num_turns": 10, "tokens_total": 1000,
"input_tokens": 10, "output_tokens": 100,
"cache_read_input_tokens": 800, "cache_creation_input_tokens": 90,
"model": "gpt-5.6-terra", "effort": "high", "agent": "codex",
"run_dir": d}
row.update(kw)
with open(os.path.join(d, "row.json"), "w") as f:
json.dump(row, f)
return d
@pytest.fixture
def tree(tmp_path):
"""Two arms, one cell each -- the shape the refusal is about."""
root = str(tmp_path / "codex_swebench_gpt56terra")
_row(root, "baseline", "sympy_s0", "20260916T000000Z", "sympy-11618")
_row(root, "baseline", "sympy_s0", "20260916T000100Z", "sympy-12096")
_row(root, "repomap", "sympy_s0", "20260916T010000Z", "sympy-11618")
return root
def _run(argv):
mod = _agg()
old = sys.argv
sys.argv = ["aggregate.py"] + argv
try:
mod.main()
finally:
sys.argv = old
def _cells(index_dir):
import csv
with open(os.path.join(index_dir, "summary_by_cell.csv")) as f:
return [(r["arm"], r["repo"], r["n_tasks"]) for r in csv.DictReader(f)]
def test_whole_tree_is_the_default(tree):
_run([tree])
assert _cells(os.path.join(tree, "index")) == [
("baseline", "sympy", "2"), ("repomap", "sympy", "1")]
def test_subset_rewrite_of_the_canonical_index_is_refused(tree):
"""`--arms repomap` must not leave the canonical index holding only repomap."""
_run([tree]) # a good index exists
before = _cells(os.path.join(tree, "index"))
with pytest.raises(SystemExit) as e:
_run([tree, "--arms", "repomap"])
assert e.value.code != 0
# and, the point of refusing rather than warning: the good index is untouched.
assert _cells(os.path.join(tree, "index")) == before
def test_subset_is_allowed_somewhere_of_its_own(tree, tmp_path):
_run([tree])
before = _cells(os.path.join(tree, "index"))
out = str(tmp_path / "repomap_only")
_run([tree, "--arms", "repomap", "--out", out])
assert _cells(out) == [("repomap", "sympy", "1")]
assert _cells(os.path.join(tree, "index")) == before
def test_naming_every_arm_is_not_a_subset(tree):
"""`--arms` that happens to list the whole tree is a no-op, not an error."""
_run([tree, "--arms", "baseline", "repomap"])
assert _cells(os.path.join(tree, "index")) == [
("baseline", "sympy", "2"), ("repomap", "sympy", "1")]
def test_superseded_run_dirs_are_not_double_counted(tree):
"""The re-run case the pilot cells are in: newest attempt wins, older is dropped.
Already the behaviour of `collect()`; pinned here because the launcher passes
`--fresh`, which starts a new stream in the SAME cell directory, so a re-run cell
always holds both attempts on disk.
"""
_row(tree, "repomap", "sympy_s0", "20260916T020000Z", "sympy-11618", num_turns=99)
_run([tree])
assert _cells(os.path.join(tree, "index")) == [
("baseline", "sympy", "2"), ("repomap", "sympy", "1")]
import csv
with open(os.path.join(tree, "index", "all_rows.csv")) as f:
kept = [r for r in csv.DictReader(f) if r["arm"] == "repomap"]
assert [r["num_turns"] for r in kept] == ["99"]