Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
"""The shared environment-fact layer: vocabulary plug-in, composition with the paid
proposer, and the arms that carry each profile."""
import os
from tokeneconomizer.microskills import envfacts as ef, observe
from tokeneconomizer.microskills import ComposedDistiller, PrefixStage
from te_bench import mechanism_arms as ci
from tokeneconomizer.config import InjectionConfig, needs_optimizer
from te_bench.adapters.swebench import SwebenchAdapter
from fixtures import write_codex_transcript
BASE = "/opt/env/base/bin/python"
PROJ = "/opt/env/proj/bin/python"
def _cmd(command, output, code):
return {"type": "command_execution", "command": f'/bin/bash -lc "{command}"',
"aggregated_output": output, "exit_code": code}
def _edit(path):
return {"type": "file_change", "changes": [{"path": path, "kind": "update"}]}
def _broken(tmp_path, i):
p = str(tmp_path / f"t{i}.jsonl")
write_codex_transcript(p, [
_edit(f"pkg/mod{i}.py"),
_cmd("python -m pytest tests/test_a.py -q", f"{BASE}: No module named pytest", 1),
_cmd(f"{PROJ} -m pytest tests/test_a.py -q", "1 passed", 0)])
return p
# ---- the Codex lane's injection profiles ---------------------------------------------
def test_codex_injection_profiles():
assert ci.CODEX_DEFAULT_PROFILE == "envfacts+distiller"
assert ci.CODEX_FREE_PROFILE == "envfacts"
# ---- vocabulary plug-in ----------------------------------------------------------------
def test_node_vocabulary_reads_a_node_repository():
obs = observe.classify_command("npx jest tests/ --ci", vocab=observe.NODE_WEB)
assert [(o.intent, o.form) for o in obs] == [("run_tests", "npx jest")]
obs = observe.classify_command("node scripts/build.js && npm test", vocab=observe.NODE_WEB)
assert [(o.intent, o.form) for o in obs] == [("run_interpreter", "node"),
("run_tests", "npm test")]
obs = observe.classify_command("npm install left-pad", vocab=observe.NODE_WEB)
assert obs[0].intent == "fetch_remote"
# the Python vocabulary does not know jest; the Node one does not know pytest
assert observe.classify_command("npx jest", vocab=observe.PYTHON_GIT) == [] or \
observe.classify_command("npx jest", vocab=observe.PYTHON_GIT)[0].intent != "run_tests"
assert observe.classify_command("pytest -q", vocab=observe.NODE_WEB) == []
assert observe.error_class("Error: Cannot find module 'left-pad'") == "module_missing"
assert observe.error_class("connect ECONNREFUSED 127.0.0.1:443") == "connection_refused"
def test_adapters_supply_a_vocabulary():
assert SwebenchAdapter().env_vocabulary().name == "python-git"
from te_bench.adapters.webbench import WebBenchAdapter
assert WebBenchAdapter.env_vocabulary(None).name == "node-web"
def test_distiller_records_its_vocabulary(tmp_path):
d = ef.EnvFactsDistiller("acme/widget", vocab=observe.NODE_WEB)
d.update(_broken(tmp_path, 0), iid="w-0")
d.save(str(tmp_path))
import json
saved = json.load(open(os.path.join(str(tmp_path), ef.STATE_SUBDIR, "acme__widget.json")))
assert saved["vocab"] == "node-web"
assert d.debug_row()["_vocab"] == "node-web"
# ---- composition with the paid proposer -----------------------------------------------
class _RecordingRunner:
"""A proposer stand-in: records every prompt, authors nothing, bills 7 eff tokens."""
def __init__(self):
self.prompts = []
def __call__(self, prompt, model="agent"):
self.prompts.append(prompt)
return '{"extractors": [], "facts": []}', 7.0
def test_composed_profile_renders_both_layers_and_briefs_the_proposer(tmp_path):
runner = _RecordingRunner()
cfg = InjectionConfig(enabled=True, profile="envfacts+distiller", propose_every=1)
stage = PrefixStage(cfg, "acme/widget", str(tmp_path), fresh=True, runner=runner,
agent="codex", adapter=SwebenchAdapter())
assert isinstance(stage.distiller, ComposedDistiller)
total_opt = 0
for i in range(6):
block = stage.render()
rd = str(tmp_path / f"run{i}"); os.makedirs(rd)
os.replace(_broken(tmp_path, i), os.path.join(rd, "transcript.jsonl"))
opt, reason = stage.after_task({"instance_id": f"w-{i}"}, {}, rd, {"resolved": True})
total_opt += opt
assert reason == "envfacts+manual-distill"
# the free layer formed and renders first
assert block.startswith(ef.HEADER) and PROJ in block
# the proposer was called every task (cadence 1) and billed through the ledger seam
assert len(runner.prompts) >= 5 and total_opt == 7.0 * len(runner.prompts)
# once the environment fact was injectable, the proposer was told not to restate it
assert any("ALREADY injected" in p and "fails here" in p for p in runner.prompts[3:])
# telemetry names the profile and carries both layers' facts
srow = {"resolved": True}
rd = str(tmp_path / "run_last"); os.makedirs(rd)
os.replace(_broken(tmp_path, 9), os.path.join(rd, "transcript.jsonl"))
stage.after_task({"instance_id": "w-9"}, {}, rd, srow)
assert srow["manual_facts"]["_profile"] == "envfacts+distiller"
assert any(k.startswith("form_repair:") for k in srow["manual_facts"])
def test_envfacts_alone_still_needs_no_runner(tmp_path):
cfg = InjectionConfig(enabled=True, profile="envfacts")
stage = PrefixStage(cfg, "acme/widget", str(tmp_path), fresh=True, runner=None,
agent="codex", adapter=SwebenchAdapter())
assert stage.render() == "" and stage.reason == "envfacts"
# ---- arms ------------------------------------------------------------------------------
def test_codex_arms_carry_the_intended_profiles():
"""Since 2026-09-12 the Codex injection DEFAULT is the composed profile.
With the project environment active on every lane the free layer mostly abstains, so
`codex-inject` carries the paid proposer and the LLM-free variant is the named ablation
`codex-inject-envfacts`.
"""
from te_bench.arms import ARMS, ARM_ORDER
from te_bench import mechanism_arms as ci
default = ARMS["codex-inject"].cfg().injection
assert default.profile == ci.CODEX_DEFAULT_PROFILE == "envfacts+distiller"
assert default.paid and default.propose_every == 10
assert needs_optimizer(ARMS["codex-inject"].cfg()) is True
# the untool stack follows the default; the ablation does not
assert ARMS["codex-inject-untool"].cfg().injection.profile == "envfacts+distiller"
assert ARMS["codex-inject-untool"].cfg().tooling.codex_untool is True
free = ARMS["codex-inject-envfacts"].cfg().injection
assert free.profile == ci.CODEX_FREE_PROFILE == "envfacts" and not free.paid
assert needs_optimizer(ARMS["codex-inject-envfacts"].cfg()) is False
# the pre-2026-09-12 name of the default still resolves to the same configuration
assert ARMS["codex-inject-distiller"].cfg().injection == default
for name in ("codex-inject", "codex-inject-untool", "codex-inject-envfacts",
"codex-inject-distiller"):
assert ARMS[name].mode == "manual4" and name in ARM_ORDER