Skip to content

Experience-Driven Black-Box Cost Control for Coding Agents

6.1 KB
Raw Blame
"""Facade: the Economizer drives all three mechanisms end-to-end (offline)."""
import os

from te_bench.facade import Economizer
from tokeneconomizer import EconomizerConfig
from tokeneconomizer.config import InjectionConfig, ToolingConfig, SpiralConfig
from te_bench.adapters import SwebenchAdapter
from fixtures import write_claude_transcript, write_codex_transcript


def _runner(prompt, model="agent"):
    return ('{"extractors":[],"facts":[]}', 5.0)


def test_generic_trimech_end_to_end(tmp_path):
    cfg = EconomizerConfig(
        injection=InjectionConfig(enabled=True, propose_every=5),
        tooling=ToolingConfig(evidence=3, judge=False),
        spiral=SpiralConfig(guard=True, nudge=True))
    econ = Economizer(cfg, agent="claude", repo="acme/widget",
                      results_root=str(tmp_path), fresh=True, runner=_runner)
    task = {"instance_id": "w-1", "problem_statement": "fix"}
    assert econ.injection_prefix(task) == ""       # nothing formed yet
    assert econ.build_tools(task, 0) == []          # no catalog learned yet
    rk = econ.run_kwargs(task)
    assert rk["spiral_guard"] is True

    rd = str(tmp_path / "run"); os.makedirs(rd)
    write_claude_transcript(os.path.join(rd, "transcript.jsonl"),
                            [("bash", {"command": "pytest"}, "ok")])
    row = {"resolved": True, "num_turns": 10, "tokens_total": 1000,
           "input_tokens": 100, "cache_read_input_tokens": 800,
           "cache_creation_input_tokens": 50, "output_tokens": 50}
    srow = {"resolved": True, "num_turns": 10}
    opt, reason = econ.after_task(task, row, rd, srow)
    assert reason == "manual-distill"
    assert "manual_facts" in srow and "aborted" in srow


def test_codex_arm_end_to_end(tmp_path):
    cfg = EconomizerConfig(
        injection=InjectionConfig(enabled=False),
        tooling=ToolingConfig(codex_untool=True),
        spiral=SpiralConfig(codex_spiral=True))
    econ = Economizer(cfg, agent="codex", repo="pydata/xarray",
                      results_root=str(tmp_path), fresh=True)
    task = {"instance_id": "x-1", "problem_statement": "fix concat"}
    assert econ.build_tools(task, 0) == []          # task 1 full contract
    rk = econ.run_kwargs(task)
    assert "codex_spiral_policy" in rk

    rd = str(tmp_path / "run"); os.makedirs(rd)
    write_codex_transcript(os.path.join(rd, "transcript.jsonl"),
                           [{"type": "command_execution", "command": "pytest"}])
    row = {"resolved": True, "num_turns": 30, "tokens_total": 5000,
           "input_tokens": 500, "cache_read_input_tokens": 4000,
           "cache_creation_input_tokens": 200, "output_tokens": 300,
           "codex_spiral_nudges": 0}
    srow = {"resolved": True, "num_turns": 30}
    opt, reason = econ.after_task(task, row, rd, srow)
    assert reason is None                            # injection disabled
    assert srow["codex_untool_active"] is True
    assert srow["codex_spiral_active"] is True
    # task 2: web should have been demoted (loaded-and-unused on task 1)
    assert "web" in econ.build_tools(task, 1)


def test_codex_trimech_end_to_end(tmp_path):
    """All three Codex mechanisms driven through the facade at once, offline.

    Built from the REGISTRY rather than a hand-written config, so this is the arm a lane
    actually runs. What it guards is that the three do not contend: injection renders a
    prompt prefix, untooling freezes a capability mask, and the spiral stage hands the
    runner its policy -- on the same task, in one `run_kwargs`, with all three telemetry
    families landing on one stream row.
    """
    from te_bench.arms import ARMS
    calls = []

    def runner(prompt, model="agent"):
        calls.append(prompt)
        return '{"extractors":[],"facts":[]}', 3.0

    econ = Economizer(ARMS["codex-trimech"].cfg(), agent="codex",
                      repo="pydata/xarray", results_root=str(tmp_path), fresh=True,
                      runner=runner, adapter=SwebenchAdapter())
    task = {"instance_id": "x-1", "problem_statement": "fix concat"}

    # one run_kwargs carries BOTH codex mechanisms' runner arguments, and they do not
    # collide -- the capability mask and the spiral policy are separate keys.
    rk = econ.run_kwargs(task)
    assert "codex_config_capabilities" in rk and "codex_spiral_policy" in rk
    assert econ.build_tools(task, 0) == []          # task 1 runs the full contract

    # ten tasks, because the composed profile proposes at cadence 10 -- fewer would leave
    # the paid half of the arm untested.
    total = 0
    for i in range(10):
        task = {"instance_id": f"x-{i}", "problem_statement": "fix concat"}
        # the per-task order the driver uses: prefix -> mask -> run_kwargs -> run -> hook
        econ.injection_prefix(task)
        econ.build_tools(task, i)
        econ.run_kwargs(task)
        rd = str(tmp_path / f"run{i}"); os.makedirs(rd)
        write_codex_transcript(os.path.join(rd, "transcript.jsonl"), [
            {"type": "file_change", "changes": [{"path": "xarray/core/concat.py",
                                                 "kind": "update"}]},
            {"type": "command_execution",
             "command": '/bin/bash -lc "python -m pytest xarray/tests/test_concat.py -q"',
             "aggregated_output": "1 passed", "exit_code": 0}])
        row = {"resolved": True, "num_turns": 30, "tokens_total": 5000,
               "input_tokens": 500, "cache_read_input_tokens": 4000,
               "cache_creation_input_tokens": 200, "output_tokens": 300,
               "codex_spiral_nudges": 0, "codex_spiral_tool_turns": 12}
        srow = {"resolved": True, "num_turns": 30}
        opt, reason = econ.after_task(task, row, rd, srow)
        total += opt
        # all three mechanisms reported on the same row
        assert srow["codex_untool_active"] is True
        assert srow["codex_spiral_active"] is True
        assert "manual_facts" in srow
        assert reason and reason.startswith("envfacts")      # the composed profile

    # the proposer's spend crossed the ledger seam rather than being invisible
    assert calls and total == 3.0 * len(calls)
    # untooling still learns across tasks with the other two active
    assert "web" in econ.build_tools(task, 10)