Experience-Driven Black-Box Cost Control for Coding Agents
Python agentic-aiagentic-codingai-codingai-coding-assistantanthropicartificial-intelligenceclaude-codecodexcoding-agentcoding-agentscost-optimizationcost-reductiondeveloper-toolsgithub-copilotllmopen-sourceopenaiproductivitytoken-optimization
CI: install and run the plugin under a real Codex CLI, on Windows too
Aaron801df4a2c256
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 82917d5..6cc24b8 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -39,3 +39,46 @@ jobs:}env:COLUMNS: "200"++ codex:+ # The plugin installed and run by a real Codex CLI (tests/test_codex_cli.py): its install+ # copy finds the package through the `source` Codex writes, which Codex writes differently on+ # Windows, and a `codex exec` session runs its hooks. The model is a local stand-in, so the+ # job needs no key and makes no model call.+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, windows-latest]+ runs-on: ${{ matrix.os }}+ defaults:+ run:+ shell: bash+ steps:+ - uses: actions/checkout@v5+ - uses: actions/setup-python@v6+ with:+ python-version: "3.13"+ - uses: actions/setup-node@v5+ with:+ node-version: "22"+ # Pinned: a new Codex can change how it installs a plugin; move it on purpose.+ - run: npm install -g @openai/[email protected]+ - run: python -m pip install pytest+ # What Codex wrote for the marketplace goes out as a notice, and on failure the summary+ # lines as an annotation, both shown on the check run without signing in.+ - run: |+ set -o pipefail+ status=0+ python -m pytest tests/test_codex_cli.py -rfEsP --tb=short 2>&1 | tee pytest.log \+ || status=$?+ grep -h " wrote: " pytest.log | sort -u | sed 's/^/::notice title=codex config.toml::/' \+ || true+ if [ "$status" -ne 0 ]; then+ msg=$(grep -E '^(FAILED|ERROR) |\.py:[0-9]+: ' pytest.log | head -60 \+ | sed 's/%/%25/g' | awk '{printf "%s%%0A", $0}')+ echo "::error title=pytest::${msg}"+ fi+ exit $status+ env:+ TE_CODEX_E2E: "1"+ COLUMNS: "200"diff --git a/tests/test_codex_cli.py b/tests/test_codex_cli.pynew file mode 100644index 0000000..d99f820--- /dev/null+++ b/tests/test_codex_cli.py@@ -0,0 +1,168 @@+"""The plugin installed and run by a real Codex CLI: CI's `codex` job (`TE_CODEX_E2E=1`).++Codex installs a COPY of plugin/ without the package (its `lib` link is dropped, or on a Windows+checkout is a plain file), so the copy finds the package through the marketplace `source` Codex+writes in its config.toml: a TOML literal string with the extended-length prefix on Windows+(0.163.0). These install the plugin as the README says, then run a `codex exec` session against+a local stand-in for the model that gives one canned answer (no key, no model call), and check+that the hooks ran and the session was learned from.+"""+import glob+import json+import os+import shutil+import subprocess+import sys+import threading+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import pytest++REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+PLUGIN = os.path.join(REPO, "plugin")+CODEX = shutil.which("codex")+SH = ["sh"] if os.name == "nt" else []++pytestmark = pytest.mark.skipif(+ os.environ.get("TE_CODEX_E2E") != "1" or not CODEX,+ reason="drives a real Codex CLI: set TE_CODEX_E2E=1 with `codex` on PATH (CI's codex job)")+++def _env(tmp_path):+ """Codex and the plugin each in their own home under the test, and no other way for the+ install copy to find the package."""+ return dict(os.environ, CODEX_HOME=str(tmp_path / "codex"),+ TOKENECONOMIZER_HOME=str(tmp_path / "te"), TE_CORE_PATH="", PYTHONPATH="")+++def _codex(env, *args, cwd=None, timeout=120):+ out = subprocess.run([CODEX, *args], capture_output=True, text=True, encoding="utf-8",+ errors="replace", stdin=subprocess.DEVNULL, cwd=cwd, env=env,+ timeout=timeout)+ assert out.returncode == 0, (args, out.stdout[-2000:], out.stderr[-2000:])+ return out+++def _install(env):+ """README's two commands; returns Codex's install copy and the `source` line it wrote."""+ os.makedirs(env["CODEX_HOME"], exist_ok=True)+ _codex(env, "plugin", "marketplace", "add", REPO)+ _codex(env, "plugin", "add", "tokeneconomizer@tokeneconomizer")+ with open(os.path.join(PLUGIN, ".codex-plugin", "plugin.json"), encoding="utf-8") as fh:+ version = json.load(fh)["version"]+ copy = os.path.join(env["CODEX_HOME"], "plugins", "cache", "tokeneconomizer",+ "tokeneconomizer", version)+ with open(os.path.join(env["CODEX_HOME"], "config.toml"), encoding="utf-8") as fh:+ source = [ln for ln in fh.read().splitlines() if ln.strip().startswith("source =")]+ print(f"{_codex(env, '--version').stdout.strip()} wrote: {' | '.join(source)}")+ return copy, source+++def test_codex_installs_the_plugin_and_its_copy_finds_the_package(tmp_path):+ env = _env(tmp_path)+ copy, source = _install(env)+ assert os.path.isfile(os.path.join(copy, "scripts", "te-hook"))+ assert not os.path.isdir(os.path.join(copy, "lib", "tokeneconomizer")), \+ "the copy carries the package: the marketplace lookup below is not what finds it"+ sys.path.insert(0, os.path.join(PLUGIN, "scripts"))+ try:+ import te_bootstrap+ finally:+ sys.path.pop(0)+ found = [p for p in te_bootstrap._marketplace_sources(copy)+ if os.path.isdir(p) and os.path.samefile(p, REPO)]+ assert found, (source, te_bootstrap._marketplace_sources(copy))+ out = subprocess.run(SH + [os.path.join(copy, "scripts", "te"), "doctor"],+ capture_output=True, text=True, timeout=60, cwd=str(tmp_path), env=env)+ assert out.returncode == 0 and "python" in out.stdout, (source, out.stderr[-1000:])+++class _Model(BaseHTTPRequestHandler):+ """The Responses API, as far as Codex reads it for a turn with no tool call: every request+ is answered with one assistant message."""+ requests = []++ def log_message(self, *args):+ pass++ def do_GET(self): # model list and the like: none here+ self.requests.append(("GET", self.path))+ self.send_response(404)+ self.send_header("Content-Length", "0")+ self.end_headers()++ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length") or 0))+ self.requests.append(("POST", self.path))+ rid = f"resp_{len(self.requests)}"++ def event(kind, **fields):+ return f"event: {kind}\ndata: {json.dumps(dict(type=kind, **fields))}\n\n"+ body = (event("response.created", response={"id": rid})+ + event("response.output_item.done", item={+ "type": "message", "role": "assistant", "id": f"msg_{rid}",+ "content": [{"type": "output_text", "text": "hi"}]})+ + event("response.completed", response={"id": rid, "usage": {+ "input_tokens": 10, "input_tokens_details": None, "output_tokens": 1,+ "output_tokens_details": None, "total_tokens": 11}})).encode()+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)+++def model():+ _Model.requests = []+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Model)+ threading.Thread(target=server.serve_forever, daemon=True).start()+ yield f"http://127.0.0.1:{server.server_address[1]}/v1"+ server.shutdown()+ server.server_close()+++def test_a_codex_session_runs_the_hooks_and_is_learned_from(tmp_path, model):+ env = _env(tmp_path)+ _, source = _install(env)+ config = os.path.join(env["CODEX_HOME"], "config.toml")+ with open(config, encoding="utf-8") as fh:+ tables = fh.read()+ with open(config, "w", encoding="utf-8") as fh: # top-level keys go before any table+ fh.write('model = "te-test-model"\nmodel_provider = "te_test"\n\n'+ f'[model_providers.te_test]\nname = "te-test"\nbase_url = "{model}"\n'+ 'wire_api = "responses"\nsupports_websockets = false\n'+ "request_max_retries = 0\nstream_max_retries = 0\n\n" + tables)+ proj = tmp_path / "proj"+ proj.mkdir()+ (proj / "a.txt").write_text("x\n")+ subprocess.run(["git", "init", "-q", str(proj)], check=True)+ # Persisted hook trust is the person's "Trust all and continue"; CI runs this checkout's own+ # hooks, which is what the flag is for.+ out = _codex(env, "exec", "--dangerously-bypass-hook-trust", "--skip-git-repo-check",+ "Say hi.", cwd=str(proj), timeout=180)+ assert ("POST", "/v1/responses") in _Model.requests, (out.stdout[-1000:], out.stderr[-2000:])++ root = env["TOKENECONOMIZER_HOME"]+ errors = os.path.join(root, "errors.log")+ logged = open(errors, encoding="utf-8").read() if os.path.exists(errors) else ""+ sessions = glob.glob(os.path.join(root, "repos", "*", "codex", "sessions", "*.json"))+ assert sessions, ("no hook recorded the session", source, logged[-2000:],+ out.stderr[-2000:])+ assert not logged, logged[-2000:]+ # SessionEnd hands the session to a detached `te fold` (on Windows, `winspawn`).+ sid = os.path.basename(sessions[0])[:-len(".json")]+ store = os.path.dirname(os.path.dirname(sessions[0]))+ deadline = time.time() + 120+ folded = {}+ while time.time() < deadline and sid not in folded:+ try:+ with open(os.path.join(store, "folded.json"), encoding="utf-8") as fh:+ folded = json.load(fh)+ except (OSError, ValueError):+ pass+ time.sleep(1)+ fold_log = os.path.join(store, "fold.log")+ assert sid in folded, (open(fold_log, encoding="utf-8").read()[-2000:]+ if os.path.exists(fold_log) else "no fold.log")