Skip to content

Experience-Driven Black-Box Cost Control for Coding Agents

66.2 KB
Raw Blame
"""The live layer's plumbing: repository identity, state, config, session logs, host I/O.

Offline and synthetic. The transcript fixtures reproduce the two properties of real session logs
that the harness's own transcripts never had, and that the live readers exist to handle: Claude
Code writes one entry PER CONTENT BLOCK, each repeating its message's usage, and Codex's rollout
wraps exec-stream item fields in PascalCase `event_msg/item_completed` records.
"""
import json
import os
import subprocess

import pytest

from tokeneconomizer import trace
from tokeneconomizer.plugin import config as live_config
from tokeneconomizer.plugin import hosts, paths
from tokeneconomizer.plugin.store import Store


# ---- repository identity ---------------------------------------------------------------
def _git_repo(root, remote=None):
    os.makedirs(os.path.join(root, ".git"), exist_ok=True)
    with open(os.path.join(root, ".git", "HEAD"), "w") as fh:
        fh.write("ref: refs/heads/main\n")
    cfg = "[core]\n\trepositoryformatversion = 0\n"
    if remote:
        cfg += f'[remote "origin"]\n\turl = {remote}\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n'
    with open(os.path.join(root, ".git", "config"), "w") as fh:
        fh.write(cfg)
    return root


def test_repo_identity_follows_the_remote_not_the_spelling(tmp_path):
    a = _git_repo(str(tmp_path / "a"), "[email protected]:Owner/Repo.git")
    b = _git_repo(str(tmp_path / "b" / "repo"), "https://github.com/owner/repo")
    ra, rb = paths.repo_identity(a), paths.repo_identity(b)
    assert ra.remote == rb.remote == "github.com/owner/repo"
    assert ra.label == "owner/repo"
    # the key carries the checkout's name for readability, and the remote for identity
    assert ra.key.endswith(rb.key.split("-")[-1])


def test_a_subdirectory_and_a_worktree_share_their_repository(tmp_path):
    main = _git_repo(str(tmp_path / "main"), "https://github.com/o/r.git")
    os.makedirs(os.path.join(main, "src", "pkg"))
    wt_git = os.path.join(main, ".git", "worktrees", "feature")
    os.makedirs(wt_git)
    with open(os.path.join(wt_git, "commondir"), "w") as fh:
        fh.write("../..")
    wt = str(tmp_path / "feature")
    os.makedirs(wt)
    with open(os.path.join(wt, ".git"), "w") as fh:
        fh.write(f"gitdir: {wt_git}\n")
    sub = paths.repo_identity(os.path.join(main, "src", "pkg"))
    assert sub.root == main
    assert paths.repo_identity(wt).remote == sub.remote == "github.com/o/r"


def test_a_directory_without_git_is_keyed_by_its_path(tmp_path, monkeypatch):
    d = str(tmp_path / "scratch")
    os.makedirs(d)
    # Fence the walk at tmp_path: a machine may well have a stray .git in /tmp itself.
    monkeypatch.setenv("GIT_CEILING_DIRECTORIES", str(tmp_path))
    r = paths.repo_identity(d)
    assert r.remote == "" and r.label == "scratch" and r.root == d


def test_state_root_honours_the_environment(tmp_path):
    assert paths.state_root({"TOKENECONOMIZER_HOME": str(tmp_path)}) == str(tmp_path)


# ---- store and config ----------------------------------------------------------------------
def test_store_writes_atomically_and_logs(tmp_path):
    s = Store(str(tmp_path / "st"))
    s.write("a/b.json", {"x": 1})
    assert s.read("a/b.json") == {"x": 1}
    assert s.read("missing.json", default={}) == {}
    s.append("log.jsonl", {"n": 1})
    s.append("log.jsonl", {"n": 2})
    assert [r["n"] for r in s.read_lines("log.jsonl")] == [1, 2]
    assert [r["n"] for r in s.read_lines("log.jsonl", limit=1)] == [2]
    with s.locked("x") as got_x:
        with s.locked("y") as got_y:              # distinct locks never deadlock
            s.write("c.json", [1])
        with s.locked("x", timeout=0.1) as again:  # held: a hook's caller still proceeds,
            assert again is False                  # and is told it is unlocked
    assert got_x and got_y
    assert not [f for f in os.listdir(s.base) if f.endswith(".tmp")]


def test_config_layers_and_never_raises_on_bad_values(tmp_path):
    root = str(tmp_path)
    repo_dir = os.path.join(root, "repos", "r")
    os.makedirs(repo_dir)
    with open(os.path.join(root, "config.json"), "w") as fh:
        # names from before the mechanisms were renamed: still read
        json.dump({"spiral": "guard", "untool_evidence": 7, "bogus": 1}, fh)
    with open(os.path.join(repo_dir, "config.json"), "w") as fh:
        json.dump({"spiral": "nope", "inject": "envfacts+distiller"}, fh)
    cfg = live_config.load_config(root, repo_dir=repo_dir, env={"TE_UNTOOL": "auto"})
    assert cfg.loopguard == "stop"                # the invalid repo value is ignored
    assert cfg.toolprune_evidence == 7 and cfg.toolprune == "auto"
    assert cfg.microskills == "envfacts+distiller"     # its proposer needs no `paid`
    joined = " ".join(cfg.warnings)
    assert "bogus" in joined and "nope" in joined and "paid" not in joined
    assert live_config.load_config(root, env={"TE_DISABLE": "1"}).enabled is False


def test_write_config_validates_before_writing(tmp_path):
    path = str(tmp_path / "config.json")
    live_config.write_config(path, {"loopguard": "off", "paid_model": "haiku"})
    assert json.load(open(path)) == {"loopguard": "off", "paid_model": "haiku"}
    with pytest.raises(ValueError, match="always asks its judge"):
        live_config.write_config(path, {"paid": "true"})          # gone: refused, and why
    assert "paid" not in json.load(open(path))
    with open(path, "w") as fh:
        json.dump({"paid": True, "loopguard": "off"}, fh)          # an old file still reads
    cfg = live_config.load_config(str(tmp_path), repo_dir=None, env={})
    assert not cfg.warnings
    with pytest.raises(ValueError):
        live_config.write_config(path, {"loopguard": "sometimes"})
    assert json.load(open(path))["loopguard"] == "off"


# ---- Claude Code transcripts -----------------------------------------------------------------
def _claude_lines(path, blocks_per_message=3, n_messages=2, model="claude-opus-5-5"):
    lines = []
    tag = os.path.splitext(os.path.basename(path))[0]     # message ids are unique per request
    for m in range(n_messages):
        usage = {"input_tokens": 10, "cache_read_input_tokens": 1000,
                 "cache_creation_input_tokens": 100, "output_tokens": 50}
        for b in range(blocks_per_message):     # one entry per content block, usage repeated
            lines.append({"type": "assistant", "message": {
                "id": f"msg_{tag}_{m}", "model": model, "role": "assistant", "usage": usage,
                "content": [{"type": "text", "text": f"block {b}"}]}})
    with open(path, "w") as fh:
        fh.write("\n".join(json.dumps(x) for x in lines) + "\n")
    return path


def test_claude_usage_counts_each_message_once_and_includes_subagents(tmp_path):
    main = _claude_lines(str(tmp_path / "sess.jsonl"))
    sub_dir = tmp_path / "sess" / "subagents"
    sub_dir.mkdir(parents=True)
    _claude_lines(str(sub_dir / "agent-1.jsonl"), blocks_per_message=2, n_messages=1)
    u = trace.claude_session_usage(main)
    assert u["requests"] == 3                     # 2 main messages + 1 subagent message
    assert u["cache_read_input_tokens"] == 3000   # not 3 x 2 x 1000 + 2 x 1000
    assert u["model"] == "claude-opus-5-5" and u["files"] == 2
    assert trace.claude_session_usage(main, include_subagents=False)["requests"] == 2


def test_deferred_tools_are_read_from_attachments(tmp_path):
    p = str(tmp_path / "t.jsonl")
    with open(p, "w") as fh:
        fh.write(json.dumps({"type": "attachment", "attachment": {
            "type": "deferred_tools_delta", "addedNames": ["CronCreate", "WebSearch"],
            "removedNames": []}}) + "\n")
        fh.write(json.dumps({"type": "attachment", "attachment": {
            "type": "deferred_tools_delta", "addedNames": [], "removedNames": ["WebSearch"]}}) + "\n")
    assert trace.claude_deferred_tools(p) == {"CronCreate"}


# ---- Codex rollouts ---------------------------------------------------------------------------
def _rollout(path):
    recs = [
        {"type": "session_meta", "payload": {"id": "thread-1", "cwd": "/work/r",
                                             "cli_version": "0.153.1"}},
        {"type": "turn_context", "payload": {"turn_id": "t1", "model": "gpt-6-astra"}},
        {"type": "event_msg", "payload": {"type": "item_completed", "turn_id": "t1", "item": {
            "type": "CommandExecution", "id": "c1", "command": ["/bin/bash", "-lc", "pytest -q"],
            "aggregated_output": "1 failed", "exit_code": 1, "status": "completed"}}},
        {"type": "event_msg", "payload": {"type": "item_completed", "turn_id": "t1", "item": {
            "type": "FileChange", "id": "f1", "changes": {"/work/r/pkg/a.py": {"type": "update"}},
            "status": "completed"}}},
        {"type": "event_msg", "payload": {"type": "item_completed", "turn_id": "t1", "item": {
            "type": "Reasoning", "id": "r1"}}},
        {"type": "event_msg", "payload": {"type": "token_count", "info": {"total_token_usage": {
            "input_tokens": 1000, "cached_input_tokens": 600, "cache_write_input_tokens": 100,
            "output_tokens": 40, "reasoning_output_tokens": 10}}}},
        {"type": "event_msg", "payload": {"type": "token_count", "info": {"total_token_usage": {
            "input_tokens": 3000, "cached_input_tokens": 2000, "cache_write_input_tokens": 200,
            "output_tokens": 90, "reasoning_output_tokens": 30}}}},
    ]
    with open(path, "w") as fh:
        fh.write("\n".join(json.dumps(r) for r in recs) + "\n")
    return path


def test_codex_rollout_reads_through_the_existing_trace_seam(tmp_path):
    src = _rollout(str(tmp_path / "rollout-x.jsonl"))
    out = str(tmp_path / "exec.jsonl")
    assert trace.codex_rollout_to_exec(src, out) == 2       # reasoning is not a tool call
    traj = trace.load_trajectory(out)
    assert traj[0][0] == "shell" and traj[0][1]["command"] == "/bin/bash -lc 'pytest -q'"
    assert traj[0][2] == "1 failed" and traj[0][3] is False
    assert traj[1][0] == "apply_patch" and traj[1][1]["path"] == "/work/r/pkg/a.py"


def test_codex_usage_is_converted_as_the_harness_converts_it(tmp_path):
    u = trace.codex_session_usage(_rollout(str(tmp_path / "rollout-x.jsonl")))
    # last cumulative total; fresh input excludes both cache fields; reasoning joins output
    assert u["input_tokens"] == 3000 - 2000 - 200
    assert u["cache_read_input_tokens"] == 2000 and u["cache_creation_input_tokens"] == 200
    assert u["output_tokens"] == 90 + 30 and u["model"] == "gpt-6-astra"


def test_codex_capability_evidence_includes_the_models_own_exec_calls(tmp_path):
    """In code mode every tool call is one `exec` custom tool; `tools.view_image(` inside its
    input is image use, and only the capability trace carries it."""
    from tokeneconomizer.toolprune import codex as cu
    src = _rollout(str(tmp_path / "rollout-x.jsonl"))
    with open(src, "a") as fh:
        fh.write(json.dumps({"type": "response_item", "payload": {
            "type": "custom_tool_call", "name": "exec", "call_id": "call_9",
            "status": "completed",
            "input": "const r = await tools.view_image({path: 'a.png'});"}}) + "\n")
    out = str(tmp_path / "cap.jsonl")
    assert trace.codex_capability_trace(src, out) == 3
    usage = cu.inspect_usage(out)
    assert usage["completed"]["image_media"] == 1 and usage["completed"]["collaboration"] == 0
    plain = str(tmp_path / "exec.jsonl")
    trace.codex_rollout_to_exec(src, plain)
    assert cu.inspect_usage(plain)["completed"]["image_media"] == 0


def test_a_compressed_rollout_is_reported_unreadable(tmp_path):
    p = str(tmp_path / "rollout-x.jsonl.zst")
    open(p, "wb").write(b"\x28\xb5\x2f\xfd")
    assert trace.codex_rollout_to_exec(p, str(tmp_path / "o.jsonl")) is None


# ---- hosts ---------------------------------------------------------------------------------
def test_host_detection():
    assert hosts.detect({"turn_id": "t"}).name == "codex"
    assert hosts.detect({"transcript_path": "/h/.codex/sessions/2026/rollout-a.jsonl"}).name == "codex"
    assert hosts.detect({"transcript_path": "/h/.claude/projects/x/s.jsonl"}).name == "claude"
    assert hosts.detect({"turn_id": "t"}, hint="claude").name == "claude"
    assert hosts.detect({"transcript_path": "/h/.copilot/session-state/s/events.jsonl"}
                        ).name == "copilot"
    # Copilot sends no transcript before Stop, and marks its hook processes; a Claude Code
    # session started from inside Copilot inherits the marker but always sends a transcript.
    copilot_env = {"COPILOT_PLUGIN_ROOT": "/p", "COPILOT_CLI": "1"}
    assert hosts.detect({"session_id": "s"}, copilot_env).name == "copilot"
    assert hosts.detect({"session_id": "s", "transcript_path": "/h/.claude/projects/x/s.jsonl"},
                        copilot_env).name == "claude"
    assert hosts.detect({"session_id": "s"}, {}).name == "claude"


def test_the_te_command_defaults_to_the_agent_whose_shell_runs_it():
    assert hosts.default_host({}) == "claude"
    assert hosts.default_host({"COPILOT_CLI": "1"}) == "copilot"
    assert hosts.default_host({"COPILOT_CLI": "1", "CLAUDECODE": "1"}) == "claude"
    assert hosts.default_host({"TE_HOST": "codex", "CLAUDECODE": "1"}) == "codex"
    assert hosts.default_host({"TE_HOST": "nonsense"}) == "claude"
    assert hosts.default_host({}, recent="pi") == "pi"
    assert hosts.default_host({"CLAUDECODE": "1"}, recent="pi") == "claude"


def test_from_a_plain_terminal_te_is_about_the_agent_that_last_worked_here(tmp_path, monkeypatch,
                                                                          capsys):
    """Seen with every non-Claude tester: `te status` from a terminal showed the empty Claude
    Code state of a repository only Codex (or Pi) had worked in."""
    from tokeneconomizer.plugin import cli
    from tokeneconomizer.plugin.paths import repo_identity, repo_state_dir
    for k in ("TE_HOST", "CLAUDECODE", "COPILOT_CLI"):
        monkeypatch.delenv(k, raising=False)
    monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path / "te"))
    work = tmp_path / "work"
    work.mkdir()
    assert cli._recent_host(str(work)) is None
    d = repo_state_dir(repo_identity(str(work)), "codex")
    os.makedirs(d)
    open(os.path.join(d, "ledger.jsonl"), "w").close()
    assert cli._recent_host(str(work)) == "codex"
    assert cli.main(["--cwd", str(work), "status"]) == 0
    assert "(codex)" in capsys.readouterr().out.splitlines()[0]


# Payloads copilot 1.0.83 sent to a Claude-format plugin hook (PascalCase events), verbatim in
# shape, captured against a scripted local model endpoint on 2026-10-07.
COPILOT_POST = {"hook_event_name": "PostToolUse", "session_id": "fcb5fc6b",
                "timestamp": "2026-10-08T04:27:38.418Z", "cwd": "/w/repo", "tool_name": "Bash",
                "tool_input": {"command": "python3 -c 'import sys; sys.exit(3)'",
                               "description": "run it"},
                "tool_result": {"result_type": "success",
                                "text_result_for_llm": "boom\n<shellId: 0 completed with exit code 3>"}}
COPILOT_FAILURE = {"hook_event_name": "PostToolUseFailure", "session_id": "10bbc978",
                   "timestamp": "2026-10-08T04:27:51.286Z", "cwd": "/w/repo", "tool_name": "Read",
                   "tool_input": {"path": "/w/repo/nope.py"}, "error": "Path does not exist"}


def test_the_copilot_payloads_observed_live_are_read():
    c = hosts.detect(COPILOT_POST, {"COPILOT_PLUGIN_ROOT": "/p"})
    assert c.name == "copilot"
    name, args, text, code = c.tool_call(COPILOT_POST)
    assert (name, args["description"], code) == ("Bash", "run it", 3)   # exit code from the text
    assert text.startswith("boom")
    name, args, text, code = c.tool_call(COPILOT_FAILURE)
    assert (name, args["path"], code) == ("Read", "/w/repo/nope.py", None)
    assert "Path does not exist" in text
    # Copilot reads a TOP-LEVEL additionalContext and ignores hookSpecificOutput, and shows a
    # notice only as a progress line ahead of the output (both verified in its TUI)
    assert c.context("SessionStart", "x") == {"additionalContext": "x"}
    m = hosts.merge_outputs(c.context("SessionStart", "a"), c.notice("n"),
                            c.context("SessionStart", "b"))
    assert m == {"additionalContext": "a\n\nb", hosts.PROGRESS: ["n"]}


def test_a_copilot_session_log_is_found_from_the_session_id(monkeypatch, tmp_path):
    monkeypatch.setenv("COPILOT_HOME", str(tmp_path / "cp"))
    c = hosts.Copilot()
    assert c.transcript({"session_id": "abc"}) == str(
        tmp_path / "cp" / "session-state" / "abc" / "events.jsonl")
    assert c.transcript({"session_id": "abc", "transcript_path": "/x/events.jsonl"}) == \
        "/x/events.jsonl"
    assert c.cwd({}, {"COPILOT_PROJECT_DIR": "/w/repo"}) == "/w/repo"


def test_claude_tool_results_and_outputs():
    h = hosts.ClaudeCode()
    name, args, text, code = h.tool_call({
        "tool_name": "Bash", "tool_input": {"command": "pytest"},
        "tool_response": {"stdout": "1 failed", "stderr": "Traceback", "interrupted": False}})
    assert (name, args["command"], code) == ("Bash", "pytest", None)   # no exit code (verified)
    assert "1 failed" in text and "Traceback" in text
    assert h.context("PostToolUse", "x") == {
        "hookSpecificOutput": {"hookEventName": "PostToolUse", "additionalContext": "x"}}
    # the hook environment of a `claude -p` session, as observed on 2.1.285
    assert h.headless({"CLAUDE_CODE_SESSION_ATTENDED": "0", "CLAUDE_CODE_ENTRYPOINT": "sdk-cli"})
    assert not h.headless({"CLAUDE_CODE_SESSION_ATTENDED": "1", "CLAUDE_CODE_ENTRYPOINT": "sdk-cli"})
    assert h.headless({"CLAUDE_CODE_ENTRYPOINT": "sdk-cli"}) and not h.headless({})


def test_the_codex_posttooluse_payload_observed_live_is_read():
    """The payload codex-cli 0.153.1 sent for `echo one` (gpt-5.6-sol, 2026-09-30), verbatim in
    shape: tool_name "Bash", a string command, a plain-string result, turn_id and model."""
    payload = {"session_id": "01a0f5e1", "turn_id": "t1", "model": "gpt-5.6-sol",
               "permission_mode": "default", "hook_event_name": "PostToolUse",
               "cwd": "/w/repo", "transcript_path": "/h/.codex/sessions/2026/09/30/rollout-x.jsonl",
               "tool_name": "Bash", "tool_use_id": "call_1", "tool_input": {"command": "echo one"},
               "tool_response": "one\n"}
    host = hosts.detect(payload)
    assert host.name == "codex"
    assert host.tool_call(payload) == ("bash", {"command": "echo one"}, "one\n", None)


def test_codex_tool_calls_normalize():
    c = hosts.Codex()
    name, args, text, code = c.tool_call({
        "tool_name": "exec_command", "tool_input": {"command": ["bash", "-lc", "go test ./..."]},
        "tool_response": {"output": "FAIL", "exit_code": 1}})
    assert (name, args["command"], code) == ("bash", "bash -lc 'go test ./...'", 1)
    name, args, _, _ = c.tool_call({"tool_name": "apply_patch", "tool_input": {
        "input": "*** Begin Patch\n*** Update File: src/a.py\n@@\n-x\n+y\n*** End Patch"}})
    assert name == "apply_patch" and args["paths"] == ["src/a.py"]
    _, _, _, code = c.tool_call({"tool_name": "shell", "tool_input": {"command": "ls"},
                                 "tool_response": "Exit code: 2\nWall time: 0.1s\nOutput:\n"})
    assert code == 2


def test_merge_outputs_concatenates_contexts_and_notices():
    h = hosts.ClaudeCode()
    m = hosts.merge_outputs(h.context("SessionStart", "a"), h.notice("n1"),
                            h.context("SessionStart", "b"), h.notice("n2"), None, {})
    assert m["hookSpecificOutput"]["additionalContext"] == "a\n\nb"
    assert m["systemMessage"] == "n1\nn2"


# ---- Pi sessions -------------------------------------------------------------------------
def _pi_session(path, cwd="/w/repo", sid="pi-1", abandoned=False):
    """A Pi v3 session tree: header, a user prompt, a bash call that fails, an edit, a bash
    call that passes -- and, optionally, an abandoned branch hanging off the first prompt."""
    lines = [{"type": "session", "version": 3, "id": sid, "timestamp": "2026-09-30T00:00:00Z",
              "cwd": cwd}]
    state = {"parent": None, "n": 0}

    def add(entry, parent="__chain__"):
        state["n"] += 1
        eid = f"e{state['n']:03d}"
        entry.update(id=eid, timestamp="2026-09-30T00:00:01Z",
                     parentId=state["parent"] if parent == "__chain__" else parent)
        lines.append(entry)
        if parent == "__chain__":
            state["parent"] = eid
        return eid

    usage = {"input": 10, "output": 50, "cacheRead": 1000, "cacheWrite": 100, "reasoning": 5,
             "totalTokens": 1160, "cost": {"total": 0.002}}

    def call(name, args, text, err):
        tc = f"call{state['n']}"
        add({"type": "message", "message": {"role": "assistant", "provider": "openai",
             "model": "gpt-5.6-sol", "usage": usage,
             "content": [{"type": "toolCall", "id": tc, "name": name, "arguments": args}]}})
        add({"type": "message", "message": {"role": "toolResult", "toolCallId": tc,
             "toolName": name, "isError": err, "content": [{"type": "text", "text": text}]}})

    first = add({"type": "message", "message": {"role": "user", "content": "fix the bug"}})
    call("bash", {"command": "pytest -q"}, "bash: pytest: command not found", True)
    call("edit", {"path": "pkg/a.py", "oldText": "x", "newText": "y"}, "ok", False)
    call("bash", {"command": "python -m pytest -q"}, "1 passed", False)
    if abandoned:                        # a branch the user walked away from, earlier in the file
        lines.insert(3, {"type": "message", "id": "zzz", "parentId": first,
                         "timestamp": "2026-09-30T00:00:02Z", "message": {
                             "role": "assistant", "model": "gpt-5.6-sol", "usage": usage,
                             "content": [{"type": "toolCall", "id": "dead", "name": "bash",
                                          "arguments": {"command": "rm -rf build"}}]}})
    with open(path, "w") as fh:
        fh.write("\n".join(json.dumps(x) for x in lines) + "\n")
    return path


def test_a_pi_session_reads_through_the_trace_seam_on_its_active_branch(tmp_path):
    src = _pi_session(str(tmp_path / "s.jsonl"), abandoned=True)
    out = str(tmp_path / "trace.jsonl")
    assert trace.pi_session_to_claude(src, out) == 3
    traj = trace.load_trajectory(out)
    assert [(n, a.get("command") or a.get("path"), ok) for n, a, _, ok in traj] == [
        ("bash", "pytest -q", False), ("edit", "pkg/a.py", True), ("bash", "python -m pytest -q", True)]
    assert "rm -rf build" not in open(out).read()          # the abandoned branch is not read


def test_a_pi_call_into_the_session_directory_is_the_plain_form(tmp_path):
    """Pi starts every command afresh in the session's directory, so `cd <that dir> && tox`
    is the same form as `tox`. Without a cwd per call it was learned as a second form, with the
    absolute path in the claim, and the two split one fact's evidence."""
    from tokeneconomizer.microskills.observe import PYTHON_GIT, observe_task
    src = str(tmp_path / "s.jsonl")
    _pi_session(src, cwd="/w/repo")
    lines = [json.loads(l) for l in open(src)]
    for e in lines:
        for b in (e.get("message") or {}).get("content") or []:
            if isinstance(b, dict) and b.get("arguments", {}).get("command") == "pytest -q":
                b["arguments"]["command"] = "cd /w/repo && pytest -q"
    open(src, "w").write("\n".join(json.dumps(e) for e in lines) + "\n")
    out = str(tmp_path / "trace.jsonl")
    trace.pi_session_to_claude(src, out)
    assert trace.call_cwds(out) == ["/w/repo"] * 3
    forms = [o.form for o in observe_task(out, PYTHON_GIT)["obs"] if o.intent == "run_tests"]
    assert forms == ["pytest", "python -m pytest"]


def test_pi_usage_maps_directly_and_keeps_pis_own_cost(tmp_path):
    u = trace.pi_session_usage(_pi_session(str(tmp_path / "s.jsonl"), abandoned=True))
    # three assistant messages on the active branch; output already includes reasoning
    assert u["requests"] == 3
    assert (u["input_tokens"], u["cache_read_input_tokens"], u["cache_creation_input_tokens"],
            u["output_tokens"]) == (30, 3000, 300, 150)
    assert u["cost_usd_vendor"] == pytest.approx(0.006) and u["model"] == "gpt-5.6-sol"
    assert trace.pi_session_header(str(tmp_path / "s.jsonl"))["cwd"] == "/w/repo"


def test_the_pi_host_reads_what_the_extension_sends():
    h = hosts.HOSTS["pi"]()
    name, args, text, code = h.tool_call({"tool_name": "bash", "tool_input": {"command": "go test"},
                                          "tool_response": {"content": "FAIL\nCommand exited with code 1",
                                                            "isError": True, "exit_code": 1}})
    assert (name, args["command"], code) == ("bash", "go test", 1) and "FAIL" in text
    assert h.headless({"TE_PI_MODE": "print"}) and not h.headless({"TE_PI_MODE": "tui"})
    assert hosts.detect({"turn_id": "t"}, hint="pi").name == "pi"


# ---- opencode sessions -------------------------------------------------------------------
def _opencode_export(path, cwd="/w/repo"):
    """An opencode export: a prompt, a bash call that fails, an edit, a bash call that passes."""
    tokens = {"input": 10, "output": 45, "reasoning": 5, "cache": {"read": 1000, "write": 100}}

    def call(cid, tool, args, state):
        return {"info": {"id": f"m-{cid}", "role": "assistant", "modelID": "gpt-5.6-sol",
                         "path": {"cwd": cwd, "root": cwd}, "cost": 0.002, "tokens": tokens},
                "parts": [{"type": "tool", "callID": cid, "tool": tool,
                           "state": dict(state, input=args)}]}

    msgs = [{"info": {"id": "u1", "role": "user"},
             "parts": [{"type": "text", "text": "fix the bug"},
                       {"type": "text", "text": "(added by opencode)", "synthetic": True}]},
            call("c1", "bash", {"command": "pytest -q"},
                 {"status": "error", "error": "bash: pytest: command not found"}),
            call("c2", "edit", {"filePath": "pkg/a.py", "oldString": "x", "newString": "y"},
                 {"status": "completed", "output": "ok", "metadata": {}}),
            call("c3", "bash", {"command": "python -m pytest -q"},
                 {"status": "completed", "output": "1 passed", "metadata": {"exit": 0}})]
    with open(path, "w") as fh:
        json.dump({"info": {"id": "oc-1"}, "messages": msgs}, fh)
    return path


def test_an_opencode_export_reads_through_the_trace_seam(tmp_path):
    src = _opencode_export(str(tmp_path / "s.json"))
    out = str(tmp_path / "t.jsonl")
    assert trace.opencode_session_to_claude(src, out) == 3
    assert trace.opencode_prompt_count(src) == 1          # opencode's own text is not a prompt
    text = open(out).read()
    assert "command not found" in text and "python -m pytest -q" in text
    assert '"is_error": true' in text and "fix the bug" in text and "added by opencode" not in text


def test_an_opencode_list_export_and_a_missing_file(tmp_path):
    src = _opencode_export(str(tmp_path / "s.json"))
    listed = str(tmp_path / "l.json")
    json.dump(json.load(open(src))["messages"], open(listed, "w"))
    assert trace.opencode_session_to_claude(listed, str(tmp_path / "t.jsonl")) == 3
    assert trace.opencode_session_to_claude(str(tmp_path / "none.json"),
                                            str(tmp_path / "x")) is None


def test_opencode_usage_counts_reasoning_as_output(tmp_path):
    u = trace.opencode_session_usage(_opencode_export(str(tmp_path / "s.json")))
    assert (u["input_tokens"], u["output_tokens"]) == (30, 150)
    assert (u["cache_read_input_tokens"], u["cache_creation_input_tokens"]) == (3000, 300)
    assert u["requests"] == 3 and u["model"] == "gpt-5.6-sol"
    assert u["cost_usd_vendor"] == 0.006


def test_the_opencode_host_reads_what_the_plugin_sends():
    h = hosts.HOSTS["opencode"]()
    name, args, text, code = h.tool_call({"tool_name": "bash", "tool_input": {"command": "go test"},
                                          "tool_response": {"content": "FAIL", "isError": True,
                                                            "exit_code": 1}})
    assert (name, args["command"], code) == ("bash", "go test", 1) and "FAIL" in text
    assert h.headless({"TE_OPENCODE_MODE": "run"}) and not h.headless({"TE_OPENCODE_MODE": "tui"})
    assert h.turn_key({"prompt_id": "s:1"}) == "s:1"
    assert hosts.detect({}, hint="opencode").name == "opencode"


def _run_events(sid="ses_x"):
    ev = [{"type": "step_start", "sessionID": sid, "part": {"type": "step-start"}},
          {"type": "text", "sessionID": sid, "part": {"type": "text", "text": "po"}},
          {"type": "text", "sessionID": sid, "part": {"type": "text", "text": "ng"}},
          {"type": "step_finish", "sessionID": sid, "part": {
              "type": "step-finish", "cost": 0.5,
              "tokens": {"input": 10, "output": 4, "reasoning": 1,
                         "cache": {"read": 7, "write": 2}}}}]
    return "\n".join(json.dumps(e) for e in ev) + "\nnot json\n"


def test_an_opencode_run_reply_is_read_from_its_json_events():
    text, usage, sid = trace.opencode_run_reply(_run_events())
    assert (text, sid) == ("pong", "ses_x")
    assert usage == {"input_tokens": 10, "cache_read_input_tokens": 7,
                     "cache_creation_input_tokens": 2, "output_tokens": 5, "cost_usd": 0.5}
    assert trace.opencode_run_reply("") == ("", None, "")


def test_the_opencode_side_call_denies_tools_and_deletes_its_session(monkeypatch):
    from tokeneconomizer.plugin import sidecall
    seen = []

    def fake_run(cmd, **kw):
        seen.append((cmd, kw))
        out = _run_events() if "--format" in cmd else "--pure --standalone"
        return subprocess.CompletedProcess(cmd, 0, out, "")

    monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
    sidecall._OC_FLAGS.clear()
    text, eff = sidecall.opencode_call("is it stuck?", model="p/m", cli="oc")
    assert text == "pong" and eff
    run = next(c for c, _ in seen if "--format" in c)
    assert run[:2] == ["oc", "run"] and "--pure" in run and "--standalone" in run
    assert run[-3:] == ["--model", "p/m", "is it stuck?"]
    kw = next(k for c, k in seen if "--format" in c)
    assert kw["env"]["TE_SIDECALL"] == "1"
    perm = json.loads(kw["env"]["OPENCODE_CONFIG_CONTENT"])["permission"]
    assert perm["bash"] == "deny" and perm["edit"] == "deny"
    assert any(c[:3] == ["oc", "session", "delete"] and c[3] == "ses_x" for c, _ in seen)


def test_the_opencode_side_call_adds_only_the_flags_its_cli_lists(monkeypatch):
    from tokeneconomizer.plugin import sidecall
    monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **kw: subprocess.CompletedProcess(
        cmd, 0, "  --format  json\n", ""))
    sidecall._OC_FLAGS.clear()
    assert sidecall.opencode_flags("oc") == []
    assert sidecall.runner_for("opencode", type("C", (), {"paid_model": ""})()) is not None


# ---- the guard sees edits made through the shell ---------------------------------------------
from tokeneconomizer import loopguard as guard  # noqa: E402
from tokeneconomizer.loopguard import turn  # noqa: E402


@pytest.mark.parametrize("cmd,want", [
    ("sed -i 's/a/b/' pkg/mod.py", ["mod.py"]),
    ("cd /t && sed -i.bak -e 's/a/b/' -e 's/c/d/' x.py y.py && pytest -q", ["x.py", "y.py"]),
    ("sed -i '' 's/a/b/' x.py", ["x.py"]),                         # BSD sed's empty suffix
    ("perl -pi -e 's/foo/bar/' src/a.rs", ["a.rs"]),
    ("cat > repro.py <<'EOF'\nprint(1 > 0)\nEOF\npython repro.py", ["repro.py"]),
    ("echo 'x' | tee -a setup.cfg", ["setup.cfg"]),
    # how Sonnet 5.5 edits: an inline script that reads the file and writes it back
    ("python - <<'E'\np='sympy/printing/str.py'; s=open(p).read()\nopen(p,'w').write(s)\nE",
     ["str.py"]),
    ("python - <<'E'\nfor p in ['x/a.py', 'x/b.py']:\n    open(p, 'w').write('')\nE",
     ["a.py", "b.py"]),
    ("python - <<'E'\nfrom pathlib import Path\nPath('m/n.py').write_text('x')\nE", ["n.py"]),
    ("git apply /tmp/fix.diff", ["(patch)"]),
    # Codex 0.163.0 edits like this, then tests, in ONE command (seen live)
    ("apply_patch <<'PATCH'\n*** Begin Patch\n*** Update File: pkg/__init__.py\n@@\n-    x\n"
     "+    y\n*** End Patch\nPATCH\npython3 -m unittest", ["__init__.py"]),
    ("apply_patch <<'P'\n*** Begin Patch\n*** Add File: a/new.py\n+x\n*** Update File: b/old.py\n"
     "*** End Patch\nP", ["new.py", "old.py"]),
    ("git checkout -- a.py b/c.py", ["a.py", "c.py"]),
    # not edits: output captures, scratch, reads, and operators inside quotes
    ("python -m pytest -q > /tmp/out.txt 2>&1; tail -5 /tmp/out.txt", []),
    ("grep -n 'x' a.py > hits.txt", []),
    ("grep -n '>' x.py", []),
    ("grep -nE 'a|b' x.py | head", []),
    ("python -c \"print(1>2)\"", []),
    ("git stash -q; python bin/test x; git stash pop -q", []),
    ("sed -n '1,20p' pkg/mod.py", []),
    # a quoted `|` in a sed script must not split the command (seen live: sympy-13091)
    ("sed -i -E 's/x (<|<=|>|>=) y/z/' numbers.py expr.py", ["numbers.py", "expr.py"]),
    ("sed -i 's/a/b/' x.py 2>&1 | tail -2", ["x.py"]),
])
def test_edits_made_through_the_shell_are_recognised(cmd, want):
    assert guard.shell_edit_targets(cmd) == want


def test_shell_edits_reach_the_state_the_judge_reads_and_the_ended_green_proxy():
    g = guard.TurnGuard(turn_key="t")
    edit = "python - <<'E'\np='pkg/mod.py'; s=open(p).read()\nopen(p,'w').write(s[::-1])\nE"
    for _ in range(2):
        g.fold("Bash", {"command": edit}, "")
    assert g.st.edit_churn == 2 and g.whole.distinct_edit_files == 1 and g.edits == 2
    digest = g.st.recent_digest()
    assert "edit_churn=2" in digest and "[edits mod.py]" in digest
    # one command that edits and then fails its tests: the turn did not end green
    g.fold("Bash", {"command": "sed -i 's/b/c/' pkg/mod.py && pytest -q"},
           "FAILED tests/test_m.py::t - AssertionError")
    assert g.edits == 3 and g.last_verify > g.last_edit and not g.resolved_proxy()


def test_reading_a_file_full_of_asserts_is_not_a_failure():
    """A test file read back is full of `assert` and `Error`: reading it, three times over, is
    no failure and no repeated failure, and opens no gate to the judge."""
    test_file = "def test_total():\n    assert total([1, 2]) == 3\n    raise ValueError('no')\n"
    g = guard.TurnGuard(turn_key="t")
    for _ in range(3):
        g.fold("read", {"path": "tests/test_total.py"}, test_file)
        g.fold("grep", {"pattern": "assert"}, "tests/test_total.py:2:    assert total")
    g.fold("bash", {"command": "cat tests/test_total.py"}, test_file, 0)
    assert (g.whole.error_outputs, g.whole.repeated_fail_sigs) == (0, 0)
    assert not guard.gate_for_judge(g.whole, dict(guard.DEFAULT_THRESHOLDS, gate_min_turns=1))
    # what did fail still counts: a non-zero exit, or a read the tool itself reports as failed
    g.fold("bash", {"command": "pytest -q"}, "FAILED tests/test_total.py - AssertionError", 1)
    g.fold("bash", {"command": "pytest -q"}, "FAILED tests/test_total.py - AssertionError", None)
    assert (g.whole.error_outputs, g.whole.repeated_fail_sigs) == (2, 1)
    s = guard.SpiralState()
    s.update("Read", {"file_path": "x.py"}, "Error: File does not exist.", failed=True)
    assert s.error_outputs == 1


def test_the_judge_is_told_how_the_last_verification_went_not_that_tests_are_red():
    g = guard.TurnGuard(turn_key="t")
    vendored = "turns=3\nrepo_context: ... = 1.2x that typical length. Tests are still NOT green. A"
    assert "No test or verification run has been seen" in g.live_digest(vendored)
    g.fold("Bash", {"command": "python bin/test sympy/core"}, "tests finished: 10 passed")
    told = g.live_digest(vendored)
    assert "The last verification run passed." in told and "NOT green" not in told
    g.fold("Edit", {"file_path": "/r/a.py", "old_string": "a", "new_string": "b"}, "ok")
    assert g.live_digest(vendored).count("There have been edits since.") == 1


class _BackgroundJudge:
    """`plugin.judge.AsyncJudge`'s interface, with the verdict handed in by the test."""

    def __init__(self):
        self.started, self.verdict = [], None

    def start(self, g, digest):
        self.started.append(digest)
        return f"job{len(self.started)}"

    def poll(self, g):
        pending = g.judge_pending or {}
        if self.verdict is None or pending.get("id") != f"job{len(self.started)}":
            return False, None
        v, self.verdict = self.verdict, None
        return True, v


def _spin(g, judge, n, envelope=6.0):
    """An edit and an identical failing test run, alternately: a loop the gate opens on."""
    out = []
    for k in range(n):
        call = (("Edit", {"file_path": "/r/a.py", "old_string": "x", "new_string": "y"}, "ok")
                if k % 2 == 0 else
                ("Bash", {"command": "pytest -q"}, "FAILED tests/test_a.py::t - AssertionError"))
        out.append(g.fold(*call, envelope=envelope, judge=judge))
    return out


def test_a_background_judge_starts_once_and_its_verdict_nudges_a_later_call():
    g, j = guard.TurnGuard(turn_key="t"), _BackgroundJudge()
    assert _spin(g, j, 10) == [None] * 10            # nothing is decided while it is out
    assert len(j.started) == 1                       # and no second job is started
    assert "NOT green" not in j.started[0]
    j.verdict = {"stuck": True, "confidence": 0.9, "reason": "loop",
                 "_nudge": "Read the failing assertion before editing again."}
    assert _spin(g, j, 1)[0] == ("nudge", "[email protected]:loop")
    assert g.nudge_body(6.0) == "Read the failing assertion before editing again."
    assert g.nudge_body(6.0) != "Read the failing assertion before editing again."  # used once


def test_a_productive_verdict_backs_the_judge_off():
    g, j = guard.TurnGuard(turn_key="t"), _BackgroundJudge()
    _spin(g, j, 7)
    j.verdict = {"stuck": False, "confidence": 0.8, "reason": "progress"}
    assert _spin(g, j, 1) == [None]
    assert g.judge_gap == g.JUDGE_BASE_GAP * 2 and g.judge_pending is None


def test_a_job_that_never_answers_stops_blocking_the_next_one():
    g, j = guard.TurnGuard(turn_key="t"), _BackgroundJudge()
    _spin(g, j, 7)
    assert len(j.started) == 1 and g.judge_pending
    g.judge_pending["since"] = 0                     # its process died
    _spin(g, j, g.judge_gap)                         # well before the backstop at 2.5x
    assert len(j.started) == 2


# ---- Copilot session logs ------------------------------------------------------------------
def copilot_events(path, calls, *, shutdown=True, model="gpt-5.4", cwd="/w/repo", sid="cp1",
                   prompts=1):
    """A Copilot `session-state/<id>/events.jsonl`: session.start, `prompts` user messages (the
    tool calls, as execution_start/complete pairs, in the first), and (unless killed) a
    session.shutdown."""
    ev = [{"type": "session.start", "data": {"sessionId": sid, "copilotVersion": "1.0.83",
                                            "selectedModel": model, "context": {"cwd": cwd}}},
          {"type": "user.message", "data": {"content": "fix it", "turnId": "0"}}]
    for n, (name, args, text, ok) in enumerate(calls):
        ev.append({"type": "tool.execution_start", "data": {
            "toolCallId": f"c{n}", "toolName": name, "arguments": args, "turnId": "0"}})
        ev.append({"type": "tool.execution_complete", "data": {
            "toolCallId": f"c{n}", "success": ok, "turnId": "0",
            "result": {"content": text, "detailedContent": text}}})
    for k in range(1, prompts):
        ev.append({"type": "user.message", "data": {"content": "and then", "turnId": str(k)}})
    if shutdown:
        ev.append({"type": "session.shutdown", "data": {
            "modelMetrics": {model: {"requests": {"count": 4}, "usage": {
                "inputTokens": 4000, "outputTokens": 80, "cacheReadTokens": 2400,
                "cacheWriteTokens": 100, "reasoningTokens": 0}}},
            "toolDefinitionsTokens": 13149}})
    os.makedirs(os.path.dirname(path), exist_ok=True)
    with open(path, "w") as fh:
        fh.write("\n".join(json.dumps(e) for e in ev) + "\n")
    return path


def test_copilot_logs_read_through_the_existing_trace_seam(tmp_path):
    p = copilot_events(str(tmp_path / "s" / "events.jsonl"), [
        ("bash", {"command": "pytest -q"}, "1 failed\n<shellId: 0 completed with exit code 1>",
         True),
        ("view", {"path": "/w/repo/a.py"}, "x = 1", True)])
    traj = trace.load_trajectory(p)
    assert [t[0] for t in traj] == ["bash", "view"] and traj[0][1]["command"] == "pytest -q"
    assert trace.tool_usage(p) == {"bash": 1, "view": 1}
    (turn,) = trace.copilot_turns(p)
    assert turn[0][0] == "bash" and turn[0][3] == 1 and turn[1][3] is None
    assert trace.copilot_session_meta(p)["cwd"] == "/w/repo"


def test_a_copilot_call_into_the_session_directory_is_the_plain_form(tmp_path):
    """Copilot's models type `cd <repo> && ` before every command: `cd /w/repo && python -m
    unittest` is the same form as the bare one."""
    from tokeneconomizer.microskills.observe import PYTHON_GIT, observe_task
    p = copilot_events(str(tmp_path / "s" / "events.jsonl"), [
        ("bash", {"command": "cd /w/repo && pytest -q"},
         "bash: pytest: command not found\n<shellId: 0 completed with exit code 127>", False),
        ("bash", {"command": "cd /w/repo && python -m unittest -q"},
         "Ran 4 tests in 0.001s\nOK\n<shellId: 0 completed with exit code 0>", True)])
    assert trace.call_cwds(p) == ["/w/repo", "/w/repo"]
    assert [o.form for o in observe_task(p, PYTHON_GIT)["obs"] if o.intent == "run_tests"] == [
        "pytest", "python -m unittest"]


def test_copilot_usage_is_converted_from_its_grand_input_total(tmp_path):
    p = copilot_events(str(tmp_path / "s" / "events.jsonl"), [])
    u = trace.copilot_session_usage(p)
    # inputTokens counts cache reads and writes (verified against a scripted endpoint)
    assert u["input_tokens"] == 4000 - 2400 - 100
    assert u["cache_read_input_tokens"] == 2400 and u["cache_creation_input_tokens"] == 100
    assert u["output_tokens"] == 80 and u["requests"] == 4 and u["model"] == "gpt-5.4"
    assert u["tool_definitions_tokens"] == 13149
    killed = trace.copilot_session_usage(
        copilot_events(str(tmp_path / "k" / "events.jsonl"), [], shutdown=False))
    assert killed["input_tokens"] is None and killed["model"] == "gpt-5.4"


def test_copilot_tools_are_grouped_by_mcp_server(tmp_path):
    servers = {"github-mcp-server", "github", "acme-db"}
    assert trace.mcp_server_of("github-mcp-server-list_issues", servers) == \
        "github-mcp-server"                                  # the longest prefix wins
    assert trace.mcp_server_of("acme-db-query", servers) == "acme-db"
    assert trace.mcp_server_of("bash", servers) is None
    home = tmp_path / "cp"
    home.mkdir()
    (home / "mcp-config.json").write_text(json.dumps({"mcpServers": {"acme-db": {}}}))
    assert trace.copilot_mcp_servers({"COPILOT_HOME": str(home)}) == {
        "github-mcp-server", "acme-db"}


@pytest.mark.real_sidecalls
def test_a_copilot_side_call_is_a_bare_completion_priced_from_its_own_log(tmp_path,
                                                                          monkeypatch):
    import subprocess
    from tokeneconomizer.plugin import sidecall
    monkeypatch.setenv("COPILOT_HOME", str(tmp_path / "cp"))
    copilot_events(str(tmp_path / "cp" / "session-state" / "side1" / "events.jsonl"), [])
    seen = {}

    def fake_run(cmd, **kw):
        seen["cmd"], seen["env"] = cmd, kw["env"]
        out = "\n".join(json.dumps(e) for e in (
            {"type": "assistant.message", "data": {"content": "- use python -m pytest"}},
            {"type": "result", "sessionId": "side1", "exitCode": 0}))
        return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="")

    monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
    text, eff = sidecall.copilot_call("propose facts")
    assert text == "- use python -m pytest" and eff and eff > 0
    cmd = seen["cmd"]
    assert cmd[:3] == ["copilot", "-p", "propose facts"] and "--disable-builtin-mcps" in cmd
    assert cmd.count("--excluded-tools") == len(sidecall.COPILOT_TOOLS)
    assert seen["env"]["TE_SIDECALL"] == "1"            # its own hooks must not recurse
    cfg = live_config.LiveConfig()
    assert sidecall.runner_for("copilot", cfg).__code__.co_names.count("copilot_call") == 1
    assert sidecall.runner_for("claude", cfg).__code__.co_names.count("claude_call") == 1
    assert sidecall.runner_for("pi", cfg).__code__.co_names.count("pi_call") == 1


@pytest.mark.real_sidecalls
def test_a_side_call_with_no_reply_is_noted_and_keeps_to_a_bare_completion(tmp_path,
                                                                           monkeypatch):
    """Under Copilot's BYOK the provider's variables do not reach the hooks, so a side call can
    run signed out and answer nothing: that is noted, and the call stays a bare completion."""
    import subprocess
    from tokeneconomizer.plugin import sidecall
    monkeypatch.setenv("TOKENECONOMIZER_HOME", str(tmp_path / "te"))
    home = tmp_path / "cp"
    home.mkdir()
    (home / "mcp-config.json").write_text(json.dumps({"mcpServers": {"acme-db": {}}}))
    monkeypatch.setenv("COPILOT_HOME", str(home))
    seen = {}

    def fake_run(cmd, **kw):
        seen["cmd"] = cmd
        return subprocess.CompletedProcess(cmd, 1, stdout="",
                                           stderr="Error: No authentication information found.")

    monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
    assert sidecall.copilot_call("propose facts") == ("", None)
    cmd = seen["cmd"]
    assert cmd[cmd.index("--disable-mcp-server") + 1] == "acme-db"
    log = (tmp_path / "te" / "errors.log").read_text()
    assert "sidecall copilot: no reply (exit 1)" in log and "No authentication" in log


def test_claude_side_calls_stay_out_of_the_persons_history_where_the_cli_allows(monkeypatch):
    import subprocess
    from tokeneconomizer.plugin import sidecall
    helps = {"new": "  --no-session-persistence   Disable session persistence", "old": "  -p"}
    monkeypatch.setattr(sidecall.subprocess, "run", lambda cmd, **kw: subprocess.CompletedProcess(
        cmd, 0, stdout=helps[cmd[0]], stderr=""))
    monkeypatch.setattr(sidecall, "_HELP", {})
    assert sidecall.claude_flags("new") == ["--no-session-persistence"]
    assert sidecall.claude_flags("old") == []


@pytest.mark.real_sidecalls
def test_a_claude_side_call_attaches_none_of_the_persons_mcp_servers(monkeypatch):
    """`--tools ""` leaves every MCP server attached: the side call drops them where the CLI can,
    while the catalog probes, which need the tools, keep them."""
    import subprocess
    from tokeneconomizer.plugin import sidecall
    helps = {"new": "  --no-session-persistence\n  --strict-mcp-config   Only use MCP servers "
                    "from --mcp-config\n  --mcp-config <configs...>", "old": "  -p"}
    seen = {}

    def fake_run(cmd, **kw):
        if cmd[1:] == ["--help"]:
            return subprocess.CompletedProcess(cmd, 0, stdout=helps[cmd[0]], stderr="")
        seen.update(cmd=cmd, cwd=kw["cwd"], env=kw["env"])
        if "--mcp-config" in cmd:
            with open(cmd[cmd.index("--mcp-config") + 1], encoding="utf-8") as fh:
                seen["config"] = json.load(fh)
        return subprocess.CompletedProcess(cmd, 0, stdout=json.dumps({
            "result": "OK", "usage": {"input_tokens": 10, "output_tokens": 1},
            "modelUsage": {"claude-haiku-4-5": {}}}), stderr="")

    monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
    monkeypatch.setattr(sidecall, "_HELP", {})
    assert sidecall.claude_call("Reply with exactly: OK", cli="new")[0] == "OK"
    cmd = seen["cmd"]
    assert "--strict-mcp-config" in cmd and seen["config"] == {"mcpServers": {}}
    config = cmd[cmd.index("--mcp-config") + 1]
    assert os.path.dirname(config) == seen["cwd"]
    # `--mcp-config` takes several values: what follows its file is an option
    assert cmd[cmd.index("--mcp-config") + 2].startswith("--")
    assert seen["env"]["TE_SIDECALL"] == "1"
    assert "--strict-mcp-config" not in sidecall.claude_flags("new")     # the probes' flags
    # a Claude Code without the flags would reject them: the call goes out without
    sidecall.claude_call("Reply with exactly: OK", cli="old")
    assert "--strict-mcp-config" not in seen["cmd"] and "--mcp-config" not in seen["cmd"]


def test_claude_cli_falls_back_to_the_native_install_when_not_on_path(monkeypatch, tmp_path):
    import os
    from tokeneconomizer.plugin import sidecall
    monkeypatch.setattr(sidecall.shutil, "which", lambda name: None)
    monkeypatch.setattr(sidecall.os.path, "expanduser",
                        lambda p: p.replace("~", str(tmp_path), 1))
    assert sidecall.claude_cli() == "claude"
    exe = tmp_path / ".local" / "bin" / ("claude.exe" if os.name == "nt" else "claude")
    exe.parent.mkdir(parents=True)
    exe.write_text("")
    assert sidecall.claude_cli() == str(exe)
    monkeypatch.setattr(sidecall.shutil, "which", lambda name: "/opt/bin/claude")
    assert sidecall.claude_cli() == "/opt/bin/claude"


# `pi --print --mode json` on Pi 1.2.0 through OpenRouter, trimmed: the header, the system and
# user messages, the reply (a thinking block, then the text), and the run's end.
PI_JSON_RUN = [
    {"type": "session", "version": 3, "id": "01a13c79", "cwd": "/tmp/te-side-x"},
    {"type": "message_end", "message": {"role": "system", "content": "",
                                        "sections": {"preamble": "You are an expert..."}}},
    {"type": "message_end", "message": {"role": "user", "content": [
        {"type": "text", "text": "propose facts"}]}},
    {"type": "message_update", "usage": {"input": 745, "output": 3}},
    {"type": "message_end", "message": {
        "role": "assistant", "provider": "openrouter", "model": "anthropic/claude-haiku-4.5",
        "content": [{"type": "thinking", "thinking": "Two lines."},
                    {"type": "text", "text": "- use python -m pytest"}],
        "usage": {"input": 745, "output": 298, "cacheRead": 0, "cacheWrite": 0,
                  "totalTokens": 1043, "cacheWrite1h": 0, "reasoning": 281,
                  "cost": {"input": 0.000745, "output": 0.00149, "total": 0.002235}},
        "stopReason": "stop"}},
    {"type": "agent_settled", "aborted": False},
]


def test_a_pi_reply_is_read_from_its_json_run():
    text, usage = trace.pi_json_reply("\n".join(json.dumps(e) for e in PI_JSON_RUN))
    assert text == "- use python -m pytest"
    assert usage == {"input_tokens": 745, "cache_read_input_tokens": 0,
                     "cache_creation_input_tokens": 0, "output_tokens": 298,
                     "cost_usd": 0.002235, "model": "anthropic/claude-haiku-4.5"}
    assert trace.pi_json_reply("") == ("", None)


@pytest.mark.real_sidecalls
def test_a_pi_side_call_is_a_bare_run_on_the_persons_own_pi(monkeypatch):
    import subprocess
    from tokeneconomizer.plugin import sidecall
    calls = []

    def fake_run(cmd, **kw):
        calls.append((cmd, kw))
        if "--no-mcp" in cmd:                       # Pi before 1.0 has no such flag
            return subprocess.CompletedProcess(cmd, 1, stdout="",
                                               stderr="Error: Unknown option: --no-mcp\n")
        out = "\n".join(json.dumps(e) for e in PI_JSON_RUN)
        return subprocess.CompletedProcess(cmd, 0, stdout=out, stderr="")

    monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
    text, eff = sidecall.pi_call("propose facts")
    assert text == "- use python -m pytest" and eff and eff > 0
    (first, kw), (second, _) = calls
    assert first[:4] == ["pi", "--print", "--mode", "json"] and first[-2:] == ["--", "propose facts"]
    for flag in ("--no-session", "--no-tools", "--no-extensions", "--no-context-files"):
        assert flag in first
    assert "--model" not in first                   # the person's own default model and login
    assert "--no-mcp" in first and "--no-mcp" not in second
    assert kw["env"]["TE_SIDECALL"] == "1" and kw["stdin"] is subprocess.DEVNULL
    calls.clear()
    sidecall.pi_call("propose facts", model="openrouter/anthropic/claude-haiku-4.5")
    assert calls[-1][0][-4:-2] == ["--model", "openrouter/anthropic/claude-haiku-4.5"]


@pytest.mark.real_sidecalls
def test_a_pi_side_call_loads_the_provider_extensions_and_the_pi_the_person_names(
        monkeypatch, tmp_path):
    """A model whose provider an extension registers is not there under `--no-extensions`; Pi
    still loads an explicit `--extension`, and `--no-tools` keeps the extension's tools out."""
    import subprocess
    from tokeneconomizer.plugin import sidecall
    calls = []

    def fake_run(cmd, **kw):
        calls.append(cmd)
        return subprocess.CompletedProcess(
            cmd, 0, stdout="\n".join(json.dumps(e) for e in PI_JSON_RUN), stderr="")

    monkeypatch.setattr(sidecall.subprocess, "run", fake_run)
    monkeypatch.chdir(tmp_path)
    cfg = live_config.load_config(str(tmp_path / "te"), env={
        "TE_PI_EXTENSIONS": "ext/gateway.ts, builtin:web", "TE_PI_CLI": "/opt/pi/bin/pi",
        "TE_PAID_MODEL": "gateway/claude-haiku"})
    assert not cfg.warnings
    text, _ = sidecall.runner_for("pi", cfg)("propose facts")
    cmd = calls[-1]
    assert text == "- use python -m pytest" and cmd[0] == "/opt/pi/bin/pi"
    assert "--no-extensions" in cmd and "--no-tools" in cmd
    assert [cmd[i + 1] for i, a in enumerate(cmd) if a == "--extension"] == [
        os.path.join(os.getcwd(), "ext", "gateway.ts"), "builtin:web"]  # the call runs elsewhere
    assert cmd[-4:] == ["--model", "gateway/claude-haiku", "--", "propose facts"]
    # unset, the call is what it was: the first `pi` on PATH, no extension
    sidecall.runner_for("pi", live_config.LiveConfig())("propose facts")
    assert calls[-1][0] == "pi" and "--extension" not in calls[-1]


def test_a_copilot_notice_is_a_progress_line_before_the_hook_output():
    import io
    from tokeneconomizer.plugin import hook
    out = io.StringIO()
    hook.main([], io.StringIO("{}"), out, {"TE_DISABLE": "1"})
    assert out.getvalue() == "{}\n"
    # what hook.main writes for an output carrying a Copilot notice
    from unittest import mock
    with mock.patch.object(hook, "run", return_value=dict(
            hosts.Copilot().notice("hello"), additionalContext="ctx")):
        out = io.StringIO()
        hook.main([], io.StringIO('{"session_id": "s"}'), out, {})
    first, last = out.getvalue().splitlines()
    assert json.loads(first) == {"type": "progress", "message": "hello"}
    assert json.loads(last) == {"additionalContext": "ctx"}


def test_copilot_side_calls_name_a_model_a_byok_provider_can_serve():
    from tokeneconomizer.plugin.sidecall import copilot_model
    byok = {"COPILOT_PROVIDER_BASE_URL": "https://api.anthropic.com",
            "COPILOT_PROVIDER_TYPE": "anthropic"}
    # `auto` is the subscription's router; a BYOK provider has none, and the call answered nothing
    assert copilot_model("", dict(byok, COPILOT_MODEL="auto")) == "claude-haiku-4-5"
    assert copilot_model("", dict(byok, COPILOT_MODEL="claude-sonnet-5")) == "claude-sonnet-5"
    assert copilot_model("claude-opus-5", byok) == "claude-opus-5"           # paid_model wins
    assert copilot_model("", {"COPILOT_MODEL": "auto"}) == ""                # subscription
    assert copilot_model("", {"COPILOT_PROVIDER_BASE_URL": "http://localhost:11434/v1"}) == ""
    # OpenRouter wants its own ids, whichever wire Copilot speaks to it
    for wire in ("anthropic", "openai"):
        assert copilot_model("", {"COPILOT_PROVIDER_BASE_URL": "https://openrouter.ai/api",
                                  "COPILOT_PROVIDER_TYPE": wire}) == "anthropic/claude-haiku-4.5"


@pytest.mark.skipif(os.name == "nt", reason="the probe trap is made of sh scripts")
def test_the_probe_repository_carries_its_trap(tmp_path):
    import subprocess
    from tokeneconomizer.plugin import probe
    repo, trap = probe.make_repo(str(tmp_path))
    bare = subprocess.run([os.path.join(trap, "pytest"), "-q"], cwd=repo, capture_output=True,
                          text=True)
    assert bare.returncode == 127 and "command not found" in bare.stderr
    # pytest is "not installed" for any interpreter the session runs, whatever really is
    for py in ("python", "python3"):
        mod = subprocess.run([os.path.join(trap, py), "-m", "pytest", "-q"], cwd=repo,
                             capture_output=True, text=True)
        assert mod.returncode == 1 and "No module named 'pytest'" in mod.stderr
    ok = subprocess.run([os.path.join(trap, "python"), "-m", "unittest", "-q"], cwd=repo,
                        capture_output=True, text=True)
    assert ok.returncode == 0 and "Ran 1 test" in ok.stderr
    assert subprocess.run(["git", "-C", repo, "status", "--porcelain"], capture_output=True,
                          text=True).stdout == ""                          # committed clean


def test_the_probe_counts_pytest_attempts_not_exit_codes():
    from tokeneconomizer.plugin import probe
    # commands two real Haiku sessions ran: the piped one exited 0 although pytest is missing
    runs, tried = probe.test_commands([
        'find . -name "pytest.ini" -o -name "go.mod" | head -20',
        "cd /r && python -m pytest tests/ -v 2>&1 | tail -20",
        "cd /r && python -m unittest discover -s tests -v 2>&1 | tail -20"])
    assert len(runs) == 2 and tried == 1
    runs, tried = probe.test_commands(["pytest -q", "python3 -m unittest", "ls tests/"])
    assert len(runs) == 2 and tried == 1
    assert probe.test_commands(['python -m unittest discover -s tests -p "test_*.py"']) == (
        ['python -m unittest discover -s tests -p "test_*.py"'], 0)


def test_a_small_envelopes_backstop_never_preempts_the_nudge():
    """Interactive repositories learn envelopes of ~4-10 calls, so the hard backstop
    (1.8-2.5x the envelope) lies inside the stuck rule's own floor: before this fix, the first
    stuck call tripped the backstop and the turn was stopped without ever being redirected."""
    from tokeneconomizer import loopguard as guard
    thr = guard.scaled_thresholds("aggressive")
    fail = ("FAIL: test_floats (tests.test_calc.TestAdd)\nTraceback (most recent call last):\n"
            "AssertionError: 0.30000000000000004 != 0.3\n\nFAILED (failures=1)\n")
    g = guard.TurnGuard(turn_key="t")
    calls = [("Bash", {"command": "python -m unittest"}, fail, 1)]
    for k in range(30):
        calls += [("Edit", {"file_path": "src/calc/__init__.py", "old_string": "x",
                            "new_string": f"y{k}"}, "ok", None),
                  ("Bash", {"command": "python -m unittest"}, fail, 1)]
    fired = []
    for i, (n, a, r, c) in enumerate(calls, 1):
        d = g.fold(n, a, r, c, envelope=4.0, thr=thr, nudge_budget=1)
        if d:
            fired.append((i, d[0]))
            if d[0] == "nudge":
                g.after_nudge()
            else:
                break
    assert fired == [(14, "nudge"), (28, "abort")]


@pytest.mark.parametrize("cmd", ["python tests/runtests.py model_inheritance -v 2",
                                 "./runtests.py basic", ".venv/bin/pytest -q tests/test_a.py",
                                 "cd web && ./node_modules/.bin/jest src", "/usr/bin/python3 -m pytest",
                                 "/w/venv/bin/pytest -q", "~/.local/bin/pytest -q",
                                 "python3 /w/repo/tests/runtests.py basic", "node --test test/"])
def test_a_runner_named_by_path_counts_as_a_verification(cmd):
    """A failing run after the last edit means the turn did not end green, whatever path the
    runner was called by -- otherwise long failing turns feed the envelope and the guard
    stops seeing them as long."""
    g = guard.TurnGuard(turn_key="t")
    g.fold("Edit", {"file_path": "/r/a.py", "old_string": "x", "new_string": "y"}, "ok")
    g.fold("Bash", {"command": cmd}, "FAILED (failures=1)", 1)
    assert g.verifications == 1 and not g.resolved_proxy()


@pytest.mark.parametrize("cmd", ["cat /w/pytest.ini", "cat pytest.ini", "ls /w/tests",
                                 "vim tests/runtests.py.bak"])
def test_a_file_named_like_a_runner_is_not_a_verification(cmd):
    assert not guard.VERIFY_RE.search(cmd)


TAP_FAILING = "# tests 9\n# suites 0\n# pass 8\n# fail 1\n# cancelled 0"


def test_a_test_run_whose_status_was_replaced_is_judged_by_its_output():
    """A pipe, `||` or a report of the status gives the command line another command's status:
    `npm test 2>&1 | tail -n 8` exits with `tail`'s 0 whatever the tests did."""
    g = guard.TurnGuard(turn_key="t")
    for _ in range(3):
        g.fold("bash", {"command": "npm test 2>&1 | tail -n 8"}, TAP_FAILING, 0)
    assert (g.whole.error_outputs, g.whole.repeated_fail_sigs) == (3, 2)
    assert not g.resolved_proxy()
    # Codex records a call as `bash -lc '<script>'`
    g = guard.TurnGuard(turn_key="t")
    g.fold("bash", {"command": "/bin/bash -lc 'cd /w && timeout 60 go test ./... 2>&1 | tail -3'"},
           "--- FAIL: TestAdd (0.00s)\nFAIL\tacme/x\t0.01s", 0)
    assert g.whole.error_outputs == 1 and not g.resolved_proxy()
    for cmd in ('python3 -m unittest tests.test_x; echo "exit=$?"', "npm test || true"):
        g = guard.TurnGuard(turn_key="t")
        g.fold("bash", {"command": cmd}, "FAILED (failures=1)\nexit=1", 0)
        assert g.whole.error_outputs == 1 and not g.resolved_proxy()
    # under pipefail the status is the tests' own; `| grep` finding nothing is no failure
    g = guard.TurnGuard(turn_key="t")
    g.fold("bash", {"command": "set -o pipefail; npm test | tail -n 8"}, TAP_FAILING, 0)
    g.fold("bash", {"command": "pytest -q | grep FAILED"}, "", 1)
    assert g.whole.error_outputs == 0 and g.resolved_proxy()


def test_a_test_run_followed_by_other_commands_is_judged_by_its_runners_summary():
    """`pytest -q; git diff` exits with `git`'s status, and its output holds the diff too: the
    runner's own summary line says how the tests went."""
    diff = "diff --git a/m.py b/m.py\n+    raise ValueError('no')\n"
    g = guard.TurnGuard(turn_key="t")
    g.fold("bash", {"command": "pytest -q; git diff"},
           "F.\n1 failed, 1 passed in 0.12s\n" + diff, 0)
    assert g.whole.error_outputs == 1 and not g.resolved_proxy()
    # passing tests, whatever the later command's status or the words in its output
    for code in (1, 0, None):
        g = guard.TurnGuard(turn_key="t")
        g.fold("bash", {"command": "pytest -q\ngit diff --exit-code"}, "2 passed in 0.10s\n" + diff,
               code)
        assert g.whole.error_outputs == 0 and g.resolved_proxy()


@pytest.mark.parametrize("out, failed", [
    ("1 failed, 4 passed, 2 warnings in 1.23s", True), ("== 1 error in 0.12s ==", True),
    ("2 passed, 1 warning in 0.30s", False), ("no tests ran in 0.01s", None),
    ("Ran 3 tests in 0.1s\n\nFAILED (failures=1)", True), ("Ran 3 tests in 0.1s\n\nOK", False),
    ("tests finished: 10 passed, 1 failed, in 3.20 seconds", True), (TAP_FAILING, True),
    ("# pass 9\n# fail 0", False), ("Tests:       1 failed, 3 passed, 4 total", True),
    ("  3 passing (20ms)\n  1 failing", True), ("--- FAIL: TestAdd (0.00s)", True),
    ("ok  \tacme/x\t0.012s", False), ("test result: FAILED. 1 passed; 1 failed", True),
    ("test result: ok. 3 passed; 0 failed", False), ("5 examples, 1 failure", True),
    ("Tests run: 4, Failures: 0, Errors: 1", True), ("[INFO] BUILD SUCCESS", False),
    ("+    raise ValueError('no')", None), ("ok 1 - add", None)])
def test_a_runners_summary_gives_its_verdict(out, failed):
    assert turn.runner_verdict(out) is failed


@pytest.mark.parametrize("cmd", ["tar -xOf pkg.tgz pkg/x.py | sed -n 1,80p",
                                 "cat tests/test_x.py | head -40", "grep -rn pytest src | head",
                                 "python -m pytest x.py --collect-only -q | rg test_",
                                 "bin/test --help | head", "python repro.py 2>&1 | tail",
                                 "pytest -q; git diff --stat", "pytest -q\npython - <<'PY'\nx\nPY"])
def test_any_other_command_keeps_its_exit_status(cmd):
    """Piped reads of source, and a test run among other commands with no runner's summary in
    the result, keep the status the agent reported: `assert` and `Error` in such output are no
    verdict."""
    g = guard.TurnGuard(turn_key="t")
    g.fold("bash", {"command": cmd}, "def f():\n    assert x\n    raise ValueError('no')\n", 0)
    assert g.whole.error_outputs == 0


@pytest.mark.parametrize("out, failing", [
    (TAP_FAILING, True), ("# pass 9\n# fail 0", False), ("not ok 3 - rates", True),
    ("  2 failing", True), ("5 examples, 0 failures", False), ("5 examples, 2 failures", True),
    ("Tests run: 4, Failures: 1, Errors: 0", True), ("Tests run: 4, Failures: 0, Errors: 0", False),
    ("--- FAIL: TestAdd (0.00s)", True), ("ok  \tacme/x\t0.01s", False)])
def test_other_ecosystems_failing_test_summaries_count_as_failures(out, failing):
    g = guard.TurnGuard(turn_key="t")
    g.fold("bash", {"command": "make test"}, out, None)
    assert (g.whole.error_outputs == 1) is failing


def test_a_unittest_failure_is_known_by_its_traceback():
    """unittest's `FAIL: test_x (...)` is not Go's `FAIL`: a Django failure is known by its
    traceback, not its test name, so the same failure in another test still repeats."""
    from tokeneconomizer.loopguard.state import _failure_sig
    out = "FAIL: test_x (validators.tests.T)\nTraceback (most recent call last):\n  ..."
    assert _failure_sig(out) == _failure_sig("Traceback (most recent call last):")


def test_codex_exec_and_copilot_p_are_told_from_their_terminal_uis(tmp_path, monkeypatch):
    """The still-stuck notice names the headless way to stop in `codex exec` and `copilot -p`.
    Codex says how a session was started only in its rollout's first record; Copilot only in its
    own command line (0.163.0, 1.0.101: the hook payloads and environments are the same)."""
    exec_log, tui_log = tmp_path / "rollout-exec.jsonl", tmp_path / "rollout-tui.jsonl"
    exec_log.write_text(json.dumps({"type": "session_meta", "payload": {
        "originator": "codex_exec", "source": "exec"}}) + "\n")
    tui_log.write_text(json.dumps({"type": "session_meta", "payload": {
        "originator": "codex-tui", "source": "vscode"}}) + "\n")
    c = hosts.Codex()
    assert c.headless({}, str(exec_log)) and not c.headless({}, str(tui_log))
    assert not c.headless({}, None) and not c.headless({}, str(tmp_path / "missing"))
    cp = hosts.Copilot()
    monkeypatch.setattr(hosts, "_ancestor_args", lambda: [
        "/bin/sh /p/scripts/te-hook --host copilot",
        "copilot -p Reply with the word ready. --model claude-haiku-4.5"])
    assert cp.headless({})
    monkeypatch.setattr(hosts, "_ancestor_args", lambda: ["copilot", "sh -c cd ~/proj && copilot"])
    assert not cp.headless({})
    monkeypatch.setattr(hosts, "_ancestor_args", lambda: ["copilot --prompt=fix it"])
    assert cp.headless({})
    assert cp.stop_hint(True) == "Press Ctrl+C to end the run."
    assert cp.stop_hint(False) == "Press Esc twice to interrupt it."