Skip to content

Replay your coding agent sessions as a map and a timeline

18.2 KB
Raw Blame
"""Tests for PR-creation / outcome / tool-category / file-op detection in builder.py.

Scope: gh/glab/GitHub-API/MCP PR creation detection (URL-
confirmed only), a closed-vocabulary tool-call category per event, file create/modify/
delete/rename detection, and a per-agent/session outcome label.
"""

from __future__ import annotations

import json

import pytest

from agentplay import builder as sm


def _line(**d):
    return json.dumps(d) + "\n"


def _repo(tmp_path, name="backend", remote="acme/backend"):
    repo = tmp_path / "work" / name
    (repo / ".git").mkdir(parents=True)
    (repo / ".git" / "config").write_text(
        f'[remote "origin"]\n\turl = [email protected]:{remote}.git\n'
    )
    return repo


def _bash(seq_id, ts, cwd, command, result, *, is_error=False):
    return [
        _line(
            type="assistant",
            timestamp=ts[0],
            cwd=cwd,
            message={
                "id": "m" + seq_id,
                "model": "claude-sonnet-5",
                "usage": {"input_tokens": 1, "output_tokens": 1},
                "content": [
                    {
                        "type": "tool_use",
                        "id": "t" + seq_id,
                        "name": "Bash",
                        "input": {"command": command},
                    }
                ],
            },
        ),
        _line(
            type="user",
            timestamp=ts[1],
            cwd=cwd,
            message={
                "role": "user",
                "content": [
                    {
                        "type": "tool_result",
                        "tool_use_id": "t" + seq_id,
                        "content": result,
                        "is_error": is_error,
                    }
                ],
            },
        ),
    ]


def _write_session(tmp_path, lines, name="s"):
    path = tmp_path / "proj" / f"{name}.jsonl"
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text("".join(line for group in lines for line in group))
    return path


# ---------------------------------------------------------------- PR creation
def test_successful_pr_create_resolves_from_url(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = [
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:02Z"),
            cwd,
            "gh pr create --repo acme/backend --base main --head feat/x "
            '--title "feat: add the thing" --body "## Why\\n\\nBecause it helps.\\n\\n- point one\\n- point two"',
            "https://github.com/acme/backend/pull/101",
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    ev = next(e for e in doc["events"] if e.get("vcs", {}).get("action") == "pr_create")
    assert ev["vcs"] == {
        "provider": "github",
        "action": "pr_create",
        "repo": "acme/backend",
        "number": 101,
    }
    assert doc["prs"] == [
        {
            "provider": "github",
            "repo": "acme/backend",
            "number": 101,
            "url": "https://github.com/acme/backend/pull/101",
            "seq": ev["seq"],
            "state": "open",
        }
    ]
    assert doc["totals"]["prsCreated"] == 1
    assert doc["session"]["outcome"] == "pr_created"


def test_failed_pr_create_is_not_counted_as_created(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = [
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:02Z"),
            cwd,
            'gh pr create --repo acme/backend --title "x" --body "some free text with a 1 in it"',
            "pull request create failed: GraphQL: already exists",
            is_error=True,
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    ev = next(e for e in doc["events"] if e.get("vcs"))
    assert ev["vcs"]["action"] == "pr_create_failed"
    assert doc["prs"] == [] and doc["totals"]["prsCreated"] == 0
    assert doc["session"]["outcome"] != "pr_created"


def test_pr_create_with_no_url_in_output_is_an_attempt_not_a_success(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = [
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:02Z"),
            cwd,
            'gh pr create --repo acme/backend --title "x" --body "free text mentioning 5 things"',
            "Opening in your browser.",
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    ev = next(e for e in doc["events"] if e.get("vcs"))
    assert ev["vcs"]["action"] == "pr_create_attempt"
    assert doc["prs"] == []


def test_ordinary_gh_pr_commands_never_read_as_create(tmp_path):
    """A generic noun/verb pair like ('pr','create') from vcs_action must not leak the
    literal sentinel 'pr_create' when pr_create_intent finds no create shape at all —
    regression for the collision between the generic label and the resolved-PR marker."""
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = (
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:02Z"),
            cwd,
            "gh pr view 101",
            '{"state":"OPEN"}',
        )
        + _bash(
            "2",
            ("2026-09-10T01:00:03Z", "2026-09-10T01:00:04Z"),
            cwd,
            "gh pr checks 101",
            "all pass",
        )
        + _bash(
            "3",
            ("2026-09-10T01:00:05Z", "2026-09-10T01:00:06Z"),
            cwd,
            "gh pr merge 101 --merge",
            "Merged",
        )
    )
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    actions = [e["vcs"]["action"] for e in doc["events"] if e.get("vcs")]
    assert actions == ["pr_view", "pr_checks", "pr_merge"]
    assert all(e["vcs"].get("number") == 101 for e in doc["events"] if e.get("vcs"))
    assert "pr_create" not in actions


def test_interrupted_pr_create_never_got_a_result(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = [
        _line(
            type="assistant",
            timestamp="2026-09-10T01:00:00Z",
            cwd=cwd,
            message={
                "id": "m1",
                "model": "claude-sonnet-5",
                "usage": {"input_tokens": 1, "output_tokens": 1},
                "content": [
                    {
                        "type": "tool_use",
                        "id": "t1",
                        "name": "Bash",
                        "input": {
                            "command": 'gh pr create --repo acme/backend --title "x" --body "y"'
                        },
                    }
                ],
            },
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    ev = next(e for e in doc["events"] if e.get("tool") == "Bash")
    assert ev["err"] is True and ev["errClass"] == "no_result"
    assert ev["vcs"]["action"] == "pr_create_failed"
    assert doc["prs"] == []


def test_bitbucket_and_gitlab_create_detected_by_shape(tmp_path):
    repo = _repo(tmp_path, "svc", "myorg/svc")
    cwd = str(repo)
    lines = _bash(
        "1",
        ("2026-09-10T01:00:00Z", "2026-09-10T01:00:02Z"),
        cwd,
        "glab mr create --title x --description y",
        "https://gitlab.com/myorg/svc/-/merge_requests/12",
    ) + _bash(
        "2",
        ("2026-09-10T01:00:03Z", "2026-09-10T01:00:04Z"),
        cwd,
        "bb pr create --title x",
        "https://bitbucket.org/myorg/svc/pull-requests/7",
    )
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    assert {(p["provider"], p["number"]) for p in doc["prs"]} == {("gitlab", 12), ("bitbucket", 7)}


def test_mcp_pr_tool_sets_vcs_category(tmp_path):
    cwd = str(_repo(tmp_path))
    lines = [
        _line(
            type="assistant",
            timestamp="2026-09-10T01:00:00Z",
            cwd=cwd,
            message={
                "id": "m1",
                "model": "claude-sonnet-5",
                "usage": {"input_tokens": 1, "output_tokens": 1},
                "content": [
                    {
                        "type": "tool_use",
                        "id": "t1",
                        "name": "mcp__github__create_pull_request",
                        "input": {"owner": "acme", "repo": "backend", "title": "x"},
                    }
                ],
            },
        ),
        _line(
            type="user",
            timestamp="2026-09-10T01:00:02Z",
            cwd=cwd,
            message={
                "role": "user",
                "content": [
                    {
                        "type": "tool_result",
                        "tool_use_id": "t1",
                        "content": "https://github.com/acme/backend/pull/102",
                    }
                ],
            },
        ),
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    ev = next(e for e in doc["events"] if e["kind"] == "mcp")
    assert ev["cat"] == "vcs" and ev["vcs"]["action"] == "pr_create" and ev["vcs"]["number"] == 102


# ---------------------------------------------------------------- shell verbs
@pytest.mark.parametrize(
    ("command", "verb"),
    [
        # Setup prefixes hand the verb to what follows, through any sequence operator.
        ("cd /x && make setup", "make setup"),
        ("cd /x; make setup", "make setup"),
        ("export AGENTPLAY_HOME=/x && python -m agentplay build", "python"),
        ("export FOO=1; pytest -q", "pytest"),
        ("cd /x && export Y=1 && pnpm dev", "pnpm dev"),
        # A newline sequences commands just as `;` does.
        ("cd /x\nmake setup", "make setup"),
        ("cd /x\nexport Y=1\npnpm dev", "pnpm dev"),
        # Shell keywords are not commands; the body's verb is. `do`/`then` may follow
        # a `;` or a newline, and blocks nest.
        ("for i in $(seq 1 60); do curl -s x; done", "curl"),
        ("if [ -f x ]; then rm x; fi", "rm"),
        ("for i in 1 2 3\ndo\n  curl x\ndone", "curl"),
        ("for i in 1 2; do if [ -f x ]; then echo up; fi; done", "echo"),
        # `$(...)` and backticks run their own commands; their words aren't this verb.
        ('P=$(uv tool dir)/agentplay; find $P -name "*.py"', "find"),
        ("M=$(ls /x | head -1)\npython3 -c 'import json'", "python3"),
        ("D=`date`; echo $D", "echo"),
        # A pipe is not a sequence: the head still owns the verb.
        ("cat x | grep y", "cat"),
        # A separator inside quotes belongs to an argument, not to the shell. Splitting
        # on it hands the verb to a fragment of that argument — here, of a path.
        ('cd "/private;token" && pytest', "pytest"),
        ("cd '/private;token' && pytest", "pytest"),
        ("cd /x; echo 'a&&b'", "echo"),
        ("cd /x\\;token && pytest", "pytest"),  # a backslash quotes the next character
        # Same rule for the `do`/`then` that opens a block body.
        ('if [ -f "a;then token" ]; then pytest -q; fi', "pytest"),
        ("for i in 'a;do token'; do curl -s x; done", "curl"),
        # Unchanged behaviour.
        ("pytest -q", "pytest"),
        ("AGENTPLAY_HOME=/x pytest -q", "pytest"),
        ("git status", "git status"),
        ("cd /x", "cd"),
        ("export FOO=1", "export"),
    ],
)
def test_shell_verb_skips_setup_prefixes_and_block_keywords(command, verb):
    assert sm._shell_verb(command) == verb


# ---------------------------------------------------------------- tool categories
@pytest.mark.parametrize(
    ("command", "cat"),
    [
        ("pytest -x tests/", "test"),
        # `_VERIFY_RE` (the base classifier, unchanged) buckets test/build/lint together as
        # kind="verify" — CAT labels that bucket "Test / verify", so a build lands there too.
        ("npm run build", "test"),
        ("npm install", "package"),
        ("aws s3 ls", "cloud"),
        ("curl -s https://x", "http"),
        ("rm -rf build/", "fs"),
        ("cat README.md", "file"),
        ("echo hi", "shell"),
        ("git status", "vcs"),
    ],
)
def test_tool_category_from_shell_verb(command, cat):
    kind, paths, touch, verb = sm.classify("Bash", {"command": command}, "/tmp")
    ev = {"kind": kind, "tool": "Bash", "verb": verb}
    assert sm.tool_category(ev) == cat


def test_tool_category_mcp_vcs_vs_other():
    assert sm.tool_category({"kind": "mcp", "server": "github"}) == "vcs"
    assert sm.tool_category({"kind": "mcp", "server": "claude-in-chrome"}) == "mcp"


# ---------------------------------------------------------------- file ops
def test_write_to_new_path_is_create_existing_path_is_modify(tmp_path):
    repo = _repo(tmp_path)
    existing = repo / "README.md"
    existing.write_text("hi\n")
    cwd = str(repo)
    lines = [
        _line(
            type="assistant",
            timestamp="2026-09-10T01:00:00Z",
            cwd=cwd,
            message={
                "id": "m1",
                "model": "claude-sonnet-5",
                "usage": {"input_tokens": 1, "output_tokens": 1},
                "content": [
                    {
                        "type": "tool_use",
                        "id": "t1",
                        "name": "Write",
                        "input": {"file_path": str(repo / "new_file.py"), "content": "x"},
                    },
                    {
                        "type": "tool_use",
                        "id": "t2",
                        "name": "Write",
                        "input": {"file_path": str(existing), "content": "y"},
                    },
                ],
            },
        ),
        _line(
            type="user",
            timestamp="2026-09-10T01:00:01Z",
            cwd=cwd,
            message={
                "role": "user",
                "content": [
                    {"type": "tool_result", "tool_use_id": "t1", "content": "ok"},
                    {"type": "tool_result", "tool_use_id": "t2", "content": "ok"},
                ],
            },
        ),
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    by_path = {f["path"]: f["op"] for f in doc["files"]}
    assert by_path["new_file.py"] == "create" and by_path["README.md"] == "modify"
    assert doc["totals"]["filesCreated"] == 1 and doc["totals"]["filesDeleted"] == 0


def test_shell_rm_mv_touch_detected(tmp_path):
    repo = _repo(tmp_path)
    (repo / "old.py").write_text("x\n")
    cwd = str(repo)
    lines = [
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:01Z"),
            cwd,
            "mv old.py new.py && touch fresh.py && rm stale.py",
            "",
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    by_path = {f["path"]: f["op"] for f in doc["files"]}
    assert by_path["old.py"] == "rename-from" and by_path["new.py"] == "rename-to"
    assert by_path["fresh.py"] == "create" and by_path["stale.py"] == "delete"


def test_apply_patch_add_delete_update(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    patch = (
        f"*** Add File: {repo}/a.py\n+x\n"
        f"*** Delete File: {repo}/b.py\n"
        f"*** Update File: {repo}/c.py\n-x\n+y\n"
    )
    lines = [
        _line(
            type="assistant",
            timestamp="2026-09-10T01:00:00Z",
            cwd=cwd,
            message={
                "id": "m1",
                "model": "claude-sonnet-5",
                "usage": {"input_tokens": 1, "output_tokens": 1},
                "content": [
                    {
                        "type": "tool_use",
                        "id": "t1",
                        "name": "apply_patch",
                        "input": {"input": patch},
                    }
                ],
            },
        ),
        _line(
            type="user",
            timestamp="2026-09-10T01:00:01Z",
            cwd=cwd,
            message={
                "role": "user",
                "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "applied"}],
            },
        ),
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    by_path = {f["path"]: f["op"] for f in doc["files"]}
    assert by_path == {"a.py": "create", "b.py": "delete", "c.py": "modify"}


def test_shell_redirect_creates_file_but_numeric_comparison_does_not(tmp_path):
    """`> 1` inside a numeric/string test (`[ "$rc" > 1 ]`) must never be read as a
    redirect to a file literally named '1' — the two are only distinguishable by this
    kind of conservative guard, not by the redirect grammar alone."""
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = [
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:01Z"),
            cwd,
            'echo hi > out.log\nrc=$?\nif [ "$rc" > 1 ]; then\n  echo bad\nfi',
            "",
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    paths = {f["path"] for f in doc["files"]}
    assert "out.log" in paths
    assert "1" not in paths


def test_fstring_alignment_spec_not_read_as_redirect(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = [
        _bash(
            "1",
            ("2026-09-10T01:00:00Z", "2026-09-10T01:00:01Z"),
            cwd,
            "python3 -c \"print(f'{x:>7.0f} rows')\"",
            "  12 rows",
        )
    ]
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    assert doc["files"] == []


# ---------------------------------------------------------------- outcome
def test_outcome_verified_when_last_verify_has_no_later_edit(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = _bash(
        "1", ("2026-09-10T01:00:00Z", "2026-09-10T01:00:01Z"), cwd, "pytest -q", "12 passed"
    )
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    assert doc["session"]["outcome"] == "verified"


def test_outcome_unresolved_errors_when_session_ends_on_failures(tmp_path):
    repo = _repo(tmp_path)
    cwd = str(repo)
    lines = _bash(
        "1",
        ("2026-09-10T01:00:00Z", "2026-09-10T01:00:01Z"),
        cwd,
        "pytest -q",
        "3 failed",
        is_error=True,
    ) + _bash(
        "2",
        ("2026-09-10T01:00:02Z", "2026-09-10T01:00:03Z"),
        cwd,
        "pytest -q -k thing",
        "1 failed",
        is_error=True,
    )
    doc = sm.build_claude_code_map(_write_session(tmp_path, lines))
    assert doc["session"]["outcome"] == "unresolved_errors"


def test_outcome_incomplete_on_empty_session():
    assert sm.classify_outcome([], "") == "incomplete"