Replay your coding agent sessions as a map and a timeline
Python agent-observabilityai-agentsclaudeclaude-codecodexcoding-agentscursordeveloper-toolsdevtoolsgithub-copilotkirollmobservabilityopencodeprivacypythonsession-replaytypescriptvisualization
"""Structured creation requests must match successful native tool results."""
import json
import pytest
from agentplay._core.ingestion.coding_agent._sources.local.parsers.claude_code import parse_claude_code
from agentplay._core.ingestion.coding_agent._sources.local.parsers.codex import parse_codex
from agentplay._core.ingestion.coding_agent.produced import extract_created_prs
def parse(
agent,
output,
*,
name="mcp__github__create_pull_request",
failed=False,
completed=True,
tool_args=None,
return_spans=False,
):
args = {"owner": "Owner", "repo": "Repo", "head": "feature", "base": "main", "body": "x" * 3000}
if tool_args is not None:
args = tool_args
if agent == "claude":
rows = [
{"type": "user", "message": {"role": "user", "content": "open PR"}},
{
"type": "assistant",
"message": {
"role": "assistant",
"content": [
{"type": "tool_use", "id": "create", "name": name, "input": args},
],
},
},
]
if completed:
rows.append(
{
"type": "user",
"message": {
"role": "user",
"content": [
{
"type": "tool_result",
"tool_use_id": "create",
"content": output,
"is_error": failed,
},
],
},
}
)
parser = parse_claude_code
elif agent == "cursor":
from agentplay._core.ingestion.coding_agent._sources.local.parsers.cursor_store import (
parse_cursor_store,
)
rows = [
{"type": "cursor_store_session", "cwd": "/repo", "createdAt": 1790006400000},
{"role": "user", "content": "open PR"},
{
"role": "assistant",
"content": [
{"type": "tool-call", "toolCallId": "create", "toolName": name, "args": args}
],
},
]
if completed:
rows.append(
{
"role": "tool",
"content": [
{
"type": "tool-result",
"toolCallId": "create",
"result": output,
"isError": failed,
}
],
}
)
parser = parse_cursor_store
elif agent == "copilot":
from agentplay._core.ingestion.coding_agent._sources.local.parsers.copilot import parse_copilot
rows = [
{"type": "user.message", "data": {"content": "open PR"}},
{
"type": "tool.execution_start",
"data": {"toolCallId": "create", "toolName": name, "arguments": args},
},
]
if completed:
rows.append(
{
"type": "tool.execution_complete",
"data": {
"toolCallId": "create",
"success": not failed,
"result": {"content": output},
},
}
)
parser = parse_copilot
elif agent == "kiro":
from agentplay._core.ingestion.coding_agent._sources.local.parsers.kiro import parse_kiro
rows = [
{"type": "kiro_session", "id": "session"},
{"payload": {"type": "user", "content": "edit"}},
{
"payload": {
"type": "tool_call",
"toolCallId": "edit",
"toolName": name,
"args": args,
}
},
]
if completed:
rows.append(
{
"payload": {
"type": "tool_result",
"toolCallId": "edit",
"success": not failed,
"content": output,
}
}
)
parser = parse_kiro
elif agent == "opencode":
from agentplay._core.ingestion.coding_agent._sources.local.parsers.opencode import (
parse_opencode,
)
rows = [
{
"id": "u",
"message": {"role": "user", "time": {"created": 1000}},
"parts": [{"type": "text", "text": "open PR"}],
},
{
"id": "a",
"message": {"role": "assistant", "time": {"created": 2000}},
"parts": [
{
"type": "tool",
"tool": name,
"callID": "create",
"state": {
"input": args,
"output": output,
"status": "error"
if failed
else "completed"
if completed
else "running",
},
}
],
},
]
parser = parse_opencode
else:
rows = [
{"type": "session_meta", "payload": {"id": "session", "cwd": "/repo"}},
{
"type": "response_item",
"payload": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "open PR"}],
},
},
{
"type": "response_item",
"payload": {
"type": "function_call",
"call_id": "create",
"name": name,
"arguments": json.dumps(args),
},
},
]
if agent == "codex-wrapper":
rows[-1]["payload"].update(
name="exec", arguments="text(await tools." + name + "(" + json.dumps(args) + "));"
)
result = {"content": output} if isinstance(output, list) else json.loads(output)
output = [{"type": "text", "text": json.dumps(result)}]
if completed:
rows.append(
{
"type": "response_item",
"payload": {
"type": "function_call_output",
"call_id": "create",
"output": output,
},
}
)
parser = parse_codex
raw = b"\n".join(json.dumps({**r, "timestamp": "2026-09-21T00:00:00Z"}).encode() for r in rows)
_, spans = parser(raw, "session")
return spans if return_spans else extract_created_prs(spans)
@pytest.mark.parametrize(
"agent", ["claude", "codex", "codex-wrapper", "cursor", "copilot", "opencode"]
)
@pytest.mark.parametrize("wrapped", [False, True])
def test_successful_structured_creation_survives_display_truncation(agent, wrapped):
result = json.dumps(
{"body": "x" * 4000, "html_url": "https://github.com/owner/repo/pull/9", "number": 9}
)
if wrapped:
result = [{"type": "text", "text": result}]
[created] = parse(agent, result)
assert created.number == 9
assert created.repo_name == "owner/repo"
assert created.source_event_id
@pytest.mark.parametrize(
"agent", ["claude", "codex", "codex-wrapper", "cursor", "copilot", "opencode"]
)
@pytest.mark.parametrize(
"result",
[
{"html_url": "https://github.com/other/repo/pull/9", "number": 9},
{"html_url": "https://github.com/owner/repo/pull/9", "number": 8},
{"html_url": "https://github.com/owner/repo/pull/9", "error": "already exists"},
{"html_url": "https://github.com/owner/repo/pull/9", "isError": True},
{"html_url": "https://github.com/owner/repo/pull/9", "number": True},
],
)
def test_conflicting_or_failed_structured_result_never_links(agent, result):
assert parse(agent, json.dumps(result)) == []
@pytest.mark.parametrize(
"agent", ["claude", "codex", "codex-wrapper", "cursor", "copilot", "opencode"]
)
def test_reads_and_unfinished_creates_never_link(agent):
result = json.dumps({"html_url": "https://github.com/owner/repo/pull/9", "number": 9})
assert parse(agent, result, name="mcp__github__pull_request_read") == []
assert parse(agent, result, completed=False) == []
@pytest.mark.parametrize("agent", ["claude", "cursor", "copilot", "opencode"])
def test_failed_tool_result_never_links(agent):
assert (
parse(agent, json.dumps({"html_url": "https://github.com/owner/repo/pull/9"}), failed=True)
== []
)
@pytest.mark.parametrize(
("command", "expected"),
[
(
"gh api repos/owner/repo/pulls -X POST -f title=change -f head=feature -f base=main",
True,
),
("gh api /repos/owner/repo/pulls --method POST --input body.json", True),
("gh api repos/owner/repo/pulls -X GET", False),
("gh api repos/owner/repo/pulls -X POST; echo fake", False),
("gh api repos/owner/repo/pulls -X POST -f title=x;echo fake", False),
("gh api repos/owner/repo/pulls -X POST\necho fake", False),
("gh api repos/owner/repo/pulls -X POST --hostname github.example", False),
],
)
def test_explicit_rest_creation_requires_single_successful_request(command, expected):
from datetime import UTC, datetime
from agentplay._core.ingestion.coding_agent.produced import (
creation_command_attributes,
creation_output_attributes,
)
from agentplay._core.ingestion.coding_agent.spans import SpanData
request = creation_command_attributes({"command": command}, "Bash")
output = json.dumps({"html_url": "https://github.com/owner/repo/pull/9", "number": 9})
response = creation_output_attributes(
output, request.get("pr.create.provider"), request.get("pr.create.repository")
)
now = datetime.now(UTC)
span = SpanData(
"event",
"session",
None,
"Bash",
"TOOL",
"OK",
now,
now,
{
"tool.name": "Bash",
"input.value": json.dumps({"command": command}),
"output.value": output,
**request,
**response,
},
)
assert bool(extract_created_prs([span])) is expected
if expected:
span.attributes["tool.name"] = "Read"
assert extract_created_prs([span]) == []
span.attributes["tool.name"] = "Bash"
span.status_code = "ERROR"
assert extract_created_prs([span]) == []
def test_codex_shell_rest_creation_uses_completed_process_body():
args = {"cmd": "gh api repos/owner/repo/pulls -X POST -f head=feature -f base=main"}
response = json.dumps(
{"body": "x" * 4000, "html_url": "https://github.com/owner/repo/pull/9", "number": 9}
)
records = [
{"type": "session_meta", "payload": {"id": "session", "cwd": "/repo"}},
{
"type": "response_item",
"payload": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "create"}],
},
},
{
"type": "response_item",
"payload": {
"type": "function_call",
"name": "exec_command",
"call_id": "create",
"arguments": json.dumps(args),
},
},
{
"type": "response_item",
"payload": {
"type": "function_call_output",
"call_id": "create",
"output": json.dumps({"output": response, "exit_code": 0, "wall_time_seconds": 1}),
},
},
]
raw = b"\n".join(
json.dumps({**row, "timestamp": "2026-09-21T00:00:00Z"}).encode() for row in records
)
_, spans = parse_codex(raw, "session")
tool = next(s for s in spans if s.tool_name == "Bash")
assert tool.attributes["pr.create.output_url"] == "https://github.com/owner/repo/pull/9"
assert "https://github.com/owner/repo/pull/9" not in tool.attributes["output.value"]
assert [created.number for created in extract_created_prs(spans)] == [9]
@pytest.mark.parametrize(
"name",
[
"mcp__evil__create_pull_request",
"mcp__gitlab__create_pull_request",
"create_pull_request",
"githubish.create_pull_request",
],
)
def test_unknown_provider_never_becomes_github_creation(name):
assert (
parse(
"claude",
json.dumps({"html_url": "https://github.com/owner/repo/pull/9", "number": 9}),
name=name,
)
== []
)
@pytest.mark.parametrize("tool_name", ["Edit", "Write"])
@pytest.mark.parametrize("status", ["OK", "ERROR", "UNSET"])
def test_only_completed_mutations_count_as_useful_edits(tool_name, status):
from datetime import UTC, datetime
from agentplay._core.ingestion.coding_agent.metrics import compute_task_counters
from agentplay._core.ingestion.coding_agent.spans import SpanData
now = datetime.now(UTC)
span = SpanData(
"event",
"session",
"session:turn:1",
tool_name,
"TOOL",
status,
now,
now,
{"tool.name": tool_name, "input.value": json.dumps({"file_path": "src.py"})},
)
counters = compute_task_counters([span], start_turn=1, end_turn=1)
assert counters.useful_edit_count == int(status == "OK")
assert counters.error_count == int(status == "ERROR")
@pytest.mark.parametrize("failed", [False, True])
def test_copilot_preserves_mcp_error_envelope(failed):
from agentplay._core.ingestion.coding_agent._sources.local.parsers.copilot import parse_copilot
body = {
"isError": failed,
"content": [
{
"type": "text",
"text": json.dumps(
{"html_url": "https://github.com/owner/repo/pull/9", "number": 9}
),
}
],
}
rows = [
{
"type": "tool.execution_start",
"data": {
"toolCallId": "create",
"toolName": "mcp__github__create_pull_request",
"arguments": {"owner": "owner", "repo": "repo"},
},
},
{
"type": "tool.execution_complete",
"data": {"toolCallId": "create", "success": True, "result": body},
},
]
_, spans = parse_copilot(
b"\n".join(json.dumps({**r, "timestamp": "2026-09-21T00:00:00Z"}).encode() for r in rows),
"session",
)
assert bool(extract_created_prs(spans)) is not failed
@pytest.mark.parametrize(
"agent", ["claude", "codex", "cursor", "copilot", "opencode", "kiro"]
)
@pytest.mark.parametrize("name", ["Edit", "Write"])
@pytest.mark.parametrize("completed", [True, False])
def test_native_mutation_completion_controls_useful_edit_metrics(agent, name, completed):
from agentplay._core.ingestion.coding_agent.metrics import compute_task_counters
spans = parse(
agent,
"done",
name=name,
completed=completed,
tool_args={
"file_path": "src.py",
"old_string": "before",
"new_string": "after",
"content": "after",
},
return_spans=True,
)
[tool] = [s for s in spans if s.tool_name == name]
assert tool.status_code == ("OK" if completed else "UNSET")
counters = compute_task_counters(spans, start_turn=1, end_turn=1)
assert counters.useful_edit_count == int(completed)
@pytest.mark.parametrize("agent", ["claude", "codex", "cursor", "copilot", "opencode"])
@pytest.mark.parametrize(
"selector",
["-R owner/repo", "--repo=owner/repo", "-Rowner/repo", "--repo github.com/owner/repo"],
)
def test_explicit_shell_repository_must_match_created_url(agent, selector):
command = "gh pr create --body '" + "x" * 2500 + "' " + selector
args = {"command": command} if agent != "codex" else {"cmd": command}
name = "exec_command" if agent == "codex" else "bash"
good = "https://github.com/owner/repo/pull/9"
bad = "https://github.com/other/repo/pull/9"
assert len(parse(agent, good, name=name, tool_args=args)) == 1
assert parse(agent, bad, name=name, tool_args=args) == []
@pytest.mark.parametrize(
"command,accepted",
[
("GH_REPO=owner/repo gh pr create", True),
("GH_REPO=other/repo gh pr create", False),
("GH_REPO=other/repo gh pr create -R owner/repo", True),
("GH_HOST=enterprise.example gh pr create -R owner/repo", False),
("gh pr create -R owner/repo --repo other/repo", False),
("gh pr create --body '-R other/repo' -R owner/repo", True),
],
)
def test_shell_repository_selectors_and_body_text(command, accepted):
assert (
bool(
parse(
"claude",
"https://github.com/owner/repo/pull/9",
name="bash",
tool_args={"command": command},
)
)
is accepted
)
def test_opencode_retains_unsupported_creation_capture():
spans = parse(
"opencode",
"https://github.com/owner/repo/pull/9",
name="bash",
tool_args={"command": "python create_pr.py && gh pr create"},
return_spans=True,
)
assert any(s.attributes.get("pr.create.capture") == "unrecognized_creation_text" for s in spans)
@pytest.mark.parametrize("poll", [False, True])
def test_host_error_overrides_zero_exit_code(poll):
from agentplay._core.ingestion.coding_agent.native_operations import ProcessEvidence
evidence = ProcessEvidence()
origin = {"name": "exec_command", "pr.create.provider": "github", "end_ts": "now"}
if poll:
evidence.observe(
origin,
{"cmd": "gh pr create"},
{"session_id": 7, "exit_code": None, "wall_time_seconds": 0, "output": ""},
)
tool = {"name": "write_stdin", "end_ts": "later"} if poll else origin
evidence.observe(
tool,
{"session_id": 7} if poll else {"cmd": "gh pr create"},
{
"isError": True,
"exit_code": 0,
"wall_time_seconds": 0,
"output": "https://github.com/owner/repo/pull/9",
},
)
assert origin["is_error"] is True
assert origin["pr.create.output_url"] == ""
assert evidence.pending == {}