Replay your coding agent sessions as a map and a timeline
Python agent-observabilityai-agentsclaudeclaude-codecodexcoding-agentscursordeveloper-toolsdevtoolsgithub-copilotkirollmobservabilityopencodeprivacypythonsession-replaytypescriptvisualization
"""Cohort regressions: literal wrappers, process completion and shell bodies."""
import json
import pytest
from agentplay._core.ingestion.coding_agent._sources.local.parsers.codex import parse_codex
from agentplay._core.ingestion.coding_agent.native_operations import literal_operations
from agentplay._core.ingestion.coding_agent.produced import (
creation_command_attributes,
extract_created_prs,
)
@pytest.mark.parametrize(
"code",
[
'text(await tools.exec_command({cmd:"gh pr create --fill"}));',
'const r = await tools.exec_command({"cmd":"gh pr create --fill"}); text(r);',
],
)
def test_literal_wrapper(code):
assert literal_operations(code) == [("exec_command", {"cmd": "gh pr create --fill"})]
@pytest.mark.parametrize(
"code",
[
'if (false) { text(await tools.exec_command({cmd:"gh pr create"})); }',
"text(await tools.exec_command({cmd:command}));",
'text(await Promise.all([tools.exec_command({cmd:"gh pr create"})]));',
'text(await tools.exec_command({cmd:"gh pr create"})); text("fake");',
'text(await tools.exec_command({cmd:"gh pr create", cmd:"echo fake"}));',
'const tools = await tools.exec_command({cmd:"gh pr create"}); text(tools);',
],
)
def test_unsupported_scripts_cannot_claim_operations(code):
assert literal_operations(code) == []
@pytest.mark.parametrize(
"command",
[
"cd /repo\ngh pr create --body \"$(cat <<'EOF'\nbody `not code`\nEOF\n)\" 2>&1",
"cat > /tmp/body.md <<'EOF'\nbody\nEOF\ngh pr create --body-file /tmp/body.md",
"git push -u origin feature && gh pr create --fill",
"cd /repo && git add -A && git commit -m \"$(cat <<'EOF'\nfix: x\n\nbody\nEOF\n)\" "
"&& git push -u origin HEAD && gh pr create --title t --body b",
],
)
def test_supported_cohort_shell_shapes(command):
assert creation_command_attributes(command)["pr.create.provider"] == "github"
@pytest.mark.parametrize(
"command",
[
"gh auth switch --user user1 2>&1 | tail -1\ngh pr create --fill\ngh auth switch --user user2 2>&1",
# git preparation is accepted only as `&&`-joined add/commit/push
# BEFORE the one create, with no redirect, pipe or fallback of its own.
"git push; gh pr create --fill",
"git push\ngh pr create --fill",
"git push || true && gh pr create --fill",
"git push 2>&1 | tail -1 && gh pr create --fill",
"git push > /tmp/out && gh pr create --fill",
"gh pr create --fill && git push",
"git log && gh pr create --fill",
"git push && gh pr create --web",
"git push && gh pr create --fill | cat",
"git push && gh pr create --fill && gh pr create --fill",
"git diff && gh pr create --fill | head -n0",
"echo 'gh pr create'",
"false || gh pr create",
"gh pr create; echo https://github.com/acme/repo/pull/1",
'gh pr create --body "$(arbitrary-command)"',
"cat <<EOF\ngh pr create\nEOF",
"gh pr create && gh pr view 2",
],
)
def test_noncausal_shell_shapes(command):
assert "pr.create.provider" not in creation_command_attributes(command)
def _parse(exit_code=0, chars="", launch_output="", final_output=None, host_error=False):
url = "https://github.com/acme/repo/pull/7"
rows = [
{"type": "session_meta", "payload": {"id": "native", "cwd": "/repo"}},
{
"type": "response_item",
"payload": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "make a change"}],
},
},
]
for call_id, code, result in [
(
"launch",
'text(await tools.exec_command({cmd:"gh pr create --fill"}));',
{"output": launch_output, "session_id": 42, "wall_time_seconds": 1},
),
(
"poll",
"text(await tools.write_stdin("
+ json.dumps({"session_id": 42, "chars": chars})
+ "));",
{
"output": final_output if final_output is not None else url,
"exit_code": exit_code,
"isError": host_error,
"wall_time_seconds": 1,
},
),
]:
rows.extend(
[
{
"type": "response_item",
"payload": {
"type": "custom_tool_call",
"name": "exec",
"call_id": call_id,
"input": code,
},
},
{
"type": "response_item",
"payload": {
"type": "custom_tool_call_output",
"call_id": call_id,
"output": [{"type": "text", "text": json.dumps(result)}],
},
},
]
)
for index, row in enumerate(rows):
row["timestamp"] = f"2026-09-21T01:00:{index:02d}Z"
_, spans = parse_codex(
data="\n".join(json.dumps(r) for r in rows).encode(), trace_id="local:codex:native"
)
return extract_created_prs(spans)
def test_async_completion_keeps_launch_identity():
result = _parse()
assert len(result) == 1
assert result[0].source_event_id.endswith(":tool:launch:op:0")
assert result[0].number == 7
assert result[0].command_sha256
@pytest.mark.parametrize(
"kwargs",
[
{"exit_code": 1},
{"host_error": True},
{"chars": "echo fake\n"},
{"exit_code": None},
{"launch_output": "error: creation failed\n"},
{"launch_output": "https://github.com/acme/repo/pull/8\n"},
],
)
def test_async_failure_or_ambiguity_never_creates_link(kwargs):
assert _parse(**kwargs) == []
def test_output_printed_before_exit_requires_successful_completion():
assert len(_parse(launch_output="https://github.com/acme/repo/pull/7\n", final_output="")) == 1
def test_native_text_process_headers_are_not_read_from_stdout():
from agentplay._core.ingestion.coding_agent.native_operations import shell_result
result = shell_result("Chunk ID: abc\nProcess running with session ID 17\nOutput:\nwaiting")
assert result["session_id"] == 17
assert result["exit_code"] is None
assert shell_result("Chunk ID: abc\nOutput:\nProcess exited with code 0") is None
def test_serial_native_input_text_result_header():
from agentplay._core.ingestion.coding_agent.native_operations import literal_results
blocks = [
{"type": "input_text", "text": "Script completed\nWall time 0.1 seconds\nOutput:\n"},
{"type": "input_text", "text": '{"exit_code":0,"wall_time_seconds":0.1,"output":"done"}'},
]
assert literal_results(blocks, 1)[0]["exit_code"] == 0
assert literal_results([*blocks, {"type": "input_text", "text": "extra"}], 1) == []
def test_json_exponent_literal_preserves_operation():
assert literal_operations(
'text(await tools.exec_command({"cmd":"pwd","yield_time_ms":1e3}));'
) == [("exec_command", {"cmd": "pwd", "yield_time_ms": 1000})]
@pytest.mark.parametrize(
"result",
[
{"exit_code": 1, "output": "failed"},
{"isError": True, "output": "failed"},
{"metadata": {"exit_code": 1}, "output": "failed"},
],
)
def test_failed_literal_patch_never_captures_edits(result):
patch = "*** Begin Patch\n*** Update File: /repo/app.py\n@@\n-old\n+new\n*** End Patch"
records = [
{"type": "session_meta", "payload": {"id": "native", "cwd": "/repo"}},
{
"type": "response_item",
"payload": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "edit"}],
},
},
{
"type": "response_item",
"payload": {
"type": "custom_tool_call",
"name": "exec",
"call_id": "patch",
"input": "text(await tools.apply_patch(" + json.dumps(patch) + "));",
},
},
{
"type": "response_item",
"payload": {
"type": "custom_tool_call_output",
"call_id": "patch",
"output": [{"type": "text", "text": json.dumps(result)}],
},
},
]
for index, row in enumerate(records):
row["timestamp"] = f"2026-09-21T01:00:{index:02d}Z"
_, spans = parse_codex(
data="\n".join(json.dumps(row) for row in records).encode(), trace_id="native"
)
operation = next(span for span in spans if span.span_id.endswith(":op:0"))
assert operation.status_code == "ERROR"
assert operation.attributes["code.edit_evidence"]["edits"] == []
def test_nested_arguments_do_not_abort_codex_transcript():
rows = [
{"type": "session_meta", "payload": {"id": "native", "cwd": "/repo"}},
{"type": "response_item", "payload": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "work"}]}},
{"type": "response_item", "payload": {"type": "function_call", "name": "exec_command", "call_id": "nested", "arguments": "[" * 2000 + "0" + "]" * 2000}},
{"type": "response_item", "payload": {"type": "function_call_output", "call_id": "nested", "output": '{"output":"done","exit_code":0}'}},
]
_, spans = parse_codex("\n".join(json.dumps(row) for row in rows).encode(), trace_id="native")
assert any(span.span_id.endswith(":tool:nested") for span in spans)
assert extract_created_prs(spans) == []
_PATCH = "*** Begin Patch\n*** Update File: /repo/app.py\n@@\n-old\n+new\n*** End Patch"
@pytest.mark.parametrize(
"code",
[
"const patch = PATCH; text(await tools.apply_patch(patch));",
"const patch=PATCH;text(await tools.apply_patch(patch))",
"const patch = PATCH; const r = await tools.apply_patch(patch); "
'text(typeof r === "string" ? r : JSON.stringify(r));',
"const p=PATCH;const r=await tools.apply_patch(p);"
'text(typeof r==="string"?r:JSON.stringify(r))',
],
)
def test_codex_bound_patch_is_a_literal_operation(code):
assert literal_operations(code.replace("PATCH", json.dumps(_PATCH))) == [
("apply_patch", _PATCH)
]
@pytest.mark.parametrize(
"code",
[
"let patch = PATCH; patch = PATCH; text(await tools.apply_patch(patch));",
"const patch = PATCH + PATCH; text(await tools.apply_patch(patch));",
"text(await tools.apply_patch(patch)); const patch = PATCH;",
"const patch = PATCH; const patch = PATCH; text(await tools.apply_patch(patch));",
"const true = PATCH; text(await tools.apply_patch(true));",
"const p = PATCH; for (const x of [1]) { text(await tools.apply_patch(p)); }",
# Emits something other than the awaited result: order proves nothing.
"const p = PATCH; const r = await tools.apply_patch(p); "
'text(typeof r === "string" ? p : JSON.stringify(r));',
"const p = PATCH; const r = await tools.apply_patch(p);",
],
)
def test_codex_bound_literal_shapes_that_prove_nothing(code):
assert literal_operations(code.replace("PATCH", json.dumps(_PATCH))) == []
def test_codex_bound_patch_captures_edit_evidence():
code = "const patch = " + json.dumps(_PATCH) + "; text(await tools.apply_patch(patch));"
records = [
{"type": "session_meta", "payload": {"id": "native", "cwd": "/repo"}},
{
"type": "response_item",
"payload": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "edit"}],
},
},
{
"type": "response_item",
"payload": {"type": "custom_tool_call", "name": "exec", "call_id": "c", "input": code},
},
{
"type": "response_item",
"payload": {
"type": "custom_tool_call_output",
"call_id": "c",
"output": [
{
"type": "input_text",
"text": "Script completed\nWall time 0.1 seconds\nOutput:\n",
},
{"type": "input_text", "text": "{}"},
],
},
},
]
for index, row in enumerate(records):
row["timestamp"] = f"2026-09-21T01:00:{index:02d}Z"
_, spans = parse_codex(
data="\n".join(json.dumps(row) for row in records).encode(), trace_id="native"
)
operation = next(span for span in spans if span.span_id.endswith(":op:0"))
assert operation.attributes["provenance.capture"] == "literal_operation"
evidence = operation.attributes["code.edit_evidence"]
assert evidence["status"] == "captured"
assert [(e["path"], e["operation"]) for e in evidence["edits"]] == [("/repo/app.py", "replace")]