Replay your coding agent sessions as a map and a timeline
Python agent-observabilityai-agentsclaudeclaude-codecodexcoding-agentscursordeveloper-toolsdevtoolsgithub-copilotkirollmobservabilityopencodeprivacypythonsession-replaytypescriptvisualization
"""Session facts (`facts.py`), their backfill, and `GET /api/stats`."""
from __future__ import annotations
import http.client
import json
import threading
from pathlib import Path
from typing import Any
import pytest
from agentplay import facts, pipeline
from agentplay.server import serve
from agentplay.store import Store
DEMO = Path(__file__).resolve().parents[2] / "viewer" / "public" / "demo"
def _ev(seq: int, kind: str, agent: str = "main", **kw: Any) -> dict[str, Any]:
return {"seq": seq, "t": seq * 1000, "agent": agent, "err": False, "kind": kind, **kw}
def _llm(seq: int, cost: float, agent: str = "main", **kw: Any) -> dict[str, Any]:
return _ev(seq, "llm", agent, model="claude-sonnet-4-5", cost=cost, **{"in": 100, "cw": 0, "cr": 900, "out": 50, **kw})
def _doc(events: list[dict[str, Any]], *, outcome: str = "completed", sid: str = "s1",
started: str = "2026-10-01T10:00:00Z", agents: list[dict[str, Any]] | None = None,
totals: dict[str, Any] | None = None, role: str = "root") -> dict[str, Any]:
return {
"schema": "agentplay/v1",
"session": {"id": sid, "agentTool": "claude-code", "startedAt": started, "durationMs": 60000,
"models": ["claude-sonnet-4-5"], "outcome": outcome, "role": role},
"totals": {"calls": 5, "requests": 3, "userTurns": 2, "subagents": 0, "errors": 1, "tokens": 0,
"costUsd": sum(e.get("cost", 0) for e in events), "filesTouched": 2, "prsCreated": 0,
**(totals or {})},
"repos": [{"id": "r0", "name": "acme/api"}],
"files": [{"id": 0, "repo": "r0", "path": "src/a.ts", "reads": 2, "edits": 1, "hits": 3},
{"id": 1, "repo": "r0", "path": "src/unused.ts", "reads": 0, "edits": 0, "hits": 0}],
"agents": agents or [{"id": "main", "kind": "main"}],
"events": events,
}
def test_commands_files_prompt_and_verify():
doc = _doc([
_ev(0, "user", chars=420),
_llm(1, 0.10),
_ev(2, "verify", verb="pytest", err=True, errClass="exit_code"),
_llm(3, 0.20),
_ev(4, "verify", verb="pytest", retryOf=2),
_ev(5, "edit", targets=[{"file": 0, "touch": "edit"}]),
_ev(6, "verify", verb="pytest"),
_ev(7, "user", chars=30),
])
f = facts.compute(doc)
assert f["cmd"]["pytest"] == {"calls": 3, "fails": 1, "retries": 1, "byClass": {"exit_code": 1},
"verify": True, "outBytes": 0}
assert f["files"] == {"acme/api:src/a.ts": {"hits": 3, "reads": 2, "edits": 1}} # untouched files dropped
assert f["firstPromptChars"] == 420 and f["userTurns"] == 2
assert f["edited"] is True and f["verifyAfterLastEdit"] is True
assert f["errorsBySource"] == {"tool": {"exit_code": 1}}
def test_waste_charges_each_request_once_in_bucket_order():
# The failed call charges the next request as a retry; the rest of an
# abandoned session is abandoned spend, never both.
doc = _doc([
_llm(0, 0.5),
_ev(1, "exec", verb="make", err=True, errClass="exit_code"),
_llm(2, 1.0),
_llm(3, 2.0),
], outcome="unresolved_errors")
w = facts.compute(doc)["waste"]
assert w["retry"] == pytest.approx(1.0)
assert w["abandoned"] == pytest.approx(2.5)
assert w["cacheMiss"] == 0.0 # every request already charged
def test_rereads_and_dead_subagents():
doc = _doc([
_ev(0, "read", targets=[{"file": 0, "touch": "read"}]),
_llm(1, 0.1),
_ev(2, "compaction", preTokens=150000),
_ev(3, "read", targets=[{"file": 0, "touch": "read"}]),
_llm(4, 0.3),
_ev(5, "subagent", err=True, errClass="no_result"),
_llm(6, 0.7, agent="sub-1"),
], agents=[{"id": "main", "kind": "main"}, {"id": "sub-1", "kind": "subagent", "launchSeq": 5}])
f = facts.compute(doc)
assert f["rereadsAfterCompaction"] == 1
assert f["preTokens"] == [150000]
assert f["waste"]["reread"] == pytest.approx(0.3)
assert f["waste"]["deadSubagent"] == pytest.approx(0.7)
def test_stamp_survives_a_map_missing_fields():
assert facts.stamp({"events": [{"kind": "llm"}]}) == {"facts": None, "factsVersion": facts.FACTS_VERSION}
def test_demo_index_facts_are_current():
"""Regenerate with `scripts/demo_facts.py` when this fails."""
index = json.loads((DEMO / "index.json").read_text(encoding="utf-8"))
for row in index["sessions"]:
doc = json.loads((DEMO / row["file"]).read_text(encoding="utf-8"))
assert row.get("factsVersion") == facts.FACTS_VERSION, row["file"]
assert row.get("facts") == facts.compute(doc), row["file"]
def _seed(tmp_path: Path, docs: list[dict[str, Any]], *, with_facts: bool) -> Store:
st = Store(tmp_path)
for d in docs:
row = {"sessionId": d["session"]["id"], "sessionDate": d["session"]["startedAt"][:10],
"startedAt": d["session"]["startedAt"], "agentTool": "claude-code", "models": [],
"repos": ["acme/api"], "calls": 5, "role": d["session"].get("role"), "contentHash": d["session"]["id"]}
if with_facts:
row.update(facts.stamp(d))
st.put(d, row)
st.save_index()
return st
def test_unchanged_sessions_get_facts_backfilled(tmp_path):
st = _seed(tmp_path, [_doc([_llm(0, 0.1)])], with_facts=False)
assert pipeline._sync_facts(st, "s1") is True
assert st.index["s1"]["factsVersion"] == facts.FACTS_VERSION
assert pipeline._sync_facts(st, "s1") is False # already current
@pytest.fixture()
def stats_server(tmp_path):
docs = [
_doc([_llm(0, 0.1)], sid="old", started="2026-09-01T10:00:00Z"),
_doc([_llm(0, 0.2)], sid="new", started="2026-10-02T10:00:00Z", totals={"prsCreated": 1}),
_doc([_llm(0, 0.3)], sid="child", started="2026-10-02T11:00:00Z", role="subagent"),
]
st = _seed(tmp_path, docs, with_facts=False) # rows written before facts existed
httpd = serve(0, store=st)
threading.Thread(target=httpd.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True).start()
try:
yield httpd, st
finally:
httpd.shutdown()
httpd.server_close()
def _get(httpd, path: str, headers: dict[str, str] | None = None):
conn = http.client.HTTPConnection("127.0.0.1", httpd.server_address[1], timeout=5)
conn.request("GET", path, headers=headers or {})
resp = conn.getresponse()
body = resp.read()
conn.close()
return resp.status, json.loads(body)
def test_stats_backfills_in_memory_and_filters(stats_server):
httpd, st = stats_server
status, body = _get(httpd, "/api/stats")
assert status == 200
ids = {s["sessionId"] for s in body["sessions"]}
assert ids == {"old", "new"} # subagent sessions are folded into their parent
new = next(s for s in body["sessions"] if s["sessionId"] == "new")
assert new["facts"]["prsCreated"] == 1
assert "facts" not in json.dumps(st.index["new"]) # the server never writes the index
_, body = _get(httpd, "/api/stats?from=2026-10-01")
assert [s["sessionId"] for s in body["sessions"]] == ["new"]
def test_stats_and_sessions_keep_facts_apart(stats_server):
httpd, _ = stats_server
_, page = _get(httpd, "/api/sessions")
assert all("facts" not in s for s in page["sessions"])
def test_stats_refuses_a_rebinding_host(stats_server):
httpd, _ = stats_server
status, _ = _get(httpd, "/api/stats", {"Host": "evil.example:80"})
assert status == 403