Enterprise agent platform for building, deploying, and operating AI teammates.
"""Admin execution-event diagnostics: correctness, auth, and payload secrecy."""
from __future__ import annotations
import json
import logging
from datetime import datetime, timedelta, timezone
from importlib import import_module
from types import SimpleNamespace
from typing import Any
import sqlalchemy as sa
from fastapi import FastAPI
from fastapi.testclient import TestClient
from sqlalchemy.exc import OperationalError
from sqlalchemy.orm import Session
from tests.web.services.task_database_shared import engine as engine_fixture
from tests.web.services.task_database_shared import task_id as task_id_fixture
from xagent.web.api import admin_execution_events as api_module
from xagent.web.api.admin_execution_events import router
from xagent.web.auth_dependencies import get_current_user
from xagent.web.models.database import get_db
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_auto_recovery import TaskAutoRecovery, TaskRecoveryEvent
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.services import execution_event_diagnostics as diagnostics_module
from xagent.web.services.execution_event_diagnostics import (
task_execution_event_diagnostics,
)
from xagent.web.services.ops_signals import (
clear_degradation,
register_degradation,
)
from xagent.web.services.task_execution_event_writer import append_fact_no_commit
engine = engine_fixture
task_id = task_id_fixture
SENTINEL = "T3A-SENTINEL"
T0 = datetime(2026, 9, 5, tzinfo=timezone.utc)
def fact(
db: Session,
task_id: int,
kind: str,
key: str,
*,
scope: str = "root",
run: str | None = None,
attempt: str | None = None,
) -> None:
append_fact_no_commit(
db,
task_id=task_id,
scope_id=scope,
run_id=run,
tool_attempt_id=attempt,
assistant_message_id="batch-1" if attempt else None,
kind=kind,
key=key,
payload={"data": {"secret": SENTINEL, "tool_params": {"q": SENTINEL}}},
occurred_at=T0 + timedelta(seconds=len(key)),
)
def make_v2(db: Session, task_id: int) -> None:
task = db.get(Task, task_id)
task.conversation_storage_version = 2
db.flush()
def diagnose(engine, task_id: int) -> dict[str, Any]:
with Session(engine) as db:
result = task_execution_event_diagnostics(db, task_id)
assert result is not None
return result
def build_client(engine, *, is_admin: bool) -> TestClient:
app = FastAPI()
app.include_router(router)
def test_db():
with Session(engine) as db:
yield db
app.dependency_overrides[get_db] = test_db
app.dependency_overrides[get_current_user] = lambda: SimpleNamespace(
id=1, is_admin=is_admin
)
return TestClient(app)
def url(task_id: int) -> str:
return f"/api/admin/execution-events/tasks/{task_id}"
def seed_two_runs(engine, task_id: int) -> None:
with Session(engine) as db:
make_v2(db, task_id)
fact(db, task_id, "input_accepted", "in-1", run="r1")
fact(db, task_id, "recovery_state", "rs-1", run="r1")
fact(db, task_id, "execution_settled", "result-r1", run="r1")
fact(db, task_id, "recovery_state", "rs-2", run="r2")
fact(db, task_id, "agent_message", "msg-1", run="r2")
fact(db, task_id, "recovery_state", "rs-child", scope="child-1", run="r2")
fact(db, task_id, "assistant_message", "am-1", scope="child-1", run="r2")
fact(db, task_id, "input_accepted", "in-2")
db.commit()
def test_non_admin_is_forbidden(engine, task_id):
response = build_client(engine, is_admin=False).get(url(task_id))
assert response.status_code == 403
assert response.json()["detail"] == "Admin access required"
def test_missing_task_is_404(engine, task_id):
response = build_client(engine, is_admin=True).get(url(task_id + 1000))
assert response.status_code == 404
def test_router_is_registered_in_the_app():
app_module = import_module("xagent.web.app")
# No ``with``: no lifespan startup. An unauthenticated call is refused by
# the auth dependency; an unregistered path would be 404.
response = TestClient(app_module.app).get(url(1))
assert response.status_code in (401, 403)
def test_v2_task_counts_runs_scopes_and_recovery_states(engine, task_id):
seed_two_runs(engine, task_id)
with Session(engine) as db:
task = db.get(Task, task_id)
task.status = TaskStatus.RUNNING
task.run_id = "r2"
task.runner_id = "runner-a"
task.lease_attempt_id = "lease-1"
task.lease_expires_at = T0
task.last_heartbeat_at = T0
task.error_message = f"boom {SENTINEL}"
db.commit()
result = build_client(engine, is_admin=True).get(url(task_id)).json()
assert result["storage"] == "v2"
assert result["task"] == {
"id": task_id,
"conversation_storage_version": 2,
"status": "running",
"control_state": "idle",
"run_id": "r2",
"state_version": 0,
"conversation_event_sequence": 8,
"runner_id": "runner-a",
"lease_attempt_id": "lease-1",
"lease_expires_at": result["task"]["lease_expires_at"],
"last_heartbeat_at": result["task"]["last_heartbeat_at"],
"has_error_message": True,
}
assert result["task"]["lease_expires_at"].startswith("2026-09-05")
assert "error_message" not in result["task"]
events = result["events"]
assert (events["total"], events["min_sequence"], events["max_sequence"]) == (
8,
1,
8,
)
assert events["latest_occurred_at"] is not None
assert events["by_scope_kind_truncated"] is False
assert events["by_scope_kind"] == [
{"scope_id": "child-1", "kind": "assistant_message", "count": 1},
{"scope_id": "child-1", "kind": "recovery_state", "count": 1},
{"scope_id": "root", "kind": "agent_message", "count": 1},
{"scope_id": "root", "kind": "execution_settled", "count": 1},
{"scope_id": "root", "kind": "input_accepted", "count": 2},
{"scope_id": "root", "kind": "recovery_state", "count": 2},
]
assert result["runs"]["truncated"] is False
assert result["runs"]["items"] == [
{
"run_id": None,
"event_count": 1,
"min_sequence": 8,
"max_sequence": 8,
"settled": False,
},
{
"run_id": "r2",
"event_count": 4,
"min_sequence": 4,
"max_sequence": 7,
"settled": False,
},
{
"run_id": "r1",
"event_count": 3,
"min_sequence": 1,
"max_sequence": 3,
"settled": True,
},
]
assert result["recovery_states"] == {
"items": [
{"scope_id": "child-1", "max_sequence": 6},
{"scope_id": "root", "max_sequence": 4},
],
"truncated": False,
}
assert result["open_tool_attempts"] == {
"open_count": 0,
"truncated": False,
"items": [],
}
assert result["auto_recovery"] == {
"state": None,
"events": [],
"events_truncated": False,
}
def test_open_tool_attempts_follow_the_recovery_reader_definition(engine, task_id):
with Session(engine) as db:
make_v2(db, task_id)
# Start only: effect unknown.
fact(db, task_id, "tool_execution_start", "tool:A:start", run="r1", attempt="A")
# Start + end, start + failed: committed outcomes.
fact(db, task_id, "tool_execution_start", "tool:B:start", run="r1", attempt="B")
fact(db, task_id, "tool_execution_end", "tool:B:end", run="r1", attempt="B")
fact(db, task_id, "tool_execution_start", "tool:C:start", run="r1", attempt="C")
fact(db, task_id, "tool_execution_failed", "tool:C:fail", run="r1", attempt="C")
# A later run's settlement end settles the original start, exactly as
# read_committed_tool_outcome counts any end row for the attempt.
fact(db, task_id, "tool_execution_start", "tool:D:start", run="r1", attempt="D")
fact(
db,
task_id,
"tool_execution_end",
"tool:D:settlement:tool_execution_end",
run="r2",
attempt="D",
)
# A settlement start with no end of any kind stays open. The original
# run id sorts after the settlement run's, so the reported run must
# come from the start row, not from an aggregate over run ids.
fact(db, task_id, "tool_execution_start", "tool:E:start", run="r9", attempt="E")
fact(
db,
task_id,
"tool_execution_start",
"tool:E:settlement:tool_execution_start",
run="r2",
attempt="E",
)
# The same attempt id in another scope is a different attempt.
fact(
db,
task_id,
"tool_execution_start",
"tool:A:start",
scope="child-1",
run="r1",
attempt="A",
)
fact(
db,
task_id,
"tool_execution_end",
"tool:A:end",
scope="child-1",
run="r1",
attempt="A",
)
# An end with no start is not a candidate.
fact(db, task_id, "tool_execution_end", "tool:F:end", run="r1", attempt="F")
# A non-tool fact never carries an attempt id and is ignored.
fact(db, task_id, "agent_message", "msg-1", run="r1")
db.commit()
result = diagnose(engine, task_id)["open_tool_attempts"]
assert result["open_count"] == 2
assert result["truncated"] is False
assert result["items"] == [
{
"tool_attempt_id": "E",
"scope_id": "root",
"run_id": "r9",
"start_sequence": 8,
"kinds": ["tool_execution_start"],
},
{
"tool_attempt_id": "A",
"scope_id": "root",
"run_id": "r1",
"start_sequence": 1,
"kinds": ["tool_execution_start"],
},
]
def test_open_tool_attempts_report_truncation(engine, task_id, monkeypatch):
monkeypatch.setattr(diagnostics_module, "MAX_OPEN_TOOL_ATTEMPTS", 2)
with Session(engine) as db:
make_v2(db, task_id)
for attempt in ("A", "B", "C"):
fact(
db,
task_id,
"tool_execution_start",
f"tool:{attempt}:start",
run="r1",
attempt=attempt,
)
db.commit()
result = diagnose(engine, task_id)["open_tool_attempts"]
assert result["open_count"] == 3
assert result["truncated"] is True
assert [item["tool_attempt_id"] for item in result["items"]] == ["C", "B"]
def test_lists_report_truncation(engine, task_id, monkeypatch):
for name in (
"MAX_EVENT_GROUPS",
"MAX_RUNS",
"MAX_RECOVERY_STATE_SCOPES",
"MAX_RECOVERY_EVENTS",
):
monkeypatch.setattr(diagnostics_module, name, 1)
seed_two_runs(engine, task_id)
with Session(engine) as db:
for event in ("scheduled", "auto_resumed"):
db.add(TaskRecoveryEvent(task_id=task_id, run_id="r2", event=event))
db.commit()
result = diagnose(engine, task_id)
assert len(result["events"]["by_scope_kind"]) == 1
assert result["events"]["by_scope_kind_truncated"] is True
assert len(result["runs"]["items"]) == 1
assert result["runs"]["truncated"] is True
assert len(result["recovery_states"]["items"]) == 1
assert result["recovery_states"]["truncated"] is True
assert [e["event"] for e in result["auto_recovery"]["events"]] == ["auto_resumed"]
assert result["auto_recovery"]["events_truncated"] is True
# Totals are aggregates, not list lengths, so they are not capped.
assert result["events"]["total"] == 8
def test_auto_recovery_state_events_and_only_whether_last_error_is_set(engine, task_id):
with Session(engine) as db:
db.add(
TaskAutoRecovery(
task_id=task_id,
run_id="r1",
reason="runner_lost",
state="scheduled",
state_detail="backoff",
paused_state_version=3,
interrupted_at=T0,
episode_started_at=T0,
no_progress_resumes=1,
total_resumes=2,
next_attempt_at=T0,
last_command_id="cmd-1",
last_error=f"{SENTINEL} Bearer sk-not-a-real-token",
)
)
db.add(TaskRecoveryEvent(task_id=task_id, run_id="r1", event="interrupted"))
db.add(
TaskRecoveryEvent(
task_id=task_id,
run_id="r1",
event="scheduled",
reason="runner_lost",
attempt=1,
next_attempt_at=T0,
detail={"note": SENTINEL},
)
)
db.commit()
result = diagnose(engine, task_id)["auto_recovery"]
state = result["state"]
assert state["state"] == "scheduled"
assert state["reason"] == "runner_lost"
assert state["state_detail"] == "backoff"
assert (state["no_progress_resumes"], state["total_resumes"]) == (1, 2)
# The run's error text is the same content as error_message: hidden.
assert state["has_last_error"] is True
assert "last_error" not in state
assert [e["event"] for e in result["events"]] == ["scheduled", "interrupted"]
assert result["events"][0]["attempt"] == 1
assert "detail" not in result["events"][0]
assert SENTINEL not in json.dumps(result)
def test_v1_task_returns_empty_sections(engine, task_id):
result = build_client(engine, is_admin=True).get(url(task_id)).json()
assert result["storage"] == "v1"
assert result["task"]["conversation_storage_version"] == 1
assert result["task"]["has_error_message"] is False
assert result["events"]["total"] == 0
assert result["events"]["min_sequence"] is None
assert result["events"]["by_scope_kind"] == []
assert result["runs"]["items"] == []
assert result["recovery_states"]["items"] == []
assert result["open_tool_attempts"]["open_count"] == 0
assert result["auto_recovery"]["state"] is None
def test_payload_content_never_appears_in_the_response(engine, task_id):
seed_two_runs(engine, task_id)
with Session(engine) as db:
fact(db, task_id, "tool_execution_start", "tool:A:start", run="r2", attempt="A")
task = db.get(Task, task_id)
task.error_message = f"boom {SENTINEL}"
db.add(
TaskAutoRecovery(
task_id=task_id,
run_id="r2",
reason="runner_lost",
state="manual",
paused_state_version=1,
interrupted_at=T0,
episode_started_at=T0,
last_error=f"{SENTINEL} connection reset",
)
)
db.add(
TaskRecoveryEvent(
task_id=task_id,
run_id="r2",
event="interrupted",
detail={"note": SENTINEL},
)
)
db.commit()
with Session(engine) as db:
stored = db.execute(
sa.text(
"SELECT count(*) FROM task_execution_events "
"WHERE CAST(payload AS TEXT) LIKE :s"
),
{"s": f"%{SENTINEL}%"},
).scalar()
assert stored == 9
response = build_client(engine, is_admin=True).get(url(task_id))
assert response.status_code == 200
assert SENTINEL not in response.text
assert response.json()["task"]["has_error_message"] is True
assert response.json()["open_tool_attempts"]["open_count"] == 1
def test_diagnostics_never_reference_the_payload_column(engine, task_id):
seed_two_runs(engine, task_id)
statements: list[str] = []
def record(conn, cursor, statement, parameters, context, executemany):
statements.append(statement)
sa.event.listen(engine, "before_cursor_execute", record)
try:
diagnose(engine, task_id)
finally:
sa.event.remove(engine, "before_cursor_execute", record)
assert any("task_execution_events" in s for s in statements)
assert not [s for s in statements if "payload" in s.lower()]
def test_database_error_returns_503_without_leaking_details(
engine, task_id, monkeypatch, caplog
):
def explode(db, requested_task_id):
raise OperationalError("SELECT secret", {"p": SENTINEL}, Exception(SENTINEL))
monkeypatch.setattr(api_module, "task_execution_event_diagnostics", explode)
with caplog.at_level(logging.WARNING, logger=api_module.logger.name):
response = build_client(engine, is_admin=True).get(url(task_id))
assert response.status_code == 503
assert SENTINEL not in response.text
messages = " ".join(r.getMessage() for r in caplog.records)
assert f"task {task_id}" in messages
assert "OperationalError" in messages
assert SENTINEL not in messages
assert not any(r.exc_info for r in caplog.records)
def test_degradations_are_reported_for_this_process(engine, task_id):
register_degradation("checkpoint_load_unavailable", "task_id=1: detail")
try:
result = diagnose(engine, task_id)
finally:
clear_degradation("checkpoint_load_unavailable")
assert "checkpoint_load_unavailable" in result["degradations"]
def test_event_sections_read_a_snapshot_at_the_task_horizon(engine, task_id):
"""Events above the horizon read first belong to a later snapshot.
Each append advances the horizon in its own transaction, so a row above
the horizon read by the request is one committed after that read. It is
simulated here by inserting rows without advancing the horizon.
"""
with Session(engine) as db:
make_v2(db, task_id)
fact(db, task_id, "tool_execution_start", "tool:A:start", run="r1", attempt="A")
db.commit()
horizon = db.get(Task, task_id).conversation_event_sequence
for offset, (kind, key, attempt) in enumerate(
[
("tool_execution_end", "tool:A:end", "A"),
("tool_execution_start", "tool:B:start", "B"),
("execution_settled", "settled:r1", None),
("recovery_state", "recovery:r1", None),
],
start=1,
):
db.add(
TaskExecutionEvent(
event_id=f"later-{offset}",
task_id=task_id,
sequence=horizon + offset,
scope_id="root",
run_id="r1",
tool_attempt_id=attempt,
idempotency_key=key,
kind=kind,
payload_version=1,
payload={},
occurred_at=T0,
)
)
db.commit()
result = diagnose(engine, task_id)
assert result["task"]["conversation_event_sequence"] == horizon
assert result["events"]["total"] == horizon
assert result["events"]["max_sequence"] == horizon
assert result["runs"]["items"][0]["settled"] is False
assert result["recovery_states"]["items"] == []
# A's end and B's start are after the snapshot: A is still open, B absent.
assert result["open_tool_attempts"]["open_count"] == 1
assert [i["tool_attempt_id"] for i in result["open_tool_attempts"]["items"]] == [
"A"
]