Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

214 KB
Raw Blame
from __future__ import annotations

import asyncio
import logging
import threading
from collections.abc import Iterator
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from typing import Any, Dict

import pytest
from sqlalchemy import create_engine
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.orm import Query, Session, sessionmaker
from sqlalchemy.pool import QueuePool, StaticPool

from tests.web.services.task_interaction_schema_shared import (
    make_row as make_interaction_row,
)
from xagent.core.agent.checkpoint import (
    CHECKPOINT_EVENT_TYPE,
    CHECKPOINT_TYPE,
    READABLE_CHECKPOINT_TYPES,
    CheckpointAccessRefusedError,
    CheckpointCorruptError,
    CheckpointUnavailableError,
)
from xagent.core.agent.trace import (
    TraceAction,
    TraceCategory,
    TraceEvent,
    TraceEventType,
    Tracer,
    TraceScope,
)
from xagent.core.tools.adapters.vibe.agent_tool import (
    _DelegatedAgentTaskEventTraceHandler,
)
from xagent.web.api.websocket import (
    ConnectionManager,
    _is_agent_checkpoint_data,
    _is_duplicate_user_message_turn,
    send_historical_data_as_stream,
)
from xagent.web.models.chat_message import TaskChatMessage
from xagent.web.models.database import Base
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task import TraceEvent as DatabaseTraceEvent
from xagent.web.models.task_interaction import TaskInteractionRequest
from xagent.web.models.user import User
from xagent.web.models.workforce import WorkforceRun
from xagent.web.services import task_events
from xagent.web.services.ops_signals import (
    CHECKPOINT_PRUNE_FAILED,
    active_degradations,
    clear_degradation,
)
from xagent.web.services.task_event_trace_handler import (
    TaskEventTraceHandler,
    get_event_type_mapping,
    serialize_trace_data,
)
from xagent.web.services.task_execution import (
    _agent_outbound_event_type,
    _persist_agent_outbound_event,
    create_final_answer_stream_event,
    create_stream_event,
    make_agent_outbound_handler,
)
from xagent.web.services.task_interaction_schema import (
    interaction_requests_table_exists,
)
from xagent.web.services.task_lease_service import (
    TASK_RUN_ID_TRACE_FIELD,
    TaskLease,
    bind_task_lease_context,
    current_task_lease,
)
from xagent.web.services.trace_handlers import (
    DatabaseTraceHandler,
    _ResolvedReadPartition,
)


def _shared_memory_sqlite_engine():
    """Keep the in-memory test database visible to off-loop worker Sessions."""

    return create_engine(
        "sqlite://",
        connect_args={"check_same_thread": False},
        poolclass=StaticPool,
    )


def test_agent_checkpoint_is_not_converted_to_websocket_stream_event() -> None:
    event = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id="365",
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "365",
            "snapshot": {"label": "dag_before_llm"},
        },
    )

    stream_event = TaskEventTraceHandler(365)._convert_trace_event_to_stream_event(
        event
    )

    assert stream_event is None


def test_action_tool_error_maps_to_tool_execution_failed() -> None:
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.ERROR, TraceCategory.TOOL),
        task_id="365",
        step_id="default",
        data={"tool_name": "execute_python_code", "error_message": "failed"},
    )

    assert get_event_type_mapping(event) == "tool_execution_failed"


def test_action_llm_error_maps_to_llm_call_failed() -> None:
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.ERROR, TraceCategory.LLM),
        task_id="365",
        step_id="365",
        data={"error_message": "read timed out"},
    )

    assert get_event_type_mapping(event) == "llm_call_failed"


def test_workforce_delegation_summary_maps_to_public_stream_event() -> None:
    event = TraceEvent(
        TraceEventType(TraceScope.TASK, TraceAction.UPDATE, TraceCategory.GENERAL),
        task_id="365",
        data={
            "event_type": "workforce_delegation_start",
            "status": "start",
            "agent_id": 12,
            "agent_name": "Researcher",
            "worker_alias": "Research",
            "worker_task_id": "agent_12_abcd1234",
            "messages": [{"role": "user", "content": "raw prompt"}],
        },
    )

    stream_event = TaskEventTraceHandler(365)._convert_trace_event_to_stream_event(
        event
    )

    assert stream_event is not None
    assert stream_event["event_type"] == "workforce_delegation_start"
    assert stream_event["data"]["worker_alias"] == "Research"
    assert stream_event["data"]["worker_task_id"] == "agent_12_abcd1234"
    assert "messages" not in stream_event["data"]
    assert "event_type" not in stream_event["data"]


def test_non_task_update_event_with_delegation_payload_is_not_promoted() -> None:
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.END, TraceCategory.TOOL),
        task_id="365",
        step_id="step-1",
        data={
            "event_type": "workforce_delegation_end",
            "tool_name": "agent_12",
            "output": "worker response",
        },
    )

    stream_event = TaskEventTraceHandler(365)._convert_trace_event_to_stream_event(
        event
    )

    assert stream_event is not None
    assert stream_event["event_type"] == "tool_execution_end"
    assert stream_event["data"]["event_type"] == "workforce_delegation_end"
    assert stream_event["data"]["output"] == "worker response"


def test_historical_stream_identifies_agent_checkpoint_payload() -> None:
    assert _is_agent_checkpoint_data(
        {
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "365",
            "snapshot": {"label": "dag_before_llm"},
        }
    )
    assert _is_agent_checkpoint_data(
        {
            "type": "checkpoint",
            "execution_id": "365",
            "pattern_state": {"status": "running"},
            "context": {"messages": []},
        }
    )
    assert not _is_agent_checkpoint_data({"event": "ai_message"})


def test_final_answer_stream_event_is_not_trace_event() -> None:
    event = create_final_answer_stream_event(
        "final_answer_delta",
        365,
        {
            "type": "final_answer_delta",
            "message_id": "final_answer_1",
            "delta": "hello",
        },
    )

    assert event["type"] == "final_answer_delta"
    assert event["task_id"] == 365
    assert event["message_id"] == "final_answer_1"
    assert event["delta"] == "hello"
    assert "event_type" not in event
    assert "data" not in event


def test_agent_outbound_event_type_separates_progress_from_questions() -> None:
    assert (
        _agent_outbound_event_type(
            {
                "message": "Still working",
                "message_type": "progress",
                "expect_response": False,
            }
        )
        == "agent_progress"
    )
    assert (
        _agent_outbound_event_type(
            {
                "message": "Need input",
                "message_type": "question",
                "expect_response": False,
            }
        )
        == "agent_message"
    )
    assert (
        _agent_outbound_event_type(
            {
                "message": "Need input",
                "message_type": "info",
                "expect_response": True,
            }
        )
        == "agent_message"
    )


@pytest.mark.asyncio
async def test_agent_outbound_handler_skips_hidden_messages(monkeypatch) -> None:
    persisted_calls: list[tuple[int, dict[str, object]]] = []
    broadcast_calls: list[tuple[dict[str, object], int]] = []
    to_thread_calls: list[tuple[object, tuple[object, ...]]] = []

    def fake_persist(task_id: int, event: dict[str, object]) -> None:
        persisted_calls.append((task_id, event))

    async def fake_to_thread(func: object, /, *args: object) -> None:
        to_thread_calls.append((func, args))

    async def fake_broadcast(event: dict[str, object], task_id: int) -> None:
        broadcast_calls.append((event, task_id))

    monkeypatch.setattr(
        "xagent.web.services.task_execution._persist_agent_outbound_event", fake_persist
    )
    monkeypatch.setattr("xagent.web.api.websocket.asyncio.to_thread", fake_to_thread)
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.broadcast_to_task", fake_broadcast
    )

    handler = make_agent_outbound_handler(365)
    await handler(
        {
            "execution_id": "exec-1",
            "message": "Hidden progress",
            "message_type": "progress",
            "expect_response": False,
            "visible": False,
        }
    )

    assert persisted_calls == []
    assert to_thread_calls == []
    assert broadcast_calls == []


@pytest.mark.asyncio
async def test_agent_outbound_handler_repairs_completed_final_answer(
    monkeypatch,
) -> None:
    broadcast_calls: list[tuple[dict[str, object], int]] = []

    def fake_reconcile(task_id: int, content: str) -> str:
        assert task_id == 365
        assert content == "[video.mp4](file:invented-id)"
        return "[video.mp4](file:real-id)"

    async def fake_to_thread(func: object, /, *args: object):
        return func(*args)

    async def fake_broadcast(event: dict[str, object], task_id: int) -> None:
        broadcast_calls.append((event, task_id))

    monkeypatch.setattr(
        "xagent.web.services.task_execution._reconcile_streamed_final_answer",
        fake_reconcile,
    )
    monkeypatch.setattr("xagent.web.api.websocket.asyncio.to_thread", fake_to_thread)
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.broadcast_to_task", fake_broadcast
    )

    handler = make_agent_outbound_handler(365)
    await handler(
        {
            "type": "final_answer_end",
            "message_id": "final-answer-1",
            "content": "[video.mp4](file:invented-id)",
        }
    )

    assert len(broadcast_calls) == 1
    event, task_id = broadcast_calls[0]
    assert task_id == 365
    assert event["type"] == "final_answer_end"
    assert event["content"] == "[video.mp4](file:real-id)"


def test_persist_agent_outbound_event_uses_payload_ids(monkeypatch) -> None:
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()
    try:
        user = User(username="tester", password_hash="hashed_password", is_admin=False)
        db.add(user)
        db.commit()
        db.refresh(user)
        task = Task(
            user_id=int(user.id),
            title="Chat task",
            description="Task chat",
            status=TaskStatus.PENDING,
        )
        db.add(task)
        db.commit()
        db.refresh(task)
    finally:
        db.close()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.task_execution.get_db", get_test_db)

    event = create_stream_event(
        "agent_progress",
        int(task.id),
        {
            "event_id": "agent-event-1",
            "step_id": "react-step-1",
            "message": "Still working",
            "expect_response": False,
        },
    )

    _persist_agent_outbound_event(int(task.id), event)

    db = SessionLocal()
    try:
        trace_event = db.query(DatabaseTraceEvent).filter_by(task_id=int(task.id)).one()
        assert trace_event.event_id == "agent-event-1"
        assert trace_event.event_type == "agent_progress"
        assert trace_event.step_id == "react-step-1"
    finally:
        db.close()


def test_persist_agent_outbound_event_sanitizes_payload(monkeypatch) -> None:
    """This function builds its own TraceEvent row without going through
    stage_trace_event_row (the staging module's "known bypass"), so it must
    sanitize for itself: PostgreSQL's jsonb rejects NUL and lone-surrogate
    code points at INSERT (#1248)."""
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()
    try:
        user = User(
            username="sanitizer", password_hash="hashed_password", is_admin=False
        )
        db.add(user)
        db.commit()
        db.refresh(user)
        task = Task(
            user_id=int(user.id),
            title="Chat task",
            description="Task chat",
            status=TaskStatus.PENDING,
        )
        db.add(task)
        db.commit()
        db.refresh(task)
    finally:
        db.close()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.task_execution.get_db", get_test_db)

    # chr, not source escapes: a lone surrogate is not encodable as UTF-8.
    nul, lone_high, replacement = chr(0x0000), chr(0xD800), chr(0xFFFD)
    event = create_stream_event(
        "agent_progress",
        int(task.id),
        {
            "event_id": "agent-event-dirty",
            "message": f"head{nul}mid{lone_high}tail",
            "expect_response": False,
        },
    )

    _persist_agent_outbound_event(int(task.id), event)

    db = SessionLocal()
    try:
        trace_event = (
            db.query(DatabaseTraceEvent).filter_by(event_id="agent-event-dirty").one()
        )
        assert trace_event.data["message"] == f"head{replacement}mid{replacement}tail"
    finally:
        db.close()


def _create_trace_handler_test_task(
    username: str,
    *,
    title: str = "Chat task",
    description: str = "Task chat",
):
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()

    user = User(username=username, password_hash="hashed_password", is_admin=False)
    db.add(user)
    db.commit()
    db.refresh(user)
    task = Task(
        user_id=int(user.id),
        title=title,
        description=description,
        status=TaskStatus.PENDING,
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    return SessionLocal, db, task


def _checkpoint_trace_row(
    *,
    task_id: int,
    event_id: str,
    execution_id: str,
    label: str,
    timestamp: datetime,
    run_id: str | None = None,
    build_id: str | None = None,
    checkpoint_type: str = CHECKPOINT_TYPE,
) -> DatabaseTraceEvent:
    data = {
        "checkpoint_type": checkpoint_type,
        "execution_id": execution_id,
        "snapshot": {"label": label},
    }
    if run_id is not None:
        data[TASK_RUN_ID_TRACE_FIELD] = run_id
    return DatabaseTraceEvent(
        task_id=task_id,
        build_id=build_id,
        event_id=event_id,
        event_type="system_update_general",
        timestamp=timestamp,
        data=data,
    )


def _create_trace_handler_test_task_without_interaction_table(
    username: str,
    *,
    title: str = "Chat task",
    description: str = "Task chat",
):
    """Same shape as _create_trace_handler_test_task, built with a filtered
    create_all that excludes task_interaction_requests -- reproducing a
    deployment upgraded to a revision before that table exists (the
    migration that creates it is not merged). The table has no inbound
    foreign keys, so excluding it cannot break create order (verified in
    the audit: 52 of 53 tables created, tasks and trace_events both
    present, has_table(target) False).
    """
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    tables = [
        table
        for table in Base.metadata.sorted_tables
        if table.name != TaskInteractionRequest.__tablename__
    ]
    Base.metadata.create_all(bind=engine, tables=tables)
    db = SessionLocal()

    user = User(username=username, password_hash="hashed_password", is_admin=False)
    db.add(user)
    db.commit()
    db.refresh(user)
    task = Task(
        user_id=int(user.id),
        title=title,
        description=description,
        status=TaskStatus.PENDING,
    )
    db.add(task)
    db.commit()
    db.refresh(task)
    return SessionLocal, db, task


def _seed_interaction_row(
    db: Session,
    *,
    task_id: int,
    resume_trace_event_id: int | None,
    status: str = "active",
    **overrides: object,
) -> TaskInteractionRequest:
    row = TaskInteractionRequest(
        **make_interaction_row(
            task_id=task_id,
            resume_trace_event_id=resume_trace_event_id,
            status=status,
            **overrides,
        )
    )
    db.add(row)
    db.commit()
    db.refresh(row)
    return row


@pytest.mark.asyncio
async def test_paused_replay_event_embeds_known_control_state(monkeypatch) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("paused-replay")
    task.status = TaskStatus.PAUSED
    task.run_id = "run-paused"
    task.state_version = 9
    task.control_state = "paused"
    db.commit()
    task_id = int(task.id)
    user_id = int(task.user_id)
    db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    paused_event = next(
        event for event in sent_events if event.get("type") == "task_paused"
    )
    assert paused_event["run_id"] == "run-paused"
    assert paused_event["state_version"] == 9
    assert paused_event["control_state"] == "paused"
    assert paused_event["status"] == "paused"


@pytest.mark.asyncio
async def test_historical_stream_format_error_redacts_exception_text(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from xagent.web.api import websocket as websocket_api

    secret = "history-storage-secret"
    sent_events: list[dict] = []

    async def fail_load(_operation):
        raise ValueError(f"malformed history: {secret}")

    async def send_personal_message(event: dict, _websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr(websocket_api, "run_db_io_cancellation_safe", fail_load)
    monkeypatch.setattr(
        websocket_api.manager,
        "send_personal_message",
        send_personal_message,
    )

    with pytest.raises(ValueError, match=secret):
        await send_historical_data_as_stream(
            websocket=object(),
            task_id=42,
            user=SimpleNamespace(id=1, is_admin=False),
        )

    assert len(sent_events) == 1
    assert sent_events[0]["event_type"] == "error"
    assert (
        sent_events[0]["data"]["message"]
        == "Task history could not be loaded. Please try again."
    )
    assert secret not in repr(sent_events[0])


@pytest.mark.asyncio
async def test_historical_replay_detaches_before_slow_network_send(
    tmp_path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """History DB/cache work finishes off-loop before the first network wait."""

    from xagent.web.api import websocket as websocket_api
    from xagent.web.models import database as database_module

    engine = create_engine(
        f"sqlite:///{tmp_path / 'history-one-slot.db'}",
        connect_args={"check_same_thread": False},
        poolclass=QueuePool,
        pool_size=1,
        max_overflow=0,
        pool_timeout=0.25,
    )
    Base.metadata.create_all(bind=engine)
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    with SessionLocal() as db:
        owner = User(
            username="history-one-slot-owner",
            password_hash="x",
            is_admin=False,
        )
        db.add(owner)
        db.flush()
        task = Task(
            user_id=int(owner.id),
            title="history one slot",
            description="history",
            status=TaskStatus.COMPLETED,
        )
        db.add(task)
        db.commit()
        owner_id = int(owner.id)
        task_id = int(task.id)

    event_loop_thread = threading.get_ident()
    cache_threads: list[int] = []
    sent_events: list[dict] = []
    cache_entered = threading.Event()
    release_cache = threading.Event()
    send_entered = asyncio.Event()
    release_send = asyncio.Event()

    def slow_cache_get(_key: str):
        assert engine.pool.checkedout() == 0
        cache_threads.append(threading.get_ident())
        cache_entered.set()
        assert threading.get_ident() != event_loop_thread
        assert release_cache.wait(timeout=30), "history cache was never released"
        return None

    def record_cache_set(*_args, **_kwargs) -> None:
        assert engine.pool.checkedout() == 0
        cache_threads.append(threading.get_ident())

    async def slow_send(event: dict, _websocket: object) -> None:
        assert engine.pool.checkedout() == 0

        def read_during_send() -> None:
            with SessionLocal() as db:
                assert db.query(Task.id).filter(Task.id == task_id).scalar() == task_id

        await asyncio.to_thread(read_during_send)
        sent_events.append(event)
        send_entered.set()
        await release_send.wait()

    monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
    monkeypatch.setattr(websocket_api, "get_session_local", lambda: SessionLocal)
    monkeypatch.setattr(websocket_api, "cache_get", slow_cache_get)
    monkeypatch.setattr(websocket_api, "cache_set", record_cache_set)
    monkeypatch.setattr(
        websocket_api.manager,
        "send_personal_message",
        slow_send,
    )

    replay = asyncio.create_task(
        send_historical_data_as_stream(
            websocket=object(),
            task_id=task_id,
            user=SimpleNamespace(id=owner_id, is_admin=False),
        )
    )
    try:
        assert await asyncio.to_thread(cache_entered.wait, 30)
        assert not replay.done()
        assert not sent_events
        release_cache.set()
        await asyncio.wait_for(send_entered.wait(), timeout=30)
        assert not replay.done()
        assert engine.pool.checkedout() == 0
    finally:
        release_cache.set()
        release_send.set()
        try:
            await asyncio.wait_for(replay, timeout=30)
        finally:
            engine.dispose()

    assert cache_threads
    assert all(thread_id != event_loop_thread for thread_id in cache_threads)
    assert [event["event_type"] for event in sent_events] == [
        "task_info",
        "historical_data_complete",
    ]


def test_historical_replay_does_not_backfill_legacy_file_outputs(
    tmp_path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A history cache miss is a read path, never a durable-file writer."""

    from xagent.core.file_storage.factory import get_unscoped_file_storage
    from xagent.core.file_storage.storage import FsspecFileStorage
    from xagent.web.api import websocket as websocket_api
    from xagent.web.models import database as database_module
    from xagent.web.models.uploaded_file import UploadedFile

    engine = create_engine(
        f"sqlite:///{tmp_path / 'history-file-output.db'}",
        connect_args={"check_same_thread": False},
        poolclass=QueuePool,
        pool_size=1,
        max_overflow=0,
        pool_timeout=0.25,
    )
    Base.metadata.create_all(bind=engine)
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    uploads_dir = tmp_path / "uploads"
    monkeypatch.setenv("XAGENT_UPLOADS_DIR", str(uploads_dir))
    monkeypatch.setenv("XAGENT_FILE_STORAGE_URI", (tmp_path / "objects").as_uri())
    get_unscoped_file_storage.cache_clear()

    with SessionLocal() as db:
        owner = User(
            username="history-file-output-owner",
            password_hash="x",
            is_admin=False,
        )
        db.add(owner)
        db.flush()
        task = Task(
            user_id=int(owner.id),
            title="legacy output history",
            description="history",
            status=TaskStatus.COMPLETED,
        )
        db.add(task)
        db.flush()
        owner_id = int(owner.id)
        task_id = int(task.id)
        output_path = (
            uploads_dir
            / f"user_{owner_id}"
            / f"web_task_{task_id}"
            / "output"
            / "legacy.txt"
        )
        output_path.parent.mkdir(parents=True)
        output_path.write_text("legacy bytes", encoding="utf-8")
        db.add(
            DatabaseTraceEvent(
                task_id=task_id,
                event_id="legacy-file-output",
                event_type="tool_execution_start",
                timestamp=datetime.now(timezone.utc),
                data={
                    "file_outputs": [
                        {
                            "path": str(output_path),
                            "filename": "legacy.txt",
                        }
                    ]
                },
            )
        )
        db.commit()

    put_observations: list[int] = []
    original_put_file = FsspecFileStorage.put_file

    def observe_put_file(
        self: FsspecFileStorage,
        source,
        key: str,
        content_type: str | None = None,
    ):
        put_observations.append(engine.pool.checkedout())
        return original_put_file(self, source, key, content_type)

    monkeypatch.setattr(database_module, "_SessionLocal", SessionLocal)
    monkeypatch.setattr(websocket_api, "cache_get", lambda *_args: None)
    monkeypatch.setattr(
        websocket_api,
        "cache_set",
        lambda *_args, **_kwargs: None,
    )
    monkeypatch.setattr(FsspecFileStorage, "put_file", observe_put_file)
    try:
        snapshot = websocket_api._load_historical_stream_snapshot_sync(
            task_id,
            actor_user_id=owner_id,
            actor_is_admin=False,
        )
        assert snapshot is not None
        with SessionLocal() as db:
            assert db.query(UploadedFile).count() == 0
        assert put_observations == []
    finally:
        engine.dispose()
        get_unscoped_file_storage.cache_clear()


def test_database_trace_handler_dedupes_user_message_turn_id() -> None:
    _, db, task = _create_trace_handler_test_task("tester")
    try:
        handler = DatabaseTraceHandler(int(task.id))
        event_type = TraceEventType(
            TraceScope.TASK,
            TraceAction.START,
            TraceCategory.MESSAGE,
        )
        first = TraceEvent(
            event_type,
            task_id=str(task.id),
            data={"message": "Repeat", "turn_id": "turn-1"},
        )
        duplicate = TraceEvent(
            event_type,
            task_id=str(task.id),
            data={"message": "Repeat", "turn_id": "turn-1"},
        )
        different_turn = TraceEvent(
            event_type,
            task_id=str(task.id),
            data={"message": "Repeat", "turn_id": "turn-2"},
        )

        handler._save_trace_event(db, first)
        handler._save_trace_event(db, duplicate)
        handler._save_trace_event(db, different_turn)

        rows = (
            db.query(DatabaseTraceEvent)
            .filter_by(task_id=int(task.id), event_type="user_message")
            .order_by(DatabaseTraceEvent.id)
            .all()
        )
        assert [row.data["turn_id"] for row in rows] == ["turn-1", "turn-2"]
    finally:
        db.close()


def test_database_trace_handler_dedupes_user_message_turn_id_per_build() -> None:
    _, db, task = _create_trace_handler_test_task("build-tester")
    try:
        event_type = TraceEventType(
            TraceScope.TASK,
            TraceAction.START,
            TraceCategory.MESSAGE,
        )
        parent_handler = DatabaseTraceHandler(int(task.id))
        worker_handler = DatabaseTraceHandler(
            int(task.id),
            build_id="agent_123_abcd1234",
        )

        parent_handler._save_trace_event(
            db,
            TraceEvent(
                event_type,
                task_id=str(task.id),
                data={"message": "Repeat", "turn_id": "turn-1"},
            ),
        )
        worker_handler._save_trace_event(
            db,
            TraceEvent(
                event_type,
                task_id="agent_123_abcd1234",
                data={"message": "Repeat", "turn_id": "turn-1"},
            ),
        )
        worker_handler._save_trace_event(
            db,
            TraceEvent(
                event_type,
                task_id="agent_123_abcd1234",
                data={"message": "Repeat", "turn_id": "turn-1"},
            ),
        )

        rows = (
            db.query(DatabaseTraceEvent)
            .filter_by(task_id=int(task.id), event_type="user_message")
            .order_by(DatabaseTraceEvent.id)
            .all()
        )
        assert [(row.build_id, row.data["turn_id"]) for row in rows] == [
            (None, "turn-1"),
            ("agent_123_abcd1234", "turn-1"),
        ]
    finally:
        db.close()


def test_database_trace_handler_build_checkpoint_does_not_update_task_pointer() -> None:
    _, db, task = _create_trace_handler_test_task(
        "checkpoint-user",
        title="Checkpoint task",
        description="Task with worker checkpoint",
    )
    try:
        handler = DatabaseTraceHandler(
            int(task.id),
            build_id="agent_123_abcd1234",
        )
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE,
            task_id="agent_123_abcd1234",
            data={
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "agent_123_abcd1234",
                "snapshot": {"label": "worker_checkpoint"},
            },
        )

        with bind_task_lease_context(
            TaskLease(int(task.id), "runner-a", "run-a", attempt_id="test-attempt")
        ):
            handler._save_trace_event(db, event)
        db.refresh(task)

        row = db.query(DatabaseTraceEvent).filter_by(task_id=int(task.id)).one()
        assert row.build_id == "agent_123_abcd1234"
        assert TASK_RUN_ID_TRACE_FIELD not in row.data
        assert task.last_checkpoint_event_id is None
    finally:
        db.close()


def test_database_trace_handler_tags_and_points_to_current_run_checkpoint() -> None:
    _, db, task = _create_trace_handler_test_task("current-run-checkpoint")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    lease = TaskLease(
        task_id=int(task.id),
        runner_id="runner-a",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    event = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task.id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": str(task.id),
            "snapshot": {"label": "current"},
        },
    )

    try:
        with bind_task_lease_context(lease):
            DatabaseTraceHandler(int(task.id))._save_trace_event(db, event)

        db.refresh(task)
        row = db.query(DatabaseTraceEvent).filter_by(task_id=int(task.id)).one()
        assert row.data[TASK_RUN_ID_TRACE_FIELD] == "run-a"
        assert task.last_checkpoint_event_id == str(event.id)
        # Dual write: the exact-row anchor points at the same row the
        # legacy string column names.
        assert task.last_checkpoint_trace_event_id == row.id
        # Mixed-version invariant: exactly one row in the task's root
        # partition carries the legacy event_id the pointer names, so an
        # old reader still resolving through the string column stays
        # unambiguous.
        matching = (
            db.query(DatabaseTraceEvent)
            .filter(
                DatabaseTraceEvent.task_id == int(task.id),
                DatabaseTraceEvent.build_id.is_(None),
                DatabaseTraceEvent.event_type == "system_update_general",
                DatabaseTraceEvent.event_id == task.last_checkpoint_event_id,
            )
            .all()
        )
        assert len(matching) == 1
        assert matching[0].id == row.id
    finally:
        db.close()


@pytest.mark.parametrize("require_persisted", [False, True])
def test_database_trace_handler_rejects_stale_checkpoint(
    require_persisted: bool,
) -> None:
    _, db, task = _create_trace_handler_test_task("stale-run-checkpoint")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-b"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-b"
    db.commit()
    stale_lease = TaskLease(
        task_id=int(task.id),
        runner_id="runner-a",
        run_id="run-a",
        attempt_id="test-attempt",
    )
    event = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task.id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": str(task.id),
            "snapshot": {"label": "stale"},
        },
        require_persisted=require_persisted,
    )

    try:
        with (
            bind_task_lease_context(stale_lease),
            pytest.raises(RuntimeError, match="lease changed"),
        ):
            DatabaseTraceHandler(int(task.id))._save_trace_event(db, event)

        db.expire_all()
        persisted = db.get(Task, int(task.id))
        assert persisted.last_checkpoint_event_id is None
        assert persisted.last_checkpoint_trace_event_id is None
        assert db.query(DatabaseTraceEvent).filter_by(task_id=int(task.id)).count() == 0
    finally:
        db.close()


@pytest.mark.parametrize("require_persisted", [False, True])
def test_database_trace_handler_pointer_update_skips_silently_when_task_is_gone(
    require_persisted: bool,
) -> None:
    """A 0-row pointer UPDATE has two distinct causes: the lease moved (an
    error worth raising), or the task row itself is gone. This test's
    in-memory engine has no FK pragma enabled, so deleting the task row
    does not fail the trace_event insert -- it reaches the pointer UPDATE
    against a task_id that matches nothing, the same state a deployment
    without FK enforcement on this legacy column would reach.

    The task-gone branch is classified exactly as the trace_events.task_id
    FK violation is classified: always zero residue, quiet for a
    best-effort event, loud for a require_persisted one. Reporting success
    for an event that was dropped is what the FK path has never done.
    """
    _, db, task = _create_trace_handler_test_task(
        f"task-gone-checkpoint-{require_persisted}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    lease = TaskLease(
        task_id=task_id, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    event = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task_id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": str(task_id),
            "snapshot": {"label": "orphaned"},
        },
        require_persisted=require_persisted,
    )

    db.query(Task).filter(Task.id == task_id).delete(synchronize_session=False)
    db.commit()

    try:
        with bind_task_lease_context(lease):
            if require_persisted:
                with pytest.raises(RuntimeError, match="no longer exists"):
                    DatabaseTraceHandler(task_id)._save_trace_event(db, event)
            else:
                # Must not raise "lease changed" -- the task is gone, not
                # leased elsewhere.
                DatabaseTraceHandler(task_id)._save_trace_event(db, event)

        # Either way the staged row is discarded, not committed as an orphan.
        assert db.query(DatabaseTraceEvent).filter_by(task_id=task_id).count() == 0
    finally:
        db.close()


def test_cached_database_trace_handler_reads_run_context_per_event() -> None:
    _, db, task = _create_trace_handler_test_task("cached-run-checkpoint")
    handler = DatabaseTraceHandler(int(task.id))
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    first = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task.id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": str(task.id),
            "snapshot": {"label": "first"},
        },
    )
    second = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task.id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": str(task.id),
            "snapshot": {"label": "second"},
        },
    )

    try:
        with bind_task_lease_context(
            TaskLease(int(task.id), "runner-a", "run-a", attempt_id="test-attempt")
        ):
            handler._save_trace_event(db, first)

        task.runner_id = "runner-b"
        task.lease_attempt_id = "test-attempt"
        task.run_id = "run-b"
        task.last_checkpoint_event_id = None
        db.commit()
        with bind_task_lease_context(
            TaskLease(int(task.id), "runner-b", "run-b", attempt_id="test-attempt")
        ):
            handler._save_trace_event(db, second)

        rows = (
            db.query(DatabaseTraceEvent)
            .filter_by(task_id=int(task.id))
            .order_by(DatabaseTraceEvent.id)
            .all()
        )
        assert [row.data[TASK_RUN_ID_TRACE_FIELD] for row in rows] == [
            "run-a",
            "run-b",
        ]
        db.refresh(task)
        assert task.last_checkpoint_event_id == str(second.id)
    finally:
        db.close()


def test_database_trace_handler_load_latest_checkpoint_is_build_scoped(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task(
        "load-user",
        title="Checkpoint task",
        description="Task with scoped checkpoints",
    )

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        parent_handler = DatabaseTraceHandler(int(task.id))
        worker_handler = DatabaseTraceHandler(
            int(task.id),
            build_id="agent_123_abcd1234",
        )
        parent_handler._save_trace_event(
            db,
            TraceEvent(
                CHECKPOINT_EVENT_TYPE,
                task_id=str(task.id),
                data={
                    "checkpoint_type": CHECKPOINT_TYPE,
                    "execution_id": "shared-execution",
                    "snapshot": {"label": "parent_checkpoint"},
                },
            ),
        )
        worker_handler._save_trace_event(
            db,
            TraceEvent(
                CHECKPOINT_EVENT_TYPE,
                task_id="agent_123_abcd1234",
                data={
                    "checkpoint_type": CHECKPOINT_TYPE,
                    "execution_id": "shared-execution",
                    "snapshot": {"label": "worker_checkpoint"},
                },
            ),
        )

        with bind_task_lease_context(
            TaskLease(int(task.id), "runner-a", "run-a", attempt_id="test-attempt")
        ):
            # The root checkpoint was written before the lease bound above,
            # so it carries no run tag. With no tagged checkpoint on record
            # for this task, the root reader's partition widens to the
            # untagged rows and it still reads its own (untagged) root
            # checkpoint -- build scoping is what is under test here, not
            # the run partition, so this must stay reachable regardless.
            assert parent_handler._sync_load_latest_checkpoint("shared-execution") == {
                "label": "parent_checkpoint"
            }
            # The worker (build-scoped) handler never shares the root
            # reader's checkpoints or partition logic at all.
            assert worker_handler._sync_load_latest_checkpoint("shared-execution") == {
                "label": "worker_checkpoint"
            }

        assert parent_handler._sync_load_latest_checkpoint("shared-execution") == {
            "label": "parent_checkpoint"
        }
    finally:
        db.close()


def test_database_trace_handler_loads_checkpoint_only_from_bound_run(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("run-scoped-load")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-b"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-b"
    db.commit()
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="run-b-checkpoint",
                execution_id="shared-execution",
                label="run-b",
                timestamp=now,
                run_id="run-b",
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="newer-run-a-checkpoint",
                execution_id="shared-execution",
                label="run-a",
                timestamp=now + timedelta(seconds=1),
                run_id="run-a",
            ),
        ]
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-b", "run-b", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "run-b"}
    finally:
        db.close()


@pytest.mark.asyncio
async def test_database_trace_handler_load_worker_inherits_run_context(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    handler = DatabaseTraceHandler(7)

    def observe_bound_run(_execution_id: str) -> dict[str, str | None]:
        lease = current_task_lease()
        return {"run_id": lease.run_id if lease is not None else None}

    monkeypatch.setattr(handler, "_sync_load_latest_checkpoint", observe_bound_run)

    with bind_task_lease_context(
        TaskLease(7, "runner-b", "run-b", attempt_id="test-attempt")
    ):
        assert await handler.load_latest_checkpoint("shared-execution") == {
            "run_id": "run-b"
        }

    assert current_task_lease() is None


def test_database_trace_handler_bound_run_widens_to_legacy_checkpoint_when_untagged(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A resume mints a fresh run id before any checkpoint has been written
    under it. Until the first checkpoint tags that run, a lease-bound reader
    must still be able to read the checkpoint written before partitioning
    existed -- refusing here is exactly the #2023 regression (a resumed
    task silently starts from an empty context because its own pre-existing
    checkpoint became unreadable under the newly minted run)."""
    SessionLocal, db, task = _create_trace_handler_test_task("run-legacy-load")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-b"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-b"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-b", "run-b", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "legacy"}
    finally:
        db.close()


def test_database_trace_handler_load_without_pk_anchor_uses_legacy_scan(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Contract for the mixed-data window: with the pointer unset, the read
    path is exactly the pre-existing legacy scan."""
    SessionLocal, db, task = _create_trace_handler_test_task("no-pk-anchor-load")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    assert task.last_checkpoint_trace_event_id is None
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy-only",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "legacy-only"}
    finally:
        db.close()


def test_database_trace_handler_load_uses_pk_anchor_over_newer_row(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Once set, the pointer is authoritative -- even over a row the legacy
    scan (newest timestamp first) would otherwise pick instead."""
    SessionLocal, db, task = _create_trace_handler_test_task("pk-anchor-load")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    anchored_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="anchored-checkpoint",
        execution_id="shared-execution",
        label="anchored",
        timestamp=now,
        run_id="run-a",
    )
    db.add(anchored_row)
    db.commit()
    db.refresh(anchored_row)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="newer-checkpoint",
            execution_id="shared-execution",
            label="newer",
            timestamp=now + timedelta(seconds=1),
            run_id="run-a",
        )
    )
    task.last_checkpoint_trace_event_id = anchored_row.id
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "anchored"}
    finally:
        db.close()


_PK_ANCHOR_SINGLE_FAULT_FIELDS = [
    "task_id",
    "event_type",
    "build_id",
    "checkpoint_type",
    "run_partition",
    "execution_id",
]


def _mutate_pk_anchor_field(
    field: str,
    row_kwargs: Dict[str, Any],
    data: Dict[str, Any],
    other_task_id: int,
) -> None:
    """Corrupt exactly one field the PK-anchored validation checks, leaving
    every other field -- including the run field -- valid, so a passing
    case can only be explained by that one conjunct doing nothing."""
    if field == "task_id":
        row_kwargs["task_id"] = other_task_id
    elif field == "event_type":
        row_kwargs["event_type"] = "agent_progress"
    elif field == "build_id":
        row_kwargs["build_id"] = "agent_123_wrongscope"
    elif field == "checkpoint_type":
        data["checkpoint_type"] = "not_a_readable_checkpoint_type"
    elif field == "run_partition":
        data[TASK_RUN_ID_TRACE_FIELD] = "run-b"
    elif field == "execution_id":
        data["execution_id"] = "other-execution"
    else:
        raise AssertionError(f"unknown single-fault field {field}")


@pytest.mark.parametrize("field", _PK_ANCHOR_SINGLE_FAULT_FIELDS)
def test_database_trace_handler_load_pk_anchor_single_fault_raises_corrupt(
    monkeypatch: pytest.MonkeyPatch,
    field: str,
) -> None:
    """Each conjunct of the PK-anchored validation must independently be
    load-bearing: a row wrong in exactly one field (every other field,
    including the run partition, valid) must still raise
    CheckpointCorruptError. A row that violates several conjuncts at once
    proves nothing about any one of them -- deleting one check still leaves
    the others to raise -- so every conjunct gets its own case here.
    """
    SessionLocal, db, task = _create_trace_handler_test_task(
        f"pk-anchor-single-fault-{field}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)

    other_task = Task(
        user_id=int(task.user_id),
        title="Other task",
        description="Other task",
        status=TaskStatus.PENDING,
    )
    db.add(other_task)
    db.commit()
    db.refresh(other_task)
    other_task_id = int(other_task.id)

    # A second, wholly valid, run-a-tagged checkpoint. It exists purely to
    # keep the run-tag probe true (and the resolved partition "run-a")
    # regardless of which single field the pointer-anchored row below is
    # corrupted in -- without it, corrupting task_id/event_type/build_id/
    # checkpoint_type would also make the row below invisible to the probe,
    # widening the partition to None and stacking a second, unintended
    # conjunct violation on top of the one each case means to isolate. The
    # PK-anchor path below raises before it ever scans for this row, so it
    # is never itself read.
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="pk-anchor-single-fault-control",
            execution_id="shared-execution",
            label="control",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
    )
    db.commit()

    row_kwargs: Dict[str, Any] = dict(
        task_id=task_id,
        build_id=None,
        event_id="pk-anchor-single-fault",
        event_type="system_update_general",
        timestamp=datetime.now(timezone.utc),
    )
    data: Dict[str, Any] = {
        "checkpoint_type": CHECKPOINT_TYPE,
        "execution_id": "shared-execution",
        TASK_RUN_ID_TRACE_FIELD: "run-a",
        "snapshot": {"label": "single-fault"},
    }
    _mutate_pk_anchor_field(field, row_kwargs, data, other_task_id)

    row = DatabaseTraceEvent(data=data, **row_kwargs)
    db.add(row)
    db.commit()
    db.refresh(row)
    task.last_checkpoint_trace_event_id = row.id
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointCorruptError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
    finally:
        db.close()


def test_database_trace_handler_load_pk_anchor_absent_run_field_still_raises_corrupt(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A pointer row missing the run-partition field entirely -- the shape
    the 20260804 backfill produces from a trace_events row written before
    that field existed -- is a pre-existing row, not a corrupt one by the
    write-direction resolver's judgment (task_interaction_anchor.py). This
    read path deliberately does not adopt that verdict: within a *tight*
    (run-tagged) partition it still raises CheckpointCorruptError for the
    same row shape, unchanged from its behavior before the shared predicate
    existed. That divergence between the two by-primary-key consumers of
    failed_checkpoint_row_conditions is intentional -- see
    task_interaction_anchor.py's module docstring and #2023, which tracks
    converging them. This cell pins the read side of that divergence;
    test_task_interaction_anchor.py's counterpart pins the write side.
    Pairs with the "run_partition" case of
    test_database_trace_handler_load_pk_anchor_single_fault_raises_corrupt,
    which covers the opposite: a run-partition field that is present and
    wrong.

    A control checkpoint row tagged with the bound run is planted first, for
    the same reason the single-fault cases above plant one: it keeps
    ``_task_has_run_tagged_checkpoint`` true, so the partition this read
    resolves to stays tight ("run-a") instead of widening to the untagged
    partition. Without it, this exact shape -- a task whose run has never
    written a tagged checkpoint, read through a bound lease -- is the one
    #2091's widening feature exists to accept rather than reject; that
    widening path (and the re-probe that guards it against a stale
    partition decision) has its own test family in this file and is not
    what this cell means to pin.
    """
    SessionLocal, db, task = _create_trace_handler_test_task(
        "pk-anchor-absent-run-field"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)

    # Keeps the run-tag probe true (and the resolved partition "run-a") so
    # the pointer row below is validated against a tight partition instead
    # of widening to the untagged one -- see the docstring above.
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="pk-anchor-absent-run-field-control",
            execution_id="shared-execution",
            label="control",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
    )
    db.commit()

    row = DatabaseTraceEvent(
        task_id=task_id,
        build_id=None,
        event_id="pk-anchor-absent-run-field",
        event_type="system_update_general",
        timestamp=datetime.now(timezone.utc),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "shared-execution",
            "snapshot": {"label": "absent-run-field"},
        },
    )
    db.add(row)
    db.commit()
    db.refresh(row)
    task.last_checkpoint_trace_event_id = row.id
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointCorruptError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
    finally:
        db.close()


def test_database_trace_handler_load_pk_anchor_without_execution_identity_loads(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The execution-identity conjunct is verification, not filtering: a
    row that carries no execution identity at all -- a legacy row written
    before root_execution_id/execution_id/snapshot.execution_id existed --
    must still load rather than being treated as a mismatch. Pairs with
    the "execution_id" case of
    test_database_trace_handler_load_pk_anchor_single_fault_raises_corrupt,
    which covers the opposite: an execution identity that is present and
    wrong.
    """
    SessionLocal, db, task = _create_trace_handler_test_task(
        "pk-anchor-no-execution-identity"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)

    row = DatabaseTraceEvent(
        task_id=task_id,
        build_id=None,
        event_id="pk-anchor-no-execution-identity",
        event_type="system_update_general",
        timestamp=datetime.now(timezone.utc),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            TASK_RUN_ID_TRACE_FIELD: "run-a",
            "snapshot": {"label": "no-execution-identity"},
        },
    )
    db.add(row)
    db.commit()
    db.refresh(row)
    task.last_checkpoint_trace_event_id = row.id
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "no-execution-identity"}
    finally:
        db.close()


def test_database_trace_handler_load_dangling_pk_anchor_falls_back_with_telemetry(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A pointer whose row no longer exists is only reachable on a database
    upgraded without the DB-level FK for this column (see the migration).
    The compatibility-window response is a legacy fallback plus a
    self-clearing degradation signal, not a corruption verdict."""
    from xagent.web.services.ops_signals import (
        CHECKPOINT_PK_ANCHOR_DANGLING,
        active_degradations,
        clear_degradation,
    )

    SessionLocal, db, task = _create_trace_handler_test_task("pk-anchor-dangling")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy-fallback",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
    )
    task.last_checkpoint_trace_event_id = 999999999
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    clear_degradation(CHECKPOINT_PK_ANCHOR_DANGLING)
    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "legacy-fallback"}
        assert CHECKPOINT_PK_ANCHOR_DANGLING in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_PK_ANCHOR_DANGLING)
        db.close()


def test_database_trace_handler_unbound_legacy_load_fails_closed_after_tagged_run(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("legacy-only-load")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="legacy-checkpoint",
                execution_id="shared-execution",
                label="legacy",
                timestamp=now,
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="newer-tagged-checkpoint",
                execution_id="shared-execution",
                label="tagged",
                timestamp=now + timedelta(seconds=1),
                run_id="run-a",
            ),
        ]
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        # A tagged run has positive proof a checkpoint exists in a partition
        # this unbound legacy reader is not authoritative for. That is a
        # refusal, not the "queried successfully, found nothing" fact that
        # ``None`` reserves.
        with pytest.raises(CheckpointAccessRefusedError) as excinfo:
            DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            )
        assert excinfo.value.reason == "superseded_legacy"
    finally:
        db.close()


def test_database_trace_handler_unbound_legacy_load_stays_legacy_compatible(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("legacy-only-load")
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
            "shared-execution"
        ) == {"label": "legacy"}
    finally:
        db.close()


def test_database_trace_handler_unbound_root_load_fails_closed_for_active_run(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("unbound-active-load")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        # An active run is in progress under a different lease; this unbound
        # legacy reader is refused, not told the checkpoint is absent.
        with pytest.raises(CheckpointAccessRefusedError) as excinfo:
            DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            )
        assert excinfo.value.reason == "active_run"
    finally:
        db.close()


def test_database_trace_handler_load_refuses_lease_bound_to_another_task(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A bound lease for a different task id is not this reader's partition
    -- distinct from ``active_run``/``superseded_legacy``: the read itself
    is contaminated by a stray context, not a policy decision about this
    task's own rows."""
    SessionLocal, db, task = _create_trace_handler_test_task("lease-mismatch-load")
    task_id = int(task.id)

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with (
            bind_task_lease_context(
                TaskLease(task_id + 1, "runner-a", "run-a", attempt_id="test-attempt")
            ),
            pytest.raises(CheckpointAccessRefusedError) as excinfo,
        ):
            DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            )
        assert excinfo.value.reason == "lease_mismatch"
    finally:
        db.close()


def test_partition_refusal_is_distinct_from_absence(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A refused partition read must never be reported the same way as a
    query that succeeded and genuinely found nothing."""
    SessionLocal, db, task = _create_trace_handler_test_task("refusal-vs-absence")
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="tagged-checkpoint",
            execution_id="shared-execution",
            label="tagged",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        # No lease bound, and a tagged run has positive proof a checkpoint
        # exists: this legacy reader is not authoritative for that
        # partition and must be refused, not told "absent".
        with pytest.raises(CheckpointAccessRefusedError):
            DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            )
        # A reader that IS bound to the tagged run reads its own partition
        # and, for an execution id with no matching row there, gets the
        # authoritative "queried successfully, found nothing" result.
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert (
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "other-execution"
                )
                is None
            )
    finally:
        db.close()


def test_read_partition_keeps_bound_run_when_a_tagged_checkpoint_exists(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The common case is unaffected by the widening: once this run has a
    tagged checkpoint on record, a lease-bound reader stays confined to its
    own run partition even though an older, untagged row for the same
    execution id also exists."""
    SessionLocal, db, task = _create_trace_handler_test_task("tagged-keeps-bound")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="legacy-checkpoint",
                execution_id="shared-execution",
                label="legacy",
                timestamp=now,
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="tagged-checkpoint",
                execution_id="shared-execution",
                label="tagged",
                timestamp=now + timedelta(seconds=1),
                run_id="run-a",
            ),
        ]
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "tagged"}
    finally:
        db.close()


def test_readable_checkpoint_type_count_has_not_drifted() -> None:
    """The parametrization below derives from
    ``sorted(READABLE_CHECKPOINT_TYPES)``, so a newly added readable type
    already produces its own case automatically -- this assertion cannot
    catch a silently skipped one. Its job is to make adding a readable type
    a deliberate act: whoever changes this set must come here and confirm
    the widening's coupling to it still holds."""
    assert len(READABLE_CHECKPOINT_TYPES) == 2


def test_resolved_read_partition_rejects_widened_with_a_run_id() -> None:
    """``widened=True`` only ever pairs with ``run_id=None`` -- a run-bound
    read is never widened (see ``_ResolvedReadPartition``'s docstring).
    ``__post_init__`` makes that illegal combination unconstructable rather
    than a fact only the docstring asserts; the three legal combinations
    below must still construct."""
    with pytest.raises(ValueError):
        _ResolvedReadPartition(run_id="run-a", widened=True)

    _ResolvedReadPartition(run_id="run-a", widened=False)
    _ResolvedReadPartition(run_id=None, widened=True)
    _ResolvedReadPartition(run_id=None, widened=False)


@pytest.mark.parametrize(
    ("build_id", "partition"),
    [
        pytest.param(None, None, id="root_reader_with_no_partition"),
        pytest.param(
            "build-a",
            _ResolvedReadPartition(run_id=None, widened=False),
            id="build_scoped_reader_with_a_resolved_partition",
        ),
    ],
)
def test_unguarded_read_asserts_partition_build_id_lockstep(
    build_id: str | None,
    partition: _ResolvedReadPartition | None,
) -> None:
    """``partition`` must be ``None`` exactly when ``build_id`` is not
    ``None`` -- the caller (``_sync_load_latest_checkpoint``) is supposed to
    guarantee that pairing, but ``_sync_load_latest_checkpoint_unguarded``
    asserts it too instead of trusting the caller silently. A future caller
    that broke the pairing without this assertion would instead fall into
    the ``else`` branch, which filters on ``build_id`` alone and skips the
    run-partition filter entirely -- an unpartitioned cross-run read, the
    exact failure mode this whole mechanism exists to prevent. Both
    parametrized cases below violate the pairing in a different direction
    and neither reaches the database: the assertion fires before any query
    is built."""
    handler = DatabaseTraceHandler(task_id=1, build_id=build_id)
    with pytest.raises(AssertionError):
        handler._sync_load_latest_checkpoint_unguarded(
            db=None,  # type: ignore[arg-type]
            execution_id="shared-execution",
            partition=partition,
        )


@pytest.mark.parametrize("checkpoint_type", sorted(READABLE_CHECKPOINT_TYPES))
@pytest.mark.parametrize("pointer_set", [False, True])
def test_legacy_checkpoint_is_read_after_a_run_id_is_minted(
    monkeypatch: pytest.MonkeyPatch,
    pointer_set: bool,
    checkpoint_type: str,
) -> None:
    """#2023: a resume mints a fresh run id before writing any checkpoint
    under it. Both read paths that can serve the pre-existing, untagged
    checkpoint must still find it once the run is minted -- the exact-row
    pointer (when set) and the legacy scan (when it is not) -- and for
    every readable checkpoint type, not only the current one."""
    SessionLocal, db, task = _create_trace_handler_test_task(
        f"minted-run-{checkpoint_type}-{pointer_set}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="legacy-checkpoint",
        execution_id="shared-execution",
        label="legacy",
        timestamp=datetime.now(timezone.utc),
        checkpoint_type=checkpoint_type,
    )
    db.add(row)
    db.commit()
    db.refresh(row)
    if pointer_set:
        task.last_checkpoint_trace_event_id = row.id
        db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "legacy"}
    finally:
        db.close()


@pytest.mark.parametrize("field", _PK_ANCHOR_SINGLE_FAULT_FIELDS)
def test_widened_partition_is_confirmed_before_rejecting_a_row_failing_any_other_condition(
    monkeypatch: pytest.MonkeyPatch,
    field: str,
) -> None:
    """Widening the partition to the untagged rows relaxes exactly one
    conjunct -- the run tag -- and none of the others. A row that is
    otherwise a legitimate untagged candidate but is wrong in some other
    way must still be rejected as corrupt, for each of those conditions
    independently (mirrors
    test_database_trace_handler_load_pk_anchor_single_fault_raises_corrupt,
    under a widened rather than a tagged partition).

    Asserting only ``CheckpointCorruptError`` here would not require
    widening to work at all: an untagged row also fails
    ``CHECKPOINT_ROW_RUN_PARTITION`` under a *narrow* partition, so the
    same raise fires whether or not the lease-bound branch ever widens --
    reverting widening entirely still leaves 5 of these 6 cases green, for
    the wrong reason. Each case therefore first pins what
    ``_root_checkpoint_read_partition`` actually resolved to -- ``None``
    (widened) for every field except ``"run_partition"``, which resolves
    ``"run-a"`` (narrow) instead, per the note below -- so a reverted
    widening branch fails on that assertion, not silently on the
    unrelated field mutation."""
    SessionLocal, db, task = _create_trace_handler_test_task(
        f"widened-single-fault-{field}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)

    other_task = Task(
        user_id=int(task.user_id),
        title="Other task",
        description="Other task",
        status=TaskStatus.PENDING,
    )
    db.add(other_task)
    db.commit()
    db.refresh(other_task)
    other_task_id = int(other_task.id)

    row_kwargs: Dict[str, Any] = dict(
        task_id=task_id,
        build_id=None,
        event_id="widened-single-fault",
        event_type="system_update_general",
        timestamp=datetime.now(timezone.utc),
    )
    data: Dict[str, Any] = {
        "checkpoint_type": CHECKPOINT_TYPE,
        "execution_id": "shared-execution",
        "snapshot": {"label": "single-fault"},
    }
    # No run tag by default: the row is otherwise a legitimate candidate
    # for the widened (untagged) partition. The "run_partition" case is
    # the one field whose mutation also flips the probe: tagging the row
    # with a different run makes _task_has_run_tagged_checkpoint return
    # True, so the reader resolves the narrow (run-bound) partition
    # instead of the widened one. What this case actually proves is that
    # a row tagged with a foreign run is rejected while the reader stays
    # confined to its own run -- not that widening and a foreign-run-
    # tagged pointer can coexist. Under one consistent snapshot they
    # cannot: any run-tagged row on the task flips the probe before
    # widening can engage. That combination is reachable only through a
    # race between the probe and a concurrent tagged-checkpoint commit,
    # pinned separately below by
    # test_widened_read_never_returns_without_a_fresh_recheck and its
    # non-race counterpart,
    # test_widened_read_is_not_flagged_stale_when_nothing_concurrent_happens.
    _mutate_pk_anchor_field(field, row_kwargs, data, other_task_id)

    row = DatabaseTraceEvent(data=data, **row_kwargs)
    db.add(row)
    db.commit()
    db.refresh(row)
    task.last_checkpoint_trace_event_id = row.id
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    # Every field except "run_partition" leaves the row untagged, so the
    # probe finds no run-tagged checkpoint for this task and the reader
    # genuinely widens. "run_partition" tags the row itself (with a
    # foreign run), which flips the probe true and keeps the reader on
    # its own narrow partition instead -- see the comment above.
    expected_partition = "run-a" if field == "run_partition" else None

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert (
                DatabaseTraceHandler(task_id)._root_checkpoint_read_partition(db).run_id
                == expected_partition
            )
            with pytest.raises(CheckpointCorruptError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
    finally:
        db.close()


@pytest.mark.parametrize(
    "shape",
    ["pointer_snapshot", "scan_snapshot", "scan_absence", "scan_corrupt"],
)
def test_widened_read_never_returns_without_a_fresh_recheck(
    monkeypatch: pytest.MonkeyPatch,
    shape: str,
) -> None:
    """A widened read's result -- whichever of its four raw shapes it takes
    -- must never reach the caller without first being re-verified against
    a fresh probe.

    ``shape`` seeds the DB so the widened read, run alone, would naturally
    produce that raw outcome:

    * ``pointer_snapshot`` -- an untagged legacy row anchored by the
      pointer; ``_load_pk_anchored_checkpoint`` returns it directly. A
      pointer naming an *untagged* row is the shape a guard keyed on the
      pointer row's own tag cannot see at all, which is why the check
      lives at the boundary instead.
    * ``scan_snapshot`` -- the same row, but with no pointer set, so the
      legacy scan finds and returns it.
    * ``scan_absence`` -- no checkpoint row at all; the scan finds nothing
      and returns ``None``.
    * ``scan_corrupt`` -- an untagged row whose ``checkpoint_type`` is
      readable but which carries no ``snapshot`` payload; the scan
      exhausts the matching set with only that verdict and raises
      ``CheckpointCorruptError``.

    After the widened read finishes but before its result reaches the
    caller, a concurrent writer commits this run's first run-tagged
    checkpoint -- the same race is reachable through the scan path and
    through the pointer path alike. Every one of the four shapes must be
    discarded in favor of ``CheckpointUnavailableError`` once that commit
    lands -- not handed back as though the widening were still current.

    The injection wraps the real ``_sync_load_latest_checkpoint_unguarded``
    so the writer's commit lands strictly after the raw read completes and
    strictly before ``_sync_load_latest_checkpoint``'s own boundary check
    runs -- it does not shortcut around any production code path to
    manufacture the race."""
    SessionLocal, db, task = _create_trace_handler_test_task(f"widened-recheck-{shape}")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)

    if shape in ("pointer_snapshot", "scan_snapshot"):
        row = _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
        db.add(row)
        db.commit()
        db.refresh(row)
        if shape == "pointer_snapshot":
            task.last_checkpoint_trace_event_id = row.id
            db.commit()
    elif shape == "scan_corrupt":
        row = DatabaseTraceEvent(
            task_id=task_id,
            build_id=None,
            event_id="payloadless-checkpoint",
            event_type="system_update_general",
            timestamp=datetime.now(timezone.utc),
            data={
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                # No "snapshot" key: a readable checkpoint_type with no
                # payload, the scan's undecodable-row shape.
            },
        )
        db.add(row)
        db.commit()
    # scan_absence: no checkpoint row seeded at all.

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    real_unguarded = DatabaseTraceHandler._sync_load_latest_checkpoint_unguarded

    def commit_first_tagged_checkpoint_after_the_raw_read(
        self: DatabaseTraceHandler,
        db: Session,
        execution_id: str,
        partition: Any,
    ) -> Any:
        # try/finally, not a plain call-then-commit: the raw read can raise
        # (the scan_corrupt shape does), and the injected commit must still
        # land before that exception reaches the boundary guard -- the race
        # this simulates is "the writer commits strictly after the raw read
        # concludes", not "only when the raw read happens to succeed".
        try:
            return real_unguarded(self, db, execution_id, partition)
        finally:
            writer = SessionLocal()
            try:
                writer.add(
                    _checkpoint_trace_row(
                        task_id=task_id,
                        event_id="run-a-first-checkpoint",
                        execution_id="shared-execution",
                        label="new",
                        timestamp=datetime.now(timezone.utc),
                        run_id="run-a",
                    )
                )
                writer.commit()
            finally:
                writer.close()

    monkeypatch.setattr(
        DatabaseTraceHandler,
        "_sync_load_latest_checkpoint_unguarded",
        commit_first_tagged_checkpoint_after_the_raw_read,
    )

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointUnavailableError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
    finally:
        db.close()


def test_widened_read_is_not_flagged_stale_when_nothing_concurrent_happens(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Counterpart to the parametrized race pin above, proving the recheck
    is not an unconditional trap: a widened read with no concurrent writer
    at all -- the ordinary, overwhelmingly common case -- must still return
    its result normally. Task has no checkpoint row whatsoever, so this
    also covers the "no checkpoint task" shape without a race."""
    SessionLocal, db, task = _create_trace_handler_test_task("widened-recheck-no-race")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert (
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
                is None
            )
    finally:
        db.close()


def test_unleased_widened_read_is_guarded_against_a_concurrent_tag(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The legacy unleased branch of ``_root_checkpoint_read_partition``
    (no lease bound; the task has never had an active run) marks its
    result ``widened=True`` too, not only the lease-bound compatibility
    branch -- see ``_ResolvedReadPartition``'s own docstring for why both
    share the same point-in-time exposure. Without this, an unleased
    read's result would skip the boundary recheck entirely and could hand
    back a stale snapshot the same way a lease-bound read could before
    this fix.

    Two assertions, since either alone is not enough to pin it: (a) the
    resolver itself reports ``widened is True`` for an unleased read with
    no run-tagged checkpoint on record; (b) end to end, a concurrent
    writer committing this task's first run-tagged checkpoint between the
    raw read and the boundary check must turn an unleased read into
    ``CheckpointUnavailableError``, not a stale snapshot."""
    SessionLocal, db, task = _create_trace_handler_test_task("unleased-widened-guard")
    task_id = int(task.id)
    assert task.run_id is None

    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    # (a) the resolver itself, called directly with no lease bound.
    partition = DatabaseTraceHandler(task_id)._root_checkpoint_read_partition(db)
    assert partition.run_id is None
    assert partition.widened is True

    # (b) end to end: a writer commits the concurrent tag strictly after
    # the raw read completes and strictly before the boundary recheck
    # runs -- same injection technique as the parametrized race test
    # above, just without a bound lease.
    real_unguarded = DatabaseTraceHandler._sync_load_latest_checkpoint_unguarded

    def commit_first_tagged_checkpoint_after_the_raw_read(
        self: DatabaseTraceHandler,
        db: Session,
        execution_id: str,
        partition: Any,
    ) -> Any:
        try:
            return real_unguarded(self, db, execution_id, partition)
        finally:
            writer = SessionLocal()
            try:
                writer.add(
                    _checkpoint_trace_row(
                        task_id=task_id,
                        event_id="concurrent-first-checkpoint",
                        execution_id="shared-execution",
                        label="new",
                        timestamp=datetime.now(timezone.utc),
                        run_id="run-a",
                    )
                )
                writer.commit()
            finally:
                writer.close()

    monkeypatch.setattr(
        DatabaseTraceHandler,
        "_sync_load_latest_checkpoint_unguarded",
        commit_first_tagged_checkpoint_after_the_raw_read,
    )

    try:
        with pytest.raises(CheckpointUnavailableError):
            DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            )
    finally:
        db.close()


def test_recheck_probe_failure_surfaces_as_unavailable_not_a_snapshot(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The freshness recheck's own probe can fail for a genuine DB reason,
    distinct from the initial probe inside
    ``_root_checkpoint_read_partition`` that decided to widen in the first
    place. That failure must surface as ``CheckpointUnavailableError``, not
    be swallowed by the boundary and let the (unverified) widened result
    through as if it had passed the recheck."""
    SessionLocal, db, task = _create_trace_handler_test_task("recheck-probe-failure")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    real_probe = DatabaseTraceHandler._task_has_run_tagged_checkpoint
    calls = {"n": 0}

    def probe_ok_once_then_fails(self: DatabaseTraceHandler, db: Session) -> bool:
        calls["n"] += 1
        if calls["n"] == 1:
            # The initial probe inside _root_checkpoint_read_partition:
            # let it run for real so the partition genuinely widens.
            return real_probe(self, db)
        # The boundary's own recheck probe: fails for a DB reason, exactly
        # as _task_has_run_tagged_checkpoint's own except arm would raise.
        raise CheckpointUnavailableError(
            f"task {self.task_id}: could not determine whether a "
            "run-tagged checkpoint exists"
        )

    monkeypatch.setattr(
        DatabaseTraceHandler,
        "_task_has_run_tagged_checkpoint",
        probe_ok_once_then_fails,
    )

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointUnavailableError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
        assert calls["n"] == 2
    finally:
        db.close()


def test_run_bound_read_issues_no_extra_probe(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A read that resolves to its own run's narrow partition (the task
    already has a run-tagged checkpoint, so ``_root_checkpoint_read_partition``
    never widens) must not pay for the boundary's freshness recheck at all:
    exactly the one probe call ``_root_checkpoint_read_partition`` itself
    issues, and no more."""
    SessionLocal, db, task = _create_trace_handler_test_task("run-bound-no-extra-probe")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="tagged-checkpoint",
            execution_id="shared-execution",
            label="tagged",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    real_probe = DatabaseTraceHandler._task_has_run_tagged_checkpoint
    calls = {"n": 0}

    def counting_probe(self: DatabaseTraceHandler, db: Session) -> bool:
        calls["n"] += 1
        return real_probe(self, db)

    monkeypatch.setattr(
        DatabaseTraceHandler, "_task_has_run_tagged_checkpoint", counting_probe
    )

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "tagged"}
        assert calls["n"] == 1
    finally:
        db.close()


def test_probe_db_failure_raises_unavailable_and_registers_signal() -> None:
    """``_task_has_run_tagged_checkpoint`` must translate its own driver
    failure into ``CheckpointUnavailableError`` -- with the message only its
    own except arm produces, distinguishing it from the caller's raw-
    exception arm, which raises the same error type for every other
    partition-resolution failure -- and must register
    ``CHECKPOINT_LOAD_UNAVAILABLE`` itself, since the caller's arm that
    normally does so is never reached. Calls the helper directly with a stub
    session, so nothing in the DB layer is patched."""
    from xagent.web.services.ops_signals import (
        CHECKPOINT_LOAD_UNAVAILABLE,
        active_degradations,
        clear_degradation,
    )

    class _BrokenFirstQuery:
        def filter(self, *args: Any, **kwargs: Any) -> "_BrokenFirstQuery":
            return self

        def first(self) -> Any:
            raise OperationalError("boom", {}, Exception("boom"))

    class _StubSession:
        def query(self, *args: Any, **kwargs: Any) -> _BrokenFirstQuery:
            return _BrokenFirstQuery()

    clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
    try:
        with pytest.raises(CheckpointUnavailableError) as exc_info:
            DatabaseTraceHandler(1)._task_has_run_tagged_checkpoint(
                _StubSession()  # type: ignore[arg-type]
            )
        assert "could not determine whether a run-tagged checkpoint exists" in str(
            exc_info.value
        )
        assert CHECKPOINT_LOAD_UNAVAILABLE in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)


@pytest.mark.parametrize("leased", [True, False])
def test_probe_failure_registers_signal_from_both_callers(
    monkeypatch: pytest.MonkeyPatch,
    leased: bool,
) -> None:
    """The signal ``_task_has_run_tagged_checkpoint`` registers on its own DB
    failure must reach /health from two of its three call sites: the
    lease-bound branch of ``_root_checkpoint_read_partition`` (which calls
    it directly) and the legacy unleased branch (which calls it only after
    the task has no active run). The third call site -- the boundary
    recheck, ``_raise_if_widening_went_stale`` -- is not exercised here:
    patching the probe's own query to fail on its very first invocation
    makes the read fail during partition resolution, before a result exists
    for the boundary recheck to re-verify, so this test's injection never
    reaches that call site's signal registration. Patches ``Query.first``
    selectively -- only for the probe's own query, which is the only one in
    the read path selecting ``DatabaseTraceEvent.id`` alone -- so this fails
    loudly if the probe stops issuing that query instead of passing for the
    wrong reason."""
    from xagent.web.services.ops_signals import (
        CHECKPOINT_LOAD_UNAVAILABLE,
        active_degradations,
        clear_degradation,
    )

    SessionLocal, db, task = _create_trace_handler_test_task(
        f"probe-failure-signal-{'leased' if leased else 'unleased'}"
    )
    if leased:
        task.status = TaskStatus.RUNNING
        task.runner_id = "runner-a"
        task.lease_attempt_id = "test-attempt"
        task.run_id = "run-a"
        db.commit()
    else:
        # No lease bound below, and the task must carry no active run --
        # otherwise _root_checkpoint_read_partition refuses on "active_run"
        # before ever calling the probe. Task.run_id defaults to None.
        assert task.run_id is None

    task_id = int(task.id)

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    original_first = Query.first

    def selective_first(self: Query) -> Any:
        # Raise only for the run-tag probe's query -- it is the only one
        # selecting DatabaseTraceEvent.id alone. Every other .first() in the
        # read path runs normally, so this test fails loudly if the probe
        # stops issuing that query instead of passing for the wrong reason.
        descriptions = [d.get("name") for d in self.column_descriptions]
        if descriptions == ["id"]:
            raise OperationalError("boom", {}, Exception("boom"))
        return original_first(self)

    monkeypatch.setattr(Query, "first", selective_first)

    clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
    try:
        if leased:
            with bind_task_lease_context(
                TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
            ):
                with pytest.raises(CheckpointUnavailableError):
                    DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                        "shared-execution"
                    )
        else:
            with pytest.raises(CheckpointUnavailableError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
        assert CHECKPOINT_LOAD_UNAVAILABLE in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
        db.close()


def test_widening_self_extinguishes_after_the_first_tagged_checkpoint(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The widening is a transient compatibility window, not a standing
    relaxation: once the resumed run writes its own first checkpoint, that
    checkpoint is tagged, the probe starts finding it, and the reader goes
    back to being confined to the run partition -- the pre-existing
    untagged row stops being reachable through this reader."""
    SessionLocal, db, task = _create_trace_handler_test_task("self-extinguish")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            # Before this run has tagged anything, the partition itself is
            # widened -- not merely serving stale content that could
            # coincidentally match a narrower partition.
            partition = DatabaseTraceHandler(task_id)._root_checkpoint_read_partition(
                db
            )
            assert partition.run_id is None
            assert partition.widened is True
            # Before this run has tagged anything, the widened partition
            # still serves the pre-existing legacy checkpoint.
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "legacy"}

            DatabaseTraceHandler(task_id)._save_trace_event(
                db,
                TraceEvent(
                    CHECKPOINT_EVENT_TYPE,
                    task_id=str(task_id),
                    data={
                        "checkpoint_type": CHECKPOINT_TYPE,
                        "execution_id": "shared-execution",
                        "snapshot": {"label": "new"},
                    },
                ),
            )
            db.commit()

            # The task now has a tagged checkpoint on record: the partition
            # itself narrows back to the run id, not just the content that
            # happens to be read.
            partition = DatabaseTraceHandler(task_id)._root_checkpoint_read_partition(
                db
            )
            assert partition.run_id == "run-a"
            assert partition.widened is False
            # The run now has a tagged checkpoint of its own: the reader is
            # confined back to the run partition and reads that one, not
            # the untagged row that answered the first read.
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "new"}
    finally:
        db.close()


def test_widening_increments_its_counter_only_when_it_engages(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """``COUNTER_CHECKPOINT_READ_PARTITION_WIDENED`` must move exactly when
    the lease-bound branch actually widens the read partition: once for a
    read that widens, not again once the run's own tagged checkpoint
    narrows the partition back, and not for an unleased reader resolving to
    the untagged partition through its own pre-existing branch -- that
    branch is legacy behaviour this PR does not change, not the widening it
    adds. Uses ``counters_snapshot()`` deltas, never absolute values: the
    counter is process-global and other tests in the same process increment
    it too."""
    from xagent.web.services.interaction_rollout import (
        COUNTER_CHECKPOINT_READ_PARTITION_WIDENED,
        counters_snapshot,
    )

    def widened_count() -> int:
        return counters_snapshot().get(COUNTER_CHECKPOINT_READ_PARTITION_WIDENED, 0)

    SessionLocal, db, task = _create_trace_handler_test_task("widening-counter")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()

    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            # 1. No run-tagged checkpoint yet: the lease-bound read widens,
            # and the counter moves by exactly one.
            before = widened_count()
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "legacy"}
            assert widened_count() - before == 1

            DatabaseTraceHandler(task_id)._save_trace_event(
                db,
                TraceEvent(
                    CHECKPOINT_EVENT_TYPE,
                    task_id=str(task_id),
                    data={
                        "checkpoint_type": CHECKPOINT_TYPE,
                        "execution_id": "shared-execution",
                        "snapshot": {"label": "new"},
                    },
                ),
            )
            db.commit()

            # 2. The run now has a tagged checkpoint of its own: the
            # partition narrows back, so this read does not widen -- the
            # counter must not move again.
            before = widened_count()
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "new"}
            assert widened_count() - before == 0
    finally:
        db.close()

    # 3. An unleased reader with no active run and no tagged row resolves
    # to the untagged partition through the unleased branch's own
    # pre-existing return None, not the lease-bound widening this PR adds
    # -- the counter must not move.
    SessionLocal2, db2, task2 = _create_trace_handler_test_task(
        "widening-counter-unleased"
    )
    task2_id = int(task2.id)
    db2.add(
        _checkpoint_trace_row(
            task_id=task2_id,
            event_id="legacy-checkpoint",
            execution_id="shared-execution",
            label="legacy",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db2.commit()

    def get_test_db2() -> Iterator[Session]:
        session = SessionLocal2()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db2)

    try:
        before = widened_count()
        assert DatabaseTraceHandler(task2_id)._sync_load_latest_checkpoint(
            "shared-execution"
        ) == {"label": "legacy"}
        assert widened_count() - before == 0
    finally:
        db2.close()


def test_database_trace_handler_prunes_only_bound_run_partition(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    _, db, task = _create_trace_handler_test_task("run-scoped-prune")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="run-a-old",
                execution_id="shared-execution",
                label="run-a-old",
                timestamp=now,
                run_id="run-a",
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="run-a-new",
                execution_id="shared-execution",
                label="run-a-new",
                timestamp=now + timedelta(seconds=1),
                run_id="run-a",
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="run-b-old",
                execution_id="shared-execution",
                label="run-b-old",
                timestamp=now + timedelta(seconds=2),
                run_id="run-b",
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="run-b-new",
                execution_id="shared-execution",
                label="run-b-new",
                timestamp=now + timedelta(seconds=3),
                run_id="run-b",
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="legacy",
                execution_id="shared-execution",
                label="legacy",
                timestamp=now + timedelta(seconds=4),
            ),
        ]
    )
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-b",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"run-a-old", "run-a-new", "run-b-new", "legacy"}
    finally:
        db.close()


def test_database_trace_handler_prunes_legacy_partition_without_tagged_runs(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    _, db, task = _create_trace_handler_test_task("legacy-scoped-prune")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="legacy-old",
                execution_id="shared-execution",
                label="legacy-old",
                timestamp=now,
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="legacy-new",
                execution_id="shared-execution",
                label="legacy-new",
                timestamp=now + timedelta(seconds=1),
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="tagged-run",
                execution_id="shared-execution",
                label="tagged-run",
                timestamp=now + timedelta(seconds=2),
                run_id="run-a",
            ),
        ]
    )
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"legacy-new", "tagged-run"}
    finally:
        db.close()


def test_database_trace_handler_prune_excludes_the_anchored_row(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Synthetic: the task's pointer is wired to an older row that the
    retention ranking would otherwise prune. Nothing in today's
    steady-state writer produces this -- the pointer always names the row
    just written, which ranks newest -- but a future back-pointing anchor
    (or a backfilled pointer) could, so prune must never delete the row a
    pointer references regardless of its rank."""
    _, db, task = _create_trace_handler_test_task("anchor-protected-prune")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    old_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="anchored-old",
        execution_id="shared-execution",
        label="anchored-old",
        timestamp=now,
        run_id="run-a",
    )
    db.add(old_row)
    db.commit()
    db.refresh(old_row)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="run-a-new",
            execution_id="shared-execution",
            label="run-a-new",
            timestamp=now + timedelta(seconds=1),
            run_id="run-a",
        )
    )
    # Synthetic: wire the pointer to the older row, which the retention
    # ranking (newest first, limit=1) would otherwise prune.
    task.last_checkpoint_trace_event_id = old_row.id
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"anchored-old", "run-a-new"}
    finally:
        db.close()


def test_prune_protects_a_trace_row_anchored_by_an_active_interaction(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The second protection source (PR-C2b): an active interaction row's
    resume_trace_event_id keeps its anchor alive even when the anchor ranks
    outside the retention window, the same way the task's exact-row pointer
    does in the test above -- but through task_interaction_requests instead
    of tasks.last_checkpoint_trace_event_id."""
    _, db, task = _create_trace_handler_test_task("interaction-anchor-protected")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    old_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="interaction-anchored-old",
        execution_id="shared-execution",
        label="interaction-anchored-old",
        timestamp=now,
        run_id="run-a",
    )
    db.add(old_row)
    db.commit()
    db.refresh(old_row)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="run-a-new",
            execution_id="shared-execution",
            label="run-a-new",
            timestamp=now + timedelta(seconds=1),
            run_id="run-a",
        )
    )
    db.commit()
    _seed_interaction_row(
        db, task_id=task_id, resume_trace_event_id=old_row.id, status="active"
    )
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"interaction-anchored-old", "run-a-new"}
    finally:
        db.close()


def test_prune_does_not_protect_a_terminal_interaction_anchor(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Reverse control for the test above: a terminated interaction row's
    resume_trace_event_id must not enter the protection set -- only
    active_slot IS NOT NULL rows do -- or the protection set would keep
    growing forever as interactions terminate instead of shrinking back to
    just the exact-row pointer."""
    _, db, task = _create_trace_handler_test_task("interaction-anchor-terminal")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    old_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="terminal-anchored-old",
        execution_id="shared-execution",
        label="terminal-anchored-old",
        timestamp=now,
        run_id="run-a",
    )
    db.add(old_row)
    db.commit()
    db.refresh(old_row)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="run-a-new",
            execution_id="shared-execution",
            label="run-a-new",
            timestamp=now + timedelta(seconds=1),
            run_id="run-a",
        )
    )
    db.commit()
    _seed_interaction_row(
        db, task_id=task_id, resume_trace_event_id=old_row.id, status="terminated"
    )
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"run-a-new"}
    finally:
        db.close()


def test_prune_protects_an_expired_but_active_interaction_anchor(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Deliberately no expires_at filter on the protection-set query: an
    expired-but-still-active request (nothing has terminated it yet) is
    still answerable, so its anchor must stay protected."""
    _, db, task = _create_trace_handler_test_task("interaction-anchor-expired")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    old_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="expired-anchored-old",
        execution_id="shared-execution",
        label="expired-anchored-old",
        timestamp=now,
        run_id="run-a",
    )
    db.add(old_row)
    db.commit()
    db.refresh(old_row)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="run-a-new",
            execution_id="shared-execution",
            label="run-a-new",
            timestamp=now + timedelta(seconds=1),
            run_id="run-a",
        )
    )
    db.commit()
    _seed_interaction_row(
        db,
        task_id=task_id,
        resume_trace_event_id=old_row.id,
        status="active",
        created_at=now - timedelta(hours=2),
        expires_at=now - timedelta(hours=1),
    )
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"expired-anchored-old", "run-a-new"}
    finally:
        db.close()


def test_prune_retains_at_most_limit_plus_two(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Both protection sources at once, anchoring two distinct rows: the
    exact-row pointer and the active interaction anchor are the two
    exceptions the docstring's limit + 2 bound accounts for."""
    _, db, task = _create_trace_handler_test_task("limit-plus-two")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    pointer_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="pointer-anchored-oldest",
        execution_id="shared-execution",
        label="pointer-anchored-oldest",
        timestamp=now,
        run_id="run-a",
    )
    db.add(pointer_row)
    db.commit()
    db.refresh(pointer_row)
    interaction_anchor_row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="interaction-anchored-middle",
        execution_id="shared-execution",
        label="interaction-anchored-middle",
        timestamp=now + timedelta(seconds=1),
        run_id="run-a",
    )
    db.add(interaction_anchor_row)
    db.commit()
    db.refresh(interaction_anchor_row)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="run-a-newest",
            execution_id="shared-execution",
            label="run-a-newest",
            timestamp=now + timedelta(seconds=2),
            run_id="run-a",
        )
    )
    task.last_checkpoint_trace_event_id = pointer_row.id
    db.commit()
    _seed_interaction_row(
        db,
        task_id=task_id,
        resume_trace_event_id=interaction_anchor_row.id,
        status="active",
    )
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        remaining = {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        }
        assert remaining == {
            "pointer-anchored-oldest",
            "interaction-anchored-middle",
            "run-a-newest",
        }
        assert len(remaining) == 3  # limit (1) + 2
    finally:
        db.close()


def test_prune_retains_exactly_the_limit_when_the_interaction_anchor_is_in_range(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Interaction-anchor analog of
    test_database_trace_handler_prune_retains_exactly_the_limit_when_the_anchor_is_in_range:
    when the protected row already ranks inside the retention window, its
    protection is redundant with natural retention, and the retained count
    must stay exactly ``limit``.

    This is also the test that actually distinguishes "exclude protected
    rows from the ranking query, then OFFSET" (M9's bug) from "OFFSET
    first, then exclude protected rows from the stale set" (the correct
    order): when every protected row ranks outside the window (as in
    test_prune_retains_at_most_limit_plus_two above), both orderings
    produce the identical final set -- removing a row that already sits
    past the OFFSET boundary cannot change which rows are within it. The
    divergence only appears when a protected row ranks inside the window,
    which is exactly this scenario.
    """
    _, db, task = _create_trace_handler_test_task("interaction-anchor-in-range")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    rows = [
        _checkpoint_trace_row(
            task_id=task_id,
            event_id=event_id,
            execution_id="shared-execution",
            label=event_id,
            timestamp=now + timedelta(seconds=offset),
            run_id="run-a",
        )
        for offset, event_id in enumerate(["oldest", "middle", "newest"])
    ]
    db.add_all(rows)
    db.commit()
    db.refresh(rows[-1])
    _seed_interaction_row(
        db, task_id=task_id, resume_trace_event_id=rows[-1].id, status="active"
    )
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"newest"}
    finally:
        db.close()


def test_prune_runs_without_the_interaction_table(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The has_table gate's reason to exist: on a deployment upgraded to a
    revision before task_interaction_requests exists, prune must still
    complete, still reclaim stale rows, and must not register
    CHECKPOINT_PRUNE_FAILED -- registering it here would be exactly the
    failure mode measured in the audit (SQLite's OperationalError landing
    in the (IntegrityError, OperationalError) handler on every checkpoint
    write, with no way for an operator to ever clear it)."""
    _, db, task = _create_trace_handler_test_task_without_interaction_table(
        "prune-without-interaction-table"
    )
    task_id = int(task.id)
    assert interaction_requests_table_exists(db) is False
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="no-table-old",
                execution_id="shared-execution",
                label="no-table-old",
                timestamp=now,
                run_id="run-a",
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="no-table-new",
                execution_id="shared-execution",
                label="no-table-new",
                timestamp=now + timedelta(seconds=1),
                run_id="run-a",
            ),
        ]
    )
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    clear_degradation(CHECKPOINT_PRUNE_FAILED)
    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert CHECKPOINT_PRUNE_FAILED not in active_degradations()
        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"no-table-new"}
    finally:
        clear_degradation(CHECKPOINT_PRUNE_FAILED)
        db.close()


@pytest.mark.parametrize(
    "injected_exception",
    [
        pytest.param(
            IntegrityError("DELETE", {}, Exception("restrict violation")),
            id="integrity_error",
        ),
        pytest.param(
            OperationalError("DELETE", {}, Exception("serialization failure")),
            id="operational_error",
        ),
    ],
)
def test_database_trace_handler_prune_registers_degradation_on_delete_error(
    monkeypatch: pytest.MonkeyPatch,
    injected_exception: Exception,
) -> None:
    """Synthetic delete-failure injection: the delete step is replaced to
    raise, isolating the except-(IntegrityError, OperationalError) branch
    from needing genuine FK enforcement (which only PostgreSQL, or a
    freshly create_all'd SQLite database, provides for this column) or a
    genuine serialization conflict. Both exception classes are covered
    because PostgreSQL's restrict-violation and its SerializationFailure /
    DeadlockDetected surface through different SQLAlchemy wrapper classes
    (IntegrityError vs. OperationalError) for the same retention race."""
    from sqlalchemy.orm import Query

    from xagent.web.services.ops_signals import (
        CHECKPOINT_PRUNE_FAILED,
        active_degradations,
        clear_degradation,
    )

    _, db, task = _create_trace_handler_test_task("prune-failure")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add_all(
        [
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="old",
                execution_id="shared-execution",
                label="old",
                timestamp=now,
            ),
            _checkpoint_trace_row(
                task_id=task_id,
                event_id="new",
                execution_id="shared-execution",
                label="new",
                timestamp=now + timedelta(seconds=1),
            ),
        ]
    )
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    def failing_delete(self: Query, *args: object, **kwargs: object) -> int:
        raise injected_exception

    monkeypatch.setattr(Query, "delete", failing_delete)

    clear_degradation(CHECKPOINT_PRUNE_FAILED)
    try:
        # Must not raise -- a prune failure can never break the checkpoint
        # write it follows.
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {"checkpoint_type": CHECKPOINT_TYPE, "execution_id": "shared-execution"},
        )
        assert CHECKPOINT_PRUNE_FAILED in active_degradations()
        assert db.query(DatabaseTraceEvent).filter_by(task_id=task_id).count() == 2
    finally:
        clear_degradation(CHECKPOINT_PRUNE_FAILED)
        db.close()


def test_websocket_trace_handler_dedupes_prior_user_message_turn_id(
    monkeypatch,
) -> None:
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()
    try:
        user = User(username="tester", password_hash="hashed_password", is_admin=False)
        db.add(user)
        db.commit()
        db.refresh(user)
        task = Task(
            user_id=int(user.id),
            title="Chat task",
            description="Task chat",
            status=TaskStatus.PENDING,
        )
        db.add(task)
        db.commit()
        db.refresh(task)
        task_id = int(task.id)
        db.add(
            DatabaseTraceEvent(
                task_id=task_id,
                event_id="first-event",
                event_type="user_message",
                timestamp=task.created_at,
                data={"message": "Repeat", "turn_id": "turn-1"},
            )
        )
        db.commit()
    finally:
        db.close()

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)

    handler = TaskEventTraceHandler(task_id)
    assert not handler._has_prior_user_message_turn(
        "user_message", {"turn_id": "turn-1"}, "first-event"
    )
    assert handler._has_prior_user_message_turn(
        "user_message", {"turn_id": "turn-1"}, "second-event"
    )
    assert not handler._has_prior_user_message_turn(
        "user_message", {"turn_id": "turn-2"}, "second-event"
    )


# ---------------------------------------------------------------------------
# Tests for skipping task event trace work when a task has no audience, and
# for dropping agent checkpoint data before serializing it rather than
# after. These land in this file because it is already the landing spot for
# TaskEventTraceHandler / checkpoint-stream coverage.
#
# The handler no longer reaches for a connection manager: it asks
# ``task_has_audience`` and publishes through ``publish_task_event``, both of
# which the hosting process backs with hooks. A test that changes the
# audience therefore changes both hooks, and points them at the same local
# connection manager -- a probe reading one registry while the sink delivers
# to another is exactly the misconfiguration ``set_task_event_sink`` exists
# to prevent. ``publish_task_event`` itself is never replaced: it is the only
# seam between the handler and the outside, and the case that pins the
# registry being re-read at delivery time depends on the real broadcast path
# running.
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "case_id, event_type, step_id, data",
    [
        (
            "checkpoint",
            CHECKPOINT_EVENT_TYPE,
            None,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "exec-i1",
                "snapshot": {"label": "before_llm"},
            },
        ),
        (
            "llm_call_start",
            TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
            "step-1",
            {"model": "gpt-test"},
        ),
        (
            "user_message",
            TraceEventType(TraceScope.TASK, TraceAction.START, TraceCategory.MESSAGE),
            None,
            {"message": "hi", "turn_id": "turn-i1"},
        ),
        (
            "data_not_a_dict",
            CHECKPOINT_EVENT_TYPE,
            None,
            ["not", "a", "dict"],
        ),
    ],
)
async def test_handle_event_does_no_work_without_an_attached_audience(
    case_id, event_type, step_id, data, monkeypatch
) -> None:
    """No audience means no work.

    Every frame ``handle_event`` could build for an unattached task would be
    discarded further down the delivery path anyway, so the audience check at
    the top of ``handle_event`` must make the whole call a no-op -- no
    serialization, no thread-pool submission, no publish -- regardless of the
    event's shape, and the skip must be counted so it is visible in the
    handler's event totals rather than absent from them. Covers a checkpoint
    event, an ordinary event, a ``user_message`` event (the one type that
    would otherwise trigger an extra database read in a worker thread inside
    ``_has_prior_user_message_turn``), and a non-dict payload.
    """
    task_id = 90101
    local_manager = ConnectionManager()
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    serialize_calls: list[object] = []
    monkeypatch.setattr(
        "xagent.web.services.task_event_trace_handler.serialize_trace_data",
        lambda payload: serialize_calls.append(payload) or payload,
    )

    thread_calls: list[object] = []

    async def fake_run_in_thread(operation, func, /, *args, **kwargs):
        thread_calls.append((operation, func, args, kwargs))
        return func(*args, **kwargs)

    monkeypatch.setattr(
        "xagent.web.services.task_event_trace_handler.run_in_thread_with_telemetry",
        fake_run_in_thread,
    )

    counter_calls: list[object] = []
    monkeypatch.setattr(
        "xagent.web.services.task_event_trace_handler.increment_counter",
        lambda name, attributes=None: counter_calls.append((name, attributes)),
    )

    broadcast_calls: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        broadcast_calls.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    handler = TaskEventTraceHandler(task_id)
    event = TraceEvent(
        event_type,
        task_id=str(task_id),
        step_id=step_id,
        data=data,
        timestamp=1_700_000_000.0,
    )

    await handler.handle_event(event)

    assert serialize_calls == []
    assert thread_calls == []
    assert broadcast_calls == []
    assert counter_calls == [
        ("xagent.websocket.trace.events", {"outcome": "no_audience"})
    ]


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "case_id",
    [
        "llm_call_start",
        "tool_execution_end",
        "user_message_first_turn",
        "user_message_duplicate_turn",
    ],
)
async def test_broadcast_payload_is_unchanged_for_an_attached_audience(
    case_id, monkeypatch
) -> None:
    """The payload handed to the host's event sink is unchanged for an
    attached audience.

    Captures at the sink's own input, not at ``send_text`` --
    the ``run_id`` / ``state_version`` stamping
    ``_with_current_task_control_state`` adds happens strictly after this
    point and depends on database state that would make a hardcoded
    expectation flaky. ``task_description`` is set directly on the handler
    (not read from the database) on purpose: that keeps the expected
    payload pinnable without depending on ``_load_task_description``'s own
    read path, which
    ``test_task_description_load_is_deferred_until_an_audience_attaches``
    covers, not this one.

    Deliberately excludes the three event families
    ``normalize_public_trace_event`` rebuilds from a field whitelist --
    ``GENERAL_ERROR_EVENT_TYPES``, a failed ``PATTERN_END_EVENT_TYPES``
    event, and a ``task_update_general`` carrying a delegation summary.
    None of those whitelists carry ``task_description``, so adding one of
    those events to this parametrization without a different expected
    payload would fail for a reason that has nothing to do with this
    change. The ``tool_execution_end`` case is still routed through
    ``redact_runtime_sensitive_payload`` (every ``tool_execution_*`` event
    is); its payload here uses only non-sensitive key names so the redirect
    is a no-op and the hardcoded expectation still matches.
    """
    SessionLocal, db, task = _create_trace_handler_test_task("i2-audience")
    task_id = int(task.id)
    if case_id == "user_message_duplicate_turn":
        db.add(
            DatabaseTraceEvent(
                task_id=task_id,
                event_id="i2-earlier-turn",
                event_type="user_message",
                timestamp=task.created_at,
                data={"message": "first", "turn_id": "turn-i2-dup"},
            )
        )
        db.commit()
    db.close()

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)

    local_manager = ConnectionManager()
    local_manager.register_connection(object(), task_id)
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    broadcast_calls: list[tuple[dict, int]] = []

    async def fake_broadcast(message, task_id_arg):
        broadcast_calls.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    handler = TaskEventTraceHandler(task_id)
    handler._task_description = "Task chat description"
    handler._task_description_loaded = True

    fixed_timestamp = 1_700_000_123.0
    expected_event_type: str | None
    expected_data: dict | None

    if case_id == "llm_call_start":
        event_type = TraceEventType(
            TraceScope.ACTION, TraceAction.START, TraceCategory.LLM
        )
        step_id = "step-llm"
        data = {"model": "gpt-test", "prompt": "hello"}
        expected_event_type = "llm_call_start"
        expected_data = {
            "model": "gpt-test",
            "prompt": "hello",
            "task_description": "Task chat description",
        }
    elif case_id == "tool_execution_end":
        event_type = TraceEventType(
            TraceScope.ACTION, TraceAction.END, TraceCategory.TOOL
        )
        step_id = "step-tool"
        data = {"tool_name": "search", "output": "result text"}
        expected_event_type = "tool_execution_end"
        expected_data = {
            "tool_name": "search",
            "output": "result text",
            "task_description": "Task chat description",
        }
    elif case_id == "user_message_first_turn":
        event_type = TraceEventType(
            TraceScope.TASK, TraceAction.START, TraceCategory.MESSAGE
        )
        step_id = None
        data = {"message": "hello there", "turn_id": "turn-i2-fresh"}
        expected_event_type = "user_message"
        expected_data = {
            "message": "hello there",
            "turn_id": "turn-i2-fresh",
            "task_description": "Task chat description",
        }
    else:
        event_type = TraceEventType(
            TraceScope.TASK, TraceAction.START, TraceCategory.MESSAGE
        )
        step_id = None
        data = {"message": "repeat", "turn_id": "turn-i2-dup"}
        expected_event_type = None
        expected_data = None

    event = TraceEvent(
        event_type,
        task_id=str(task_id),
        step_id=step_id,
        data=data,
        timestamp=fixed_timestamp,
    )

    await handler.handle_event(event)

    if case_id == "user_message_duplicate_turn":
        assert broadcast_calls == []
        return

    assert len(broadcast_calls) == 1
    message, broadcast_task_id = broadcast_calls[0]
    assert broadcast_task_id == task_id
    assert message["type"] == "trace_event"
    assert message["event_type"] == expected_event_type
    assert message["task_id"] == task_id
    assert message["timestamp"] == fixed_timestamp
    assert isinstance(message["event_id"], str) and message["event_id"]
    assert message["data"] == expected_data
    if step_id is not None:
        assert message["step_id"] == step_id
    else:
        assert "step_id" not in message
    assert "parent_id" not in message


@pytest.mark.parametrize(
    "case_id",
    [
        "current_shape_checkpoint",
        "legacy_shape_both_dicts",
        "legacy_shape_pattern_state_none",
        "legacy_shape_context_none",
        "legacy_shape_pattern_state_model_dump_object",
        "legacy_shape_context_to_dict_object",
        "non_checkpoint",
        "current_shape_checkpoint_unserializable_snapshot",
        "current_shape_checkpoint_circular_reference_snapshot",
    ],
)
def test_raw_and_serialized_checkpoint_checks_drop_the_same_set(case_id) -> None:
    """The pre-serialization checkpoint check and the post-serialization one
    drop the same set of frames, from the client's point of view.

    Both checks stay because neither subsumes the other -- see the comment
    at their call site in ``_convert_trace_event_to_stream_event``. This
    parametrization is the mechanical guard for that claim, plus the one
    behavioral difference the raw check introduces (the
    ``_unserializable_snapshot`` case) and the one case where today's code
    already drops the frame for an unrelated reason (the
    ``_circular_reference_snapshot`` case, ``RecursionError``).

    Per-case commentary is written as an inline string literal at the top
    of that case's branch, rather than as this function's own docstring,
    because several cases each carry a distinct piece of text and a single
    function can only have one docstring.
    """
    task_id = 90308
    handler = TaskEventTraceHandler(task_id)

    if case_id == "current_shape_checkpoint":
        data = {
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "exec-i3-1",
            "snapshot": {"label": "before_llm"},
        }
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is None

    elif case_id == "legacy_shape_both_dicts":
        data = {"type": "checkpoint", "pattern_state": {}, "context": {}}
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is None

    elif case_id == "legacy_shape_pattern_state_none":
        """Characterization, not endorsement.

        ``is_agent_checkpoint_data``'s second branch requires *both*
        ``pattern_state`` and ``context`` to be dicts
        (``task_event_trace_handler.py:233-245``), while the producer at
        ``core/agent/runtime.py:1655-1682`` leaves either one ``None`` when the
        pattern has no ``get_state`` or the context has no ``to_dict`` -- and
        attaches ``execution_snapshot`` independently of both. A payload of that
        shape is therefore not classified as a checkpoint and does reach the
        client today, ``type: "checkpoint"`` and all. This case records that
        behaviour so a change to it is visible; it is not a statement that
        shipping such a payload to a browser is correct. If that is later judged
        a leak, fix it deliberately and update this case -- do not assume a red
        here means the predicate was broken.
        """
        data = {"type": "checkpoint", "pattern_state": None, "context": {}}
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is not None

    elif case_id == "legacy_shape_context_none":
        """Characterization, not endorsement.

        ``is_agent_checkpoint_data``'s second branch requires *both*
        ``pattern_state`` and ``context`` to be dicts
        (``task_event_trace_handler.py:233-245``), while the producer at
        ``core/agent/runtime.py:1655-1682`` leaves either one ``None`` when the
        pattern has no ``get_state`` or the context has no ``to_dict`` -- and
        attaches ``execution_snapshot`` independently of both. A payload of that
        shape is therefore not classified as a checkpoint and does reach the
        client today, ``type: "checkpoint"`` and all. This case records that
        behaviour so a change to it is visible; it is not a statement that
        shipping such a payload to a browser is correct. If that is later judged
        a leak, fix it deliberately and update this case -- do not assume a red
        here means the predicate was broken.
        """
        data = {"type": "checkpoint", "pattern_state": {}, "context": None}
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is not None

    elif case_id == "legacy_shape_pattern_state_model_dump_object":
        """Regression guard, not a description of today's traffic.

        The standard web execution path wires ``PatternRuntime`` with a
        ``TraceCheckpointStore`` (``service.py:579`` -> ``:593`` ->
        ``execution_adapter.py:296`` -> ``runner.py:235``), and that store exposes
        both ``checkpoint`` and ``write_checkpoint``, so ``_emit_checkpoint``
        always takes its first branch and nests the legacy payload under
        ``snapshot``. No producer on that path emits the top-level legacy shape
        this case feeds in. The case is kept because the same predicate logic has
        two implementations and three call sites, one of them in historical
        replay. No stored rows of this shape were found when this was audited;
        the check is kept so this copy of the predicate stays in step with the
        copy historical replay calls. Do not delete the post-serialization
        check on the grounds that this case is synthetic.
        """

        class _ModelDumpPatternState:
            def model_dump(self) -> dict:
                return {"steps": []}

        data = {
            "type": "checkpoint",
            "pattern_state": _ModelDumpPatternState(),
            "context": {},
        }
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is None

    elif case_id == "legacy_shape_context_to_dict_object":
        """Regression guard, not a description of today's traffic.

        The standard web execution path wires ``PatternRuntime`` with a
        ``TraceCheckpointStore`` (``service.py:579`` -> ``:593`` ->
        ``execution_adapter.py:296`` -> ``runner.py:235``), and that store exposes
        both ``checkpoint`` and ``write_checkpoint``, so ``_emit_checkpoint``
        always takes its first branch and nests the legacy payload under
        ``snapshot``. No producer on that path emits the top-level legacy shape
        this case feeds in. The case is kept because the same predicate logic has
        two implementations and three call sites, one of them in historical
        replay. No stored rows of this shape were found when this was audited;
        the check is kept so this copy of the predicate stays in step with the
        copy historical replay calls. Do not delete the post-serialization
        check on the grounds that this case is synthetic.
        """

        class _ToDictContext:
            def to_dict(self) -> dict:
                return {"messages": []}

        data = {
            "type": "checkpoint",
            "pattern_state": {},
            "context": _ToDictContext(),
        }
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is None

    elif case_id == "non_checkpoint":
        data = {"model": "gpt-test", "prompt": "hello"}
        event = TraceEvent(
            TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
            task_id=str(task_id),
            step_id="step-1",
            data=data,
            timestamp=1.0,
        )
        assert handler._convert_trace_event_to_stream_event(event) is not None

    elif case_id == "current_shape_checkpoint_unserializable_snapshot":
        """The one drop this change adds.

        Before this change the frame went out as an error placeholder:
        ``serialize_trace_data`` replaces the whole payload with
        ``_serialization_error`` / ``_original_type`` / ``_error``
        (``task_event_trace_handler.py:347-351``), and ``:565-566`` then adds
        ``task_description`` on top whenever the task has a non-empty
        description -- so the client received a frame carrying none of the
        checkpoint's own content. After this change the raw check catches it
        first and nothing is broadcast.

        This case is the mechanical counterpart of the reverse constraint
        appended to ``serialize_trace_data``'s docstring: it holds the raw check
        responsible for every payload that pass replaces wholesale. If someone
        adds truncation or field pruning there, this case is where it shows up.
        """

        class _Unserializable:
            pass

        data = {
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "exec-i3-8",
            "snapshot": {"weird": _Unserializable()},
        }
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is None

    else:
        assert case_id == "current_shape_checkpoint_circular_reference_snapshot"
        # Before this change, a circular reference in the snapshot made
        # ``serialize_trace_data`` raise ``RecursionError`` while walking it;
        # that exception propagated out of
        # ``_convert_trace_event_to_stream_event`` uncaught and was only
        # stopped by ``handle_event``'s own ``except Exception``, which drops
        # the frame and logs a warning. No client received it before this
        # change either -- the only difference after this change is that the
        # raw check catches it first and that warning is no longer written.
        circular: dict = {}
        circular["self"] = circular
        data = {
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "exec-i3-9",
            "snapshot": circular,
        }
        event = TraceEvent(
            CHECKPOINT_EVENT_TYPE, task_id=str(task_id), data=data, timestamp=1.0
        )
        assert handler._convert_trace_event_to_stream_event(event) is None


@pytest.mark.asyncio
async def test_checkpoint_event_with_an_audience_never_calls_serialize_trace_data(
    monkeypatch,
) -> None:
    """Extra guard on the ``serialize_trace_data`` call count: with an
    audience attached, a checkpoint event never reaches
    ``serialize_trace_data`` at all -- the raw check in
    ``_convert_trace_event_to_stream_event`` returns before
    ``_serialize_data`` is called.
    """
    task_id = 90309
    local_manager = ConnectionManager()
    local_manager.register_connection(object(), task_id)
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    serialize_calls: list[object] = []
    monkeypatch.setattr(
        "xagent.web.services.task_event_trace_handler.serialize_trace_data",
        lambda payload: serialize_calls.append(payload) or payload,
    )

    handler = TaskEventTraceHandler(task_id)
    handler._task_description = None
    handler._task_description_loaded = True
    event = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task_id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": "exec-audience",
            "snapshot": {"label": "before_llm"},
        },
        timestamp=1_700_000_800.0,
    )

    await handler.handle_event(event)

    assert serialize_calls == []


class _MinimalDuckTypedSink:
    """A stand-in that only implements ``send_text``, the one attribute
    ``ConnectionManager`` and its callers actually require of a registered
    connection -- used to cover the audience paths that do not need a real
    ``starlette`` ``WebSocket`` (v1 SSE, and the downstream adapters that
    hand the engine's ``register_connection`` a different object type)."""

    def __init__(self) -> None:
        self.sent: list[str] = []

    async def send_text(self, text: str) -> None:
        self.sent.append(text)


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "case_id, register",
    [
        (
            "web_chat_websocket",
            lambda m, t: m.register_connection(_MinimalDuckTypedSink(), t),
        ),
        # Builder preview: no historical replay, and no reconciliation
        # channel either.
        #
        # The web chat, guest widget and share-link paths call
        # ``handle_status_request`` right after registering, so a frame
        # skipped before attach still reaches the client from the database.
        # Several paths do not replay -- v1 SSE among them -- but each of
        # those documents a reconciliation channel the client is expected to
        # use instead. This one has neither.
        #
        # Its safety rests on ordering **for the first preview turn on a
        # socket**: ``websocket.py:3604`` writes
        # ``websocket.state.preview_task_id`` one line before
        # ``register_connection``, and the task starts only after that. **On
        # later turns the branch at ``websocket.py:3606`` reuses the id
        # without re-registering**, so safety then rests on the socket still
        # being in the registry -- which ``broadcast_to_task`` can revoke on
        # a send error (``websocket.py:1371``) without clearing
        # ``preview_task_id``. That case is not a regression here: with the
        # socket out of the registry, the frame is dropped before this
        # change (by ``broadcast_to_task``) and after it (by the audience
        # check), so no client sees a difference either way.
        #
        # If anyone adds reconnect, resume, or any second way to bind a
        # socket to a preview task that is already running, the skip becomes
        # a permanent frame loss on this path. Re-review the audience check
        # before adding one.
        (
            "builder_preview_websocket",
            lambda m, t: m.register_connection(_MinimalDuckTypedSink(), t),
        ),
        (
            "v1_sse_sink",
            lambda m, t: m.register_connection(_MinimalDuckTypedSink(), t),
        ),
        (
            "guest_widget_websocket",
            lambda m, t: m.register_connection(_MinimalDuckTypedSink(), t),
        ),
        (
            "share_link_websocket",
            lambda m, t: m.register_connection(_MinimalDuckTypedSink(), t),
        ),
        (
            "downstream_duck_typed_adapter",
            lambda m, t: m.register_connection(_MinimalDuckTypedSink(), t),
        ),
    ],
)
async def test_every_audience_registration_is_visible_to_the_audience_check(
    case_id, register, monkeypatch
) -> None:
    """The audience check's only source of truth is ``ConnectionManager``.

    All six audience paths -- the web chat WebSocket, the Builder preview
    WebSocket, a v1 SSE sink, the guest widget WebSocket, the share-link
    WebSocket, and the two downstream adapters that hand another object
    to the engine's own ``register_connection`` -- go through the exact
    same ``ConnectionManager.register_connection`` call. This does not
    assert there are only six registration call sites (a closed-world claim
    a seventh audience could violate either way); it asserts that whatever
    registers through this mechanism is visible to
    ``has_connections_for_task`` and lets a subsequent ``handle_event``
    reach the host's event sink. See the comment on the
    ``builder_preview_websocket`` case above for the one path whose safety is
    not fully covered by this mechanical guard.
    """
    task_id = 90410
    unregistered_task_id = 90411
    local_manager = ConnectionManager()
    register(local_manager, task_id)
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    # Paired on purpose: a probe wired to something that always answers yes
    # would satisfy the first assertion alone.
    assert task_events.task_has_audience(task_id) is True
    assert task_events.task_has_audience(unregistered_task_id) is False

    broadcast_calls: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        broadcast_calls.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    handler = TaskEventTraceHandler(task_id)
    handler._task_description = None
    handler._task_description_loaded = True
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
        task_id=str(task_id),
        step_id="step-1",
        data={"model": "gpt-test"},
        timestamp=1_700_000_200.0,
    )

    await handler.handle_event(event)

    assert len(broadcast_calls) == 1


def _raise_audience_failure(task_id: int) -> bool:
    raise RuntimeError("audience registry unavailable")


@pytest.mark.asyncio
async def test_audience_check_failure_is_contained_in_the_handler(
    monkeypatch, caplog
) -> None:
    """A failing audience check never costs a frame, and never escapes.

    First segment: the registered probe raises. ``task_has_audience``
    contains that failure itself, answers that an audience is present, and
    warns once -- so the handler does its full work and the frame goes out.
    A probe that breaks may cost work; it may not cost content.

    Second segment (the one that actually matters): a real ``Tracer`` with a
    real ``TaskEventTraceHandler`` registered, firing a
    ``require_persisted=True`` event. ``Tracer.trace_event`` collects every
    handler's exception into ``handler_errors`` and re-raises once
    dispatch finishes when ``require_persisted`` is set
    (``core/agent/trace.py:1313-1324``) -- that collection logic lives in
    ``Tracer``, not in the handler, so a stub tracer would test nothing
    here, and a stub handler would replace the very object whose ``try:``
    placement this segment exists to pin. If the audience check were ever
    moved outside ``handle_event``'s existing ``try:``, this is the segment
    that would turn red: the exception would reach ``Tracer.trace_event``
    uncaught by the handler, join ``handler_errors``, and turn one required
    checkpoint persistence into a failed one.

    The two segments deliberately patch different names. The first patches
    the probe hook, because what it pins is that ``task_has_audience``
    swallows a probe failure. The second patches the name the handler module
    imported, because what it pins is where the call site sits: a probe
    failure is swallowed inside ``task_has_audience``, so with the probe
    patched this segment would stay green with the ``if`` on either side of
    the ``try``. Do not unify the two.
    """
    task_id = 90511

    # --- first segment: the failure is contained and costs no frame ---
    monkeypatch.setattr(task_events, "_warned_failing_probe", False)
    monkeypatch.setattr(task_events, "_task_audience_probe", _raise_audience_failure)

    published: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        published.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    handler = TaskEventTraceHandler(task_id)
    handler._task_description = None
    handler._task_description_loaded = True
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
        task_id=str(task_id),
        step_id="step-1",
        data={"model": "gpt-test"},
        timestamp=1_700_000_300.0,
    )

    with caplog.at_level(logging.WARNING, logger="xagent.web.services.task_events"):
        await handler.handle_event(event)

    assert len(published) == 1
    warnings = [
        record
        for record in caplog.records
        if record.name == "xagent.web.services.task_events"
        and record.levelname == "WARNING"
    ]
    assert len(warnings) == 1
    assert "assuming an audience is present" in warnings[0].getMessage()

    # --- second segment: the real Tracer, the hard requirement ---
    caplog.clear()
    monkeypatch.setattr(
        "xagent.web.services.task_event_trace_handler.task_has_audience",
        _raise_audience_failure,
    )

    tracer = Tracer()
    real_handler = TaskEventTraceHandler(task_id)
    tracer.add_handler(real_handler)

    checkpoint_data = {
        "checkpoint_type": CHECKPOINT_TYPE,
        "execution_id": "exec-required",
        "snapshot": {"label": "before_llm"},
    }

    # Not raising is the assertion: a required-persistence event whose only
    # handler is a TaskEventTraceHandler whose audience check is broken
    # must still complete without ``Tracer.trace_event`` raising.
    await tracer.trace_event(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task_id),
        data=checkpoint_data,
        require_persisted=True,
    )


@pytest.mark.asyncio
async def test_late_attach_replays_events_skipped_while_unattached(
    monkeypatch,
) -> None:
    """A client that attaches late does not lose content it is owed on the
    WebSocket paths, and a client that attaches while a frame is still
    being built receives that frame.

    First segment (WebSocket paths): while nothing is attached,
    ``handle_event`` publishes nothing for these rows -- the audience
    question answers no for them. In
    production ``DatabaseTraceHandler`` persists every one of them
    independently of what this handler does (both handlers are notified by
    the same ``Tracer.trace_event`` call); here the rows are seeded
    directly to isolate this segment from ``DatabaseTraceHandler``'s own
    behavior. ``send_historical_data_as_stream`` reads from
    ``trace_events``, not from anything the WebSocket handler buffered, so
    it still surfaces every non-checkpoint row to a client that attaches
    afterward. Checkpoint rows are excluded from replay by a different,
    pre-existing mechanism
    (``test_historical_replay_skips_checkpoint_rows_before_streaming``);
    this segment does not re-test that, only that ordinary rows survive.

    Second segment (a sink that attaches while the frame is being built):
    one audience is already attached, and a second sink registers itself
    from inside ``serialize_trace_data`` -- part-way through the very call
    that is building this frame -- and still receives it. What this pins is
    that ``handle_event`` must not snapshot the connection registry before
    doing its work. It asks the audience question once, for the emptiness
    answer only, and the registry is read again at delivery time:
    ``broadcast_to_task`` calls ``connections_for_task`` when it fans out.
    That is why this segment delivers through the real broadcast path
    instead of a stand-in sink -- the second read lives there, not in the
    handler. Storing that list in a local at the top of ``handle_event`` and
    broadcasting to the stored copy would drop this sink, and this segment
    turns red.

    The loss this change is allowed to introduce is a different one and it
    is structural, not a race: once the registry is empty on
    ``handle_event``'s first line, the call returns and nothing else in it
    runs, so no attach afterward recovers that particular frame for the
    newly attached sink. **There is no mechanical guard for that loss; it
    is recorded here in prose only.** It cannot be written as an assertion
    because the sink does not exist while the call runs -- nothing could
    have written to it, so asserting that it stayed empty would hold no
    matter what the production code did. The WebSocket paths recover such
    frames through the database replay the first segment covers. The v1 SSE
    path does not, and documents that gap itself:
    ``v1/_events_stream.py``'s module docstring (lines 16-24) tells v1 SSE
    clients that a step already in progress at attach time has no matching
    start on their stream and that they must reconcile with
    ``GET /v1/chat/tasks/{task_id}/steps``.
    """
    # ---- first segment ----
    SessionLocal, db, task = _create_trace_handler_test_task("i6-late-attach")
    task_id = int(task.id)
    user_id = int(task.user_id)
    db.close()

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)

    local_manager = ConnectionManager()
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    broadcast_calls: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        broadcast_calls.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    handler = TaskEventTraceHandler(task_id)
    handler._task_description = None
    handler._task_description_loaded = True

    events = [
        (
            "i6-llm-row",
            TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
            "step-a",
            {"model": "gpt-test"},
        ),
        (
            "i6-user-message-row",
            TraceEventType(TraceScope.TASK, TraceAction.START, TraceCategory.MESSAGE),
            None,
            {"message": "hi", "turn_id": "turn-i6"},
        ),
    ]
    for event_id, event_type, step_id, data in events:
        trace_event = TraceEvent(
            event_type,
            task_id=str(task_id),
            step_id=step_id,
            data=data,
            timestamp=1_700_000_400.0,
        )
        trace_event.id = event_id
        await handler.handle_event(trace_event)

    assert broadcast_calls == []

    base_time = datetime(2026, 6, 1, tzinfo=timezone.utc)
    db2 = SessionLocal()
    try:
        db2.add_all(
            [
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="i6-llm-row",
                    event_type="llm_call_start",
                    timestamp=base_time + timedelta(seconds=1),
                    data={"model": "gpt-test"},
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="i6-user-message-row",
                    event_type="user_message",
                    timestamp=base_time + timedelta(seconds=2),
                    data={"message": "hi", "turn_id": "turn-i6"},
                ),
            ]
        )
        db2.commit()
    finally:
        db2.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    streamed_event_ids = {
        event.get("event_id")
        for event in sent_events
        if event.get("type") == "trace_event"
    }
    assert "i6-llm-row" in streamed_event_ids
    assert "i6-user-message-row" in streamed_event_ids

    # ---- second segment ----
    sse_manager = ConnectionManager()
    monkeypatch.setattr(
        task_events, "_task_audience_probe", sse_manager.has_connections_for_task
    )
    monkeypatch.setattr(task_events, "_task_event_sink", sse_manager.broadcast_to_task)

    sse_handler = TaskEventTraceHandler(task_id)
    sse_handler._task_description = None
    sse_handler._task_description_loaded = True

    in_flight_event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
        task_id=str(task_id),
        step_id="step-b",
        data={"model": "gpt-test"},
        timestamp=1_700_000_500.0,
    )

    incumbent_sink = _MinimalDuckTypedSink()
    sse_manager.register_connection(incumbent_sink, task_id)

    late_sink = _MinimalDuckTypedSink()

    def register_late_sink_mid_serialization(payload):
        sse_manager.register_connection(late_sink, task_id)
        return serialize_trace_data(payload)

    monkeypatch.setattr(
        "xagent.web.services.task_event_trace_handler.serialize_trace_data",
        register_late_sink_mid_serialization,
    )

    await sse_handler.handle_event(in_flight_event)

    assert len(incumbent_sink.sent) == 1
    assert len(late_sink.sent) == 1


@pytest.mark.asyncio
async def test_skipped_frames_are_recoverable_through_the_real_persistence_path(
    monkeypatch,
) -> None:
    """The safety claim, end to end: no audience -> no frame, but a real
    ``DatabaseTraceHandler`` on the same dispatch still writes the row that
    a later attach replays.

    The sibling test seeds ``trace_events`` rows directly to isolate the
    replay reader. This one does not isolate anything: one ``Tracer`` with
    both real handlers, one real database session, and the same replay
    function the websocket endpoints call after registering a connection.

    Two details are load bearing. Patching ``database._SessionLocal`` reaches
    both readers: ``DatabaseTraceHandler``, which binds the real ``get_db`` at
    import time, and the replay reader, which imports it inside the function
    body. And the event is deliberately not a checkpoint: replay filters
    checkpoint rows out on two separate mechanisms, so a checkpoint would
    prove nothing about this change.
    """
    SessionLocal, db, task = _create_trace_handler_test_task("i6-e2e-replay")
    task_id = int(task.id)
    user_id = int(task.user_id)
    db.close()

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)

    empty_manager = ConnectionManager()
    monkeypatch.setattr(
        task_events, "_task_audience_probe", empty_manager.has_connections_for_task
    )

    published: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        published.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    tracer = Tracer()
    tracer.add_handler(DatabaseTraceHandler(task_id))
    tracer.add_handler(TaskEventTraceHandler(task_id))

    await tracer.trace_event(
        TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
        task_id=str(task_id),
        step_id="e2e-step",
        data={"model": "gpt-test"},
    )

    # Nothing was listening, so nothing was published.
    assert published == []

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    replayed = [
        event
        for event in sent_events
        if event.get("type") == "trace_event"
        and event.get("event_type") == "llm_call_start"
    ]
    assert len(replayed) == 1
    assert replayed[0]["data"]["model"] == "gpt-test"


@pytest.mark.asyncio
async def test_task_description_load_is_deferred_until_an_audience_attaches(
    monkeypatch,
) -> None:
    """The task description read happens at most once, and only once an
    audience exists.

    First three segments pin the lazy-cache contract untouched by this
    change: no read while unattached, exactly one read on the first event
    after attach, and no re-read on later events. The fourth segment is the
    failure tier -- see ``failing_sync_load_task_description``'s docstring
    below for why a NULL description lands in the same bucket as a real
    database failure.
    """
    task_id = 90712
    local_manager = ConnectionManager()
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    load_calls: list[int] = []

    def counting_stub(self: TaskEventTraceHandler) -> None:
        load_calls.append(1)
        self._task_description = "Loaded description"

    monkeypatch.setattr(
        TaskEventTraceHandler, "_sync_load_task_description", counting_stub
    )

    broadcast_calls: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        broadcast_calls.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    handler = TaskEventTraceHandler(task_id)

    def make_event(step_id: str) -> TraceEvent:
        return TraceEvent(
            TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
            task_id=str(task_id),
            step_id=step_id,
            data={"model": "gpt-test"},
            timestamp=1_700_000_600.0,
        )

    # Segment 1: no audience, 3 events, zero reads.
    for i in range(3):
        await handler.handle_event(make_event(f"pre-{i}"))
    assert load_calls == []
    assert broadcast_calls == []

    # Segment 2: register an audience, one event, exactly one read.
    local_manager.register_connection(object(), task_id)
    await handler.handle_event(make_event("first-after-attach"))
    assert len(load_calls) == 1
    assert len(broadcast_calls) == 1

    # Segment 3: two more events, no further reads.
    await handler.handle_event(make_event("second-after-attach"))
    await handler.handle_event(make_event("third-after-attach"))
    assert len(load_calls) == 1
    assert len(broadcast_calls) == 3

    # Segment 4: failure tier -- a fresh handler so the cache starts empty.
    failing_handler = TaskEventTraceHandler(task_id)
    fail_calls: list[int] = []

    def failing_sync_load_task_description(self: TaskEventTraceHandler) -> None:
        """Failure tier: one attempt, no retry, and NULL descriptions land here too.

        ``_load_task_description`` sets ``_task_description_loaded`` outside its
        ``try`` (``task_event_trace_handler.py:478``), so a failed read is still a
        read and never retried. Two inputs reach this tier, not one: a real
        database failure, and a task row whose ``description`` is NULL -- the info
        log at ``task_event_trace_handler.py:496-498`` slices ``task.description``
        unconditionally, so ``None[:50]`` raises ``TypeError`` and the outer
        ``except Exception`` swallows it. ``Task.description`` is
        ``Column(Text)`` (``web/models/task.py:280``), i.e. nullable. This change
        moves that single attempt to the moment an audience attaches, so the
        warning this path writes now lands at attach time instead of at task
        start.
        """
        fail_calls.append(1)
        raise RuntimeError("simulated task description read failure")

    monkeypatch.setattr(
        TaskEventTraceHandler,
        "_sync_load_task_description",
        failing_sync_load_task_description,
    )

    await failing_handler.handle_event(make_event("failure-1"))
    # (a) handle_event did not raise -- reaching this line proves that.
    assert len(fail_calls) == 1
    assert len(broadcast_calls) == 4
    message, _ = broadcast_calls[-1]
    # (b) the frame that did go out carries no task_description field.
    assert "task_description" not in message["data"]

    await failing_handler.handle_event(make_event("failure-2"))
    # (c) the load is not retried on a later event.
    assert len(fail_calls) == 1


@pytest.mark.asyncio
@pytest.mark.parametrize("audience_present", [False, True])
async def test_delegated_child_traces_follow_the_parent_task_audience(
    audience_present, monkeypatch
) -> None:
    """A delegated child agent's traces are gated by the *parent* task's
    audience, not the child's own.

    ``_DelegatedAgentTaskEventTraceHandler`` is constructed with the parent
    task id and wraps a ``TaskEventTraceHandler`` bound to that same parent
    id (``agent_tool.py:97``); the child event's own ``task_id`` field never
    reaches the audience check or the published frame's ``task_id``.
    """
    parent_task_id = 90813
    child_task_id = 90814
    local_manager = ConnectionManager()
    monkeypatch.setattr(
        task_events, "_task_audience_probe", local_manager.has_connections_for_task
    )

    if audience_present:
        local_manager.register_connection(object(), parent_task_id)

    broadcast_calls: list[object] = []

    async def fake_broadcast(message, task_id_arg):
        broadcast_calls.append((message, task_id_arg))

    monkeypatch.setattr(task_events, "_task_event_sink", fake_broadcast)

    delegated_handler = _DelegatedAgentTaskEventTraceHandler(
        task_id=parent_task_id, metadata={"source": "xagent-agent-tool-child"}
    )
    delegated_handler._handler._task_description = None
    delegated_handler._handler._task_description_loaded = True
    event = TraceEvent(
        TraceEventType(TraceScope.ACTION, TraceAction.START, TraceCategory.LLM),
        task_id=str(child_task_id),
        step_id="child-step-1",
        data={"model": "gpt-test"},
        timestamp=1_700_000_700.0,
    )

    await delegated_handler.handle_event(event)

    if not audience_present:
        assert broadcast_calls == []
        return

    assert len(broadcast_calls) == 1
    message, broadcast_task_id_arg = broadcast_calls[0]
    assert broadcast_task_id_arg == parent_task_id
    assert message["task_id"] == parent_task_id


def test_historical_replay_duplicate_turn_helper_allows_distinct_turns() -> None:
    seen: set[str] = set()

    assert not _is_duplicate_user_message_turn(
        "user_message", {"message": "Repeat", "turn_id": "turn-1"}, seen
    )
    assert _is_duplicate_user_message_turn(
        "user_message", {"message": "Repeat", "turn_id": "turn-1"}, seen
    )
    assert not _is_duplicate_user_message_turn(
        "user_message", {"message": "Repeat", "turn_id": "turn-2"}, seen
    )


@pytest.mark.asyncio
async def test_historical_replay_skips_audit_only_trace_events(monkeypatch) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("audit-history")
    try:
        task_id = int(task.id)
        user_id = int(task.user_id)
        base_time = datetime(2026, 5, 22, tzinfo=timezone.utc)
        db.add_all(
            [
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="audit-workforce",
                    event_type="task_update_general",
                    timestamp=base_time + timedelta(seconds=1),
                    data={
                        "__audit_only__": True,
                        "event_type": "workforce_delegation_start",
                        "worker_task_id": "agent_123_abcd1234",
                    },
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="visible-tool",
                    event_type="tool_execution_start",
                    timestamp=base_time + timedelta(seconds=2),
                    data={"tool_name": "call_agent_worker"},
                ),
            ]
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    trace_event_ids = [
        event.get("event_id")
        for event in sent_events
        if event.get("type") == "trace_event"
    ]

    assert "audit-workforce" not in trace_event_ids
    assert "visible-tool" in trace_event_ids


@pytest.mark.asyncio
async def test_historical_replay_promotes_workforce_delegation_summary(
    monkeypatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("delegation-history")
    try:
        task_id = int(task.id)
        user_id = int(task.user_id)
        base_time = datetime(2026, 5, 22, tzinfo=timezone.utc)
        db.add_all(
            [
                WorkforceRun(
                    workforce_id=999,
                    task_id=task_id,
                    user_id=user_id,
                    status="completed",
                    snapshot={},
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="delegation-end",
                    event_type="task_update_general",
                    timestamp=base_time + timedelta(seconds=1),
                    data={
                        "event_type": "workforce_delegation_end",
                        "status": "end",
                        "worker_task_id": "agent_123_abcd1234",
                        "worker_alias": "Writer",
                        "output": "draft complete",
                        "messages": [{"role": "user", "content": "raw prompt"}],
                    },
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    build_id="agent_123_abcd1234",
                    event_id="delegated-internal",
                    event_type="llm_call_start",
                    timestamp=base_time,
                    data={
                        "source": "xagent-agent-tool-child",
                        "worker_task_id": "agent_123_abcd1234",
                    },
                ),
            ]
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    delegation_event = next(
        event for event in sent_events if event.get("event_id") == "delegation-end"
    )
    assert delegation_event["event_type"] == "workforce_delegation_end"
    assert delegation_event["data"]["worker_alias"] == "Writer"
    assert delegation_event["data"]["worker_task_id"] == "agent_123_abcd1234"
    assert delegation_event["data"]["output"] == "draft complete"
    assert "messages" not in delegation_event["data"]
    assert "event_type" not in delegation_event["data"]
    assert not any(
        event.get("event_id") == "delegated-internal" for event in sent_events
    )


@pytest.mark.asyncio
async def test_historical_replay_includes_child_trace_for_non_workforce_task(
    monkeypatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task(
        "non-workforce-child-history"
    )
    try:
        task_id = int(task.id)
        user_id = int(task.user_id)
        base_time = datetime(2026, 5, 22, tzinfo=timezone.utc)
        db.add_all(
            [
                DatabaseTraceEvent(
                    task_id=task_id,
                    build_id="agent_123_abcd1234",
                    event_id="delegated-progress",
                    event_type="agent_progress",
                    timestamp=base_time,
                    data={
                        "source": "xagent-agent-tool-child",
                        "worker_task_id": "agent_123_abcd1234",
                        "message": "Child is working",
                    },
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    build_id="builder-session",
                    event_id="unrelated-build",
                    event_type="agent_progress",
                    timestamp=base_time + timedelta(seconds=1),
                    data={"source": "builder", "message": "Internal build trace"},
                ),
            ]
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    delegated_event = next(
        event for event in sent_events if event.get("event_id") == "delegated-progress"
    )
    assert delegated_event["event_type"] == "agent_progress"
    assert delegated_event["data"]["source"] == "xagent-agent-tool-child"
    assert not any(event.get("event_id") == "unrelated-build" for event in sent_events)


@pytest.mark.asyncio
async def test_historical_replay_skips_checkpoint_rows_before_streaming(
    monkeypatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("checkpoint-history")
    try:
        task_id = int(task.id)
        user_id = int(task.user_id)
        base_time = datetime(2026, 5, 22, tzinfo=timezone.utc)
        db.add_all(
            [
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="checkpoint-row",
                    event_type=str(CHECKPOINT_EVENT_TYPE),
                    timestamp=base_time + timedelta(seconds=1),
                    data={
                        "checkpoint_type": CHECKPOINT_TYPE,
                        "execution_id": str(task_id),
                        "snapshot": {"context": {"messages": ["large"]}},
                    },
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="llm-row",
                    event_type="llm_call_start",
                    timestamp=base_time + timedelta(seconds=2),
                    data={"model_name": "test-model"},
                ),
            ]
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    streamed_event_ids = {
        event.get("event_id")
        for event in sent_events
        if event.get("type") == "trace_event"
    }
    assert "checkpoint-row" not in streamed_event_ids
    assert "llm-row" in streamed_event_ids


@pytest.mark.asyncio
async def test_historical_replay_marks_assistant_chat_history_for_chat_display(
    monkeypatch,
) -> None:
    SessionLocal, db, task = _create_trace_handler_test_task("chat-history-display")
    try:
        task_id = int(task.id)
        user_id = int(task.user_id)
        db.add(
            TaskChatMessage(
                task_id=task_id,
                user_id=user_id,
                role="assistant",
                content="Final answer",
                message_type="assistant",
                created_at=datetime(2026, 5, 27, tzinfo=timezone.utc),
            )
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    assistant_events = [
        event
        for event in sent_events
        if event.get("type") == "trace_event"
        and event.get("event_type") == "agent_message"
        and event.get("data", {}).get("message") == "Final answer"
    ]
    assert len(assistant_events) == 1
    assistant_data = assistant_events[0]["data"]
    assert assistant_data["role"] == "assistant"
    assert assistant_data["expect_response"] is False
    assert assistant_data["source"] == "chat_history"
    assert assistant_data["display"] == "chat"


@pytest.mark.asyncio
async def test_historical_replay_orders_equal_timestamps_by_id(
    monkeypatch,
) -> None:
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()
    try:
        user = User(username="tester", password_hash="hashed_password", is_admin=False)
        db.add(user)
        db.commit()
        db.refresh(user)
        task = Task(
            user_id=int(user.id),
            title="Chat task",
            description="Chat task",
            status=TaskStatus.COMPLETED,
        )
        db.add(task)
        db.commit()
        db.refresh(task)

        task_id = int(task.id)
        user_id = int(user.id)
        timestamp = datetime(2026, 5, 22, tzinfo=timezone.utc)
        db.add_all(
            [
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="first-row",
                    event_type="llm_call_start",
                    timestamp=timestamp,
                    data={"model_name": "first-model"},
                ),
                DatabaseTraceEvent(
                    task_id=task_id,
                    event_id="second-row",
                    event_type="llm_call_end",
                    timestamp=timestamp,
                    data={"response": "done"},
                ),
            ]
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    streamed_event_ids = [
        event.get("event_id")
        for event in sent_events
        if event.get("type") == "trace_event"
        and event.get("event_id") in {"first-row", "second-row"}
    ]
    assert streamed_event_ids == ["first-row", "second-row"]


@pytest.mark.asyncio
async def test_historical_replay_uses_turn_id_before_legacy_content_dedupe(
    monkeypatch,
) -> None:
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()
    try:
        user = User(username="tester", password_hash="hashed_password", is_admin=False)
        db.add(user)
        db.commit()
        db.refresh(user)
        task = Task(
            user_id=int(user.id),
            title="Chat task",
            description="Chat task",
            status=TaskStatus.COMPLETED,
        )
        db.add(task)
        db.commit()
        db.refresh(task)

        task_id = int(task.id)
        user_id = int(user.id)
        base_time = datetime(2026, 5, 22, tzinfo=timezone.utc)
        db.add(
            DatabaseTraceEvent(
                task_id=task_id,
                event_id="trace-turn-a",
                event_type="user_message",
                timestamp=base_time + timedelta(seconds=1),
                data={"message": "Repeat", "turn_id": "turn-A"},
            )
        )
        db.add_all(
            [
                TaskChatMessage(
                    task_id=task_id,
                    user_id=user_id,
                    role="user",
                    content="Repeat",
                    message_type="user_message",
                    turn_id="turn-A",
                    created_at=base_time + timedelta(seconds=2),
                ),
                TaskChatMessage(
                    task_id=task_id,
                    user_id=user_id,
                    role="user",
                    content="Repeat",
                    message_type="user_message",
                    turn_id="turn-B",
                    created_at=base_time + timedelta(seconds=3),
                ),
            ]
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    user_message_events = [
        event
        for event in sent_events
        if event.get("type") == "trace_event"
        and event.get("event_type") == "user_message"
    ]

    assert [
        (event["data"].get("message"), event["data"].get("turn_id"))
        for event in user_message_events
    ] == [("Repeat", "turn-A"), ("Repeat", "turn-B")]


@pytest.mark.asyncio
async def test_historical_replay_dedupes_file_only_turns_by_turn_id(
    monkeypatch,
) -> None:
    engine = _shared_memory_sqlite_engine()
    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
    Base.metadata.create_all(bind=engine)
    db = SessionLocal()
    try:
        user = User(username="tester", password_hash="hashed_password", is_admin=False)
        db.add(user)
        db.commit()
        db.refresh(user)
        task = Task(
            user_id=int(user.id),
            title="Chat task",
            description="Chat task",
            status=TaskStatus.COMPLETED,
        )
        db.add(task)
        db.commit()
        db.refresh(task)

        task_id = int(task.id)
        user_id = int(user.id)
        base_time = datetime(2026, 5, 22, tzinfo=timezone.utc)
        attachments = [{"file_id": "fid-only", "name": "only.pdf"}]
        db.add(
            DatabaseTraceEvent(
                task_id=task_id,
                event_id="trace-file-only",
                event_type="user_message",
                timestamp=base_time + timedelta(seconds=1),
                data={"message": "", "turn_id": "turn-file", "files": attachments},
            )
        )
        db.add(
            TaskChatMessage(
                task_id=task_id,
                user_id=user_id,
                role="user",
                content="",
                message_type="user_message",
                turn_id="turn-file",
                attachments=attachments,
                created_at=base_time + timedelta(seconds=2),
            )
        )
        db.commit()
    finally:
        db.close()

    sent_events: list[dict] = []

    async def send_personal_message(event: dict, websocket: object) -> None:
        sent_events.append(event)

    monkeypatch.setattr("xagent.web.models.database._SessionLocal", SessionLocal)
    monkeypatch.setattr("xagent.web.api.websocket.cache_get", lambda *args: None)
    monkeypatch.setattr(
        "xagent.web.api.websocket.cache_set", lambda *args, **kwargs: None
    )
    monkeypatch.setattr(
        "xagent.web.api.websocket.manager.send_personal_message",
        send_personal_message,
    )

    await send_historical_data_as_stream(
        websocket=object(),
        task_id=task_id,
        user=SimpleNamespace(id=user_id, is_admin=False),
    )

    user_message_events = [
        event
        for event in sent_events
        if event.get("type") == "trace_event"
        and event.get("event_type") == "user_message"
    ]

    assert [
        (event["data"].get("turn_id"), event["data"].get("files"))
        for event in user_message_events
    ] == [("turn-file", attachments)]


_BROKEN_ANCHOR_LABEL = "anchored-broken"


def _unidentified_checkpoint_row(
    *,
    task_id: int,
    label: str,
    timestamp: datetime,
    run_id: str,
) -> DatabaseTraceEvent:
    """A checkpoint row carrying no execution identity at all.

    The pointer anchors it (checkpoint_execution_id() returns "" and the
    anchor's identity conjunct short-circuits), but the legacy scan excludes
    it: its coalesce(nullif(...)) predicate is NULL and NULL = :execution_id
    is never true. This is the shape that makes the anchor's payload verdict
    load-bearing -- without it seeding the scan's accumulators, an unreadable
    checkpoint would come back as "no checkpoint".
    """
    return DatabaseTraceEvent(
        task_id=task_id,
        build_id=None,
        event_id=f"unidentified-{label}",
        event_type="system_update_general",
        timestamp=timestamp,
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            TASK_RUN_ID_TRACE_FIELD: run_id,
            "snapshot": {"label": label},
        },
    )


def _decode_failing_on(label: str, exception: Exception):
    """Decode stub that fails for exactly one row, identified by its label.

    Injected rather than built from a genuinely broken blob because these
    tests pin the anchor's *classification* of a decode failure, and the
    generic (non-CheckpointMessageDecodeError) branch has no data-level
    trigger at all.
    """

    def fake_decode(db, *, task_id, data, strict=False, verify_blob_hashes=True):
        snapshot = data.get("snapshot") if isinstance(data, dict) else None
        if isinstance(snapshot, dict) and snapshot.get("label") == label:
            raise exception
        return data

    return fake_decode


def _bind_checkpoint_read_session(
    monkeypatch: pytest.MonkeyPatch,
    SessionLocal,
) -> None:
    def get_test_db() -> Iterator[Session]:
        session = SessionLocal()
        try:
            yield session
        finally:
            session.close()

    monkeypatch.setattr("xagent.web.services.trace_handlers.get_db", get_test_db)


def test_database_trace_handler_load_pk_anchor_undecodable_payload_falls_back_to_older_row(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The retention guarantee _prune_checkpoint_history documents: older
    rows are kept so an unreadable latest can fall back. The anchor's
    identity stays authoritative, but its payload does not."""
    from xagent.web.services.trace_message_storage import CheckpointMessageDecodeError

    SessionLocal, db, task = _create_trace_handler_test_task("anchor-undecodable-older")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="older",
            execution_id="shared-execution",
            label="older-readable",
            timestamp=now,
            run_id="run-a",
        )
    )
    broken = _checkpoint_trace_row(
        task_id=task_id,
        event_id="broken",
        execution_id="shared-execution",
        label=_BROKEN_ANCHOR_LABEL,
        timestamp=now + timedelta(seconds=1),
        run_id="run-a",
    )
    db.add(broken)
    db.commit()
    db.refresh(broken)
    task.last_checkpoint_trace_event_id = broken.id
    db.commit()

    _bind_checkpoint_read_session(monkeypatch, SessionLocal)
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.decode_trace_event_data",
        _decode_failing_on(
            _BROKEN_ANCHOR_LABEL, CheckpointMessageDecodeError("blob is gone")
        ),
    )

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                "shared-execution"
            ) == {"label": "older-readable"}
    finally:
        db.close()


def test_database_trace_handler_load_pk_anchor_undecodable_payload_without_older_row_is_corrupt(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The anchored row is the only checkpoint and the scan cannot even see
    it (no execution identity). The deferral must still resolve to corrupt,
    never to ``None`` -- an unreadable checkpoint is not an absent one."""
    from xagent.web.services.trace_message_storage import CheckpointMessageDecodeError

    SessionLocal, db, task = _create_trace_handler_test_task("anchor-undecodable-only")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    broken = _unidentified_checkpoint_row(
        task_id=task_id,
        label=_BROKEN_ANCHOR_LABEL,
        timestamp=datetime.now(timezone.utc),
        run_id="run-a",
    )
    db.add(broken)
    db.commit()
    db.refresh(broken)
    task.last_checkpoint_trace_event_id = broken.id
    db.commit()

    _bind_checkpoint_read_session(monkeypatch, SessionLocal)
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.decode_trace_event_data",
        _decode_failing_on(
            _BROKEN_ANCHOR_LABEL, CheckpointMessageDecodeError("blob is gone")
        ),
    )

    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointCorruptError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
    finally:
        db.close()


def test_database_trace_handler_load_pk_anchor_generic_decode_failure_is_unavailable(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A transient (non-decode-class) failure on the anchored row is
    retryable, not terminal: unavailable, so a2a maps it to 503 rather than
    the terminal 400 a corrupt verdict earns."""
    from xagent.web.services.ops_signals import (
        CHECKPOINT_DECODE_FALLBACK,
        CHECKPOINT_LOAD_UNAVAILABLE,
        active_degradations,
        clear_degradation,
    )

    SessionLocal, db, task = _create_trace_handler_test_task("anchor-generic-only")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    broken = _unidentified_checkpoint_row(
        task_id=task_id,
        label=_BROKEN_ANCHOR_LABEL,
        timestamp=datetime.now(timezone.utc),
        run_id="run-a",
    )
    db.add(broken)
    db.commit()
    db.refresh(broken)
    task.last_checkpoint_trace_event_id = broken.id
    db.commit()

    _bind_checkpoint_read_session(monkeypatch, SessionLocal)
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.decode_trace_event_data",
        _decode_failing_on(_BROKEN_ANCHOR_LABEL, RuntimeError("blob prefetch failed")),
    )

    clear_degradation(CHECKPOINT_DECODE_FALLBACK)
    clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointUnavailableError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
        active = active_degradations()
        assert CHECKPOINT_DECODE_FALLBACK in active
        assert CHECKPOINT_LOAD_UNAVAILABLE in active
    finally:
        clear_degradation(CHECKPOINT_DECODE_FALLBACK)
        clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
        db.close()


@pytest.mark.parametrize("decodes_a_row", [True, False])
def test_database_trace_handler_decode_fallback_clears_after_a_successful_decode(
    monkeypatch: pytest.MonkeyPatch,
    decodes_a_row: bool,
) -> None:
    """The decode-fallback signal is process-wide and its clear is evidence
    that a decode worked, so an anchored read that resolves its pointer and
    decodes the row has to retire it: that read returns before the legacy
    scan, and the scan holds the only other clear. Without this the signal
    stays on /health for the life of the process as soon as the anchor
    starts succeeding, which is the normal steady state.

    The second case pins the clear to the decode rather than to the attempt.
    A read with nothing to decode -- no pointer, no rows -- proves nothing
    about the decode layer and must leave a set signal alone; a clear placed
    at the top of the anchored read next to the dangling clear would retire
    it there and report a false all-clear.
    """
    from xagent.web.services.ops_signals import (
        CHECKPOINT_DECODE_FALLBACK,
        active_degradations,
        clear_degradation,
        register_degradation,
    )

    SessionLocal, db, task = _create_trace_handler_test_task(
        f"decode-fallback-clear-{decodes_a_row}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    expected: Dict[str, Any] | None = None
    if decodes_a_row:
        row = _checkpoint_trace_row(
            task_id=task_id,
            event_id="anchored",
            execution_id="shared-execution",
            label="anchored",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
        db.add(row)
        db.commit()
        db.refresh(row)
        task.last_checkpoint_trace_event_id = row.id
        db.commit()
        expected = {"label": "anchored"}

    _bind_checkpoint_read_session(monkeypatch, SessionLocal)

    register_degradation(
        CHECKPOINT_DECODE_FALLBACK, "left over from an earlier fallback"
    )
    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert (
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
                == expected
            )
        if decodes_a_row:
            assert CHECKPOINT_DECODE_FALLBACK not in active_degradations()
        else:
            assert CHECKPOINT_DECODE_FALLBACK in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_DECODE_FALLBACK)
        db.close()


@pytest.mark.parametrize("failing_call", ["pointer_query", "row_fetch"])
def test_database_trace_handler_load_pk_anchor_database_failure_is_unavailable(
    monkeypatch: pytest.MonkeyPatch,
    failing_call: str,
) -> None:
    """_sync_load_latest_checkpoint promises every database touch is
    translated into a CheckpointReadError subclass. Both of the anchor's own
    database calls have to honour that, or a transient connection failure
    escapes as a raw driver exception no caller catches."""
    from xagent.web.services.ops_signals import (
        CHECKPOINT_LOAD_UNAVAILABLE,
        active_degradations,
        clear_degradation,
    )

    SessionLocal, db, task = _create_trace_handler_test_task(
        f"anchor-db-{failing_call}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    row = _checkpoint_trace_row(
        task_id=task_id,
        event_id="anchored",
        execution_id="shared-execution",
        label="anchored",
        timestamp=datetime.now(timezone.utc),
        run_id="run-a",
    )
    db.add(row)
    db.commit()
    db.refresh(row)
    task.last_checkpoint_trace_event_id = row.id
    db.commit()

    _bind_checkpoint_read_session(monkeypatch, SessionLocal)

    boom = OperationalError("SELECT", {}, Exception("connection reset"))
    if failing_call == "pointer_query":
        original_one_or_none = Query.one_or_none

        def failing_one_or_none(self: Query):  # type: ignore[no-untyped-def]
            descriptions = self.column_descriptions
            if (
                descriptions
                and descriptions[0]["name"] == "last_checkpoint_trace_event_id"
            ):
                raise boom
            return original_one_or_none(self)

        monkeypatch.setattr(Query, "one_or_none", failing_one_or_none)
    else:

        def failing_get(self: Session, *args: object, **kwargs: object):  # type: ignore[no-untyped-def]
            raise boom

        monkeypatch.setattr(Session, "get", failing_get)

    clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            with pytest.raises(CheckpointUnavailableError):
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
        assert CHECKPOINT_LOAD_UNAVAILABLE in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_LOAD_UNAVAILABLE)
        db.close()


@pytest.mark.parametrize("anchor_set", [True, False])
def test_database_trace_handler_anchored_read_clears_the_dangling_signal(
    monkeypatch: pytest.MonkeyPatch,
    anchor_set: bool,
) -> None:
    """The clear must sit before the pointer lookup, not after a pointer
    resolves to a row.

    ops_signals state is process-wide, so a signal that only cleared once
    some pointer resolved would stay set forever in a process where no task
    carries an anchor. The anchorless case is the one that pins the
    placement: move the clear below the pointer lookup and the anchored case
    still passes while that one fails.
    """
    from xagent.web.services.ops_signals import (
        CHECKPOINT_PK_ANCHOR_DANGLING,
        active_degradations,
        clear_degradation,
        register_degradation,
    )

    SessionLocal, db, task = _create_trace_handler_test_task(
        f"anchor-clears-dangling-{anchor_set}"
    )
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    expected: Dict[str, Any] | None = None
    if anchor_set:
        row = _checkpoint_trace_row(
            task_id=task_id,
            event_id="anchored",
            execution_id="shared-execution",
            label="anchored",
            timestamp=datetime.now(timezone.utc),
            run_id="run-a",
        )
        db.add(row)
        db.commit()
        db.refresh(row)
        task.last_checkpoint_trace_event_id = row.id
        db.commit()
        expected = {"label": "anchored"}
    assert (task.last_checkpoint_trace_event_id is not None) is anchor_set

    _bind_checkpoint_read_session(monkeypatch, SessionLocal)

    register_degradation(
        CHECKPOINT_PK_ANCHOR_DANGLING, "left over from another task's read"
    )
    try:
        with bind_task_lease_context(
            TaskLease(task_id, "runner-a", "run-a", attempt_id="test-attempt")
        ):
            assert (
                DatabaseTraceHandler(task_id)._sync_load_latest_checkpoint(
                    "shared-execution"
                )
                == expected
            )
        assert CHECKPOINT_PK_ANCHOR_DANGLING not in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_PK_ANCHOR_DANGLING)
        db.close()


def test_database_trace_handler_prune_retains_exactly_the_limit_when_the_anchor_is_in_range(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The steady state: the pointer names the row just written, which ranks
    inside the retention window. Ranking before protecting makes the
    documented retention count exact -- excluding the anchor from the
    candidate set instead shifts every remaining row's OFFSET rank by one
    and keeps limit + 1."""
    _, db, task = _create_trace_handler_test_task("anchor-in-range-prune")
    task_id = int(task.id)
    now = datetime.now(timezone.utc)
    rows = [
        _checkpoint_trace_row(
            task_id=task_id,
            event_id=event_id,
            execution_id="shared-execution",
            label=event_id,
            timestamp=now + timedelta(seconds=offset),
            run_id="run-a",
        )
        for offset, event_id in enumerate(["oldest", "middle", "newest"])
    ]
    db.add_all(rows)
    db.commit()
    db.refresh(rows[-1])
    task.last_checkpoint_trace_event_id = rows[-1].id
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 1,
    )

    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {
                "checkpoint_type": CHECKPOINT_TYPE,
                "execution_id": "shared-execution",
                TASK_RUN_ID_TRACE_FIELD: "run-a",
            },
        )

        assert {
            row.event_id
            for row in db.query(DatabaseTraceEvent).filter_by(task_id=task_id).all()
        } == {"newest"}
    finally:
        db.close()


def test_database_trace_handler_prune_with_nothing_stale_clears_the_failure_signal(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A healthy steady state prunes nothing. If the clear sat after the
    delete, that state could never retire a previously-latched signal, and
    it would show up as permanent /health noise."""
    from xagent.web.services.ops_signals import (
        CHECKPOINT_PRUNE_FAILED,
        active_degradations,
        clear_degradation,
        register_degradation,
    )

    _, db, task = _create_trace_handler_test_task("prune-nothing-stale")
    task_id = int(task.id)
    db.add(
        _checkpoint_trace_row(
            task_id=task_id,
            event_id="only",
            execution_id="shared-execution",
            label="only",
            timestamp=datetime.now(timezone.utc),
        )
    )
    db.commit()
    monkeypatch.setattr(
        "xagent.web.services.trace_handlers.get_checkpoint_history_limit",
        lambda: 5,
    )

    register_degradation(CHECKPOINT_PRUNE_FAILED, "left over from an earlier race")
    try:
        DatabaseTraceHandler(task_id)._prune_checkpoint_history(
            db,
            {"checkpoint_type": CHECKPOINT_TYPE, "execution_id": "shared-execution"},
        )
        assert CHECKPOINT_PRUNE_FAILED not in active_degradations()
    finally:
        clear_degradation(CHECKPOINT_PRUNE_FAILED)
        db.close()


def test_database_trace_handler_flush_without_primary_key_refuses_to_write_the_anchor(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The guard that replaced a bare assert. A no-op flush (rather than a
    raising one) is what exercises it: a raising flush would land in the
    generic handler instead and prove nothing about the guard."""
    _, db, task = _create_trace_handler_test_task("flush-without-pk")
    task.status = TaskStatus.RUNNING
    task.runner_id = "runner-a"
    task.lease_attempt_id = "test-attempt"
    task.run_id = "run-a"
    db.commit()
    task_id = int(task.id)
    lease = TaskLease(
        task_id=task_id, runner_id="runner-a", run_id="run-a", attempt_id="test-attempt"
    )
    event = TraceEvent(
        CHECKPOINT_EVENT_TYPE,
        task_id=str(task_id),
        data={
            "checkpoint_type": CHECKPOINT_TYPE,
            "execution_id": str(task_id),
            "snapshot": {"label": "unflushed"},
        },
    )

    monkeypatch.setattr(Session, "flush", lambda self, *a, **k: None)

    try:
        with (
            bind_task_lease_context(lease),
            pytest.raises(
                RuntimeError, match="refusing to write a NULL checkpoint anchor"
            ),
        ):
            DatabaseTraceHandler(task_id)._save_trace_event(db, event)

        db.expire_all()
        persisted = db.get(Task, task_id)
        assert persisted.last_checkpoint_trace_event_id is None
        assert db.query(DatabaseTraceEvent).filter_by(task_id=task_id).count() == 0
    finally:
        db.close()