Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

45.0 KB
Raw Blame
"""Unknown-outcome signals for execution events.

Every case where the system does not know whether something durable happened
produces one structured WARNING/INFO line that locates the task and one
bounded-attribute counter increment: a tool whose external effect is unknown,
a turn or command acceptance whose COMMIT outcome is unknown, a finalize whose
lost acknowledgement could not be resolved, a settlement deferred to lease
recovery. Each test drives the real code path; metrics are asserted as
exported.
"""

from __future__ import annotations

import logging
from datetime import timedelta
from typing import Any
from unittest.mock import MagicMock

import pytest
import sqlalchemy as sa
from sqlalchemy.exc import IntegrityError, InterfaceError, OperationalError
from sqlalchemy.orm import Session

from tests.core.agent.test_react import FakeTool
from tests.web.api.test_websocket_owner_actor import _task as _owner_task
from tests.web.api.test_websocket_owner_actor import _user as _owner_user
from tests.web.api.test_websocket_owner_actor import db_session as db_session_fixture
from tests.web.services.test_execution_event_persistence_settlement import (
    _clear_bg_manager as _clear_bg_manager_fixture,
)
from tests.web.services.test_execution_event_persistence_settlement import (
    _fail_once,
    _react_execution,
    _run_scheduled_turn,
)
from tests.web.services.test_execution_event_recovery import tracer_for
from tests.web.services.test_execution_event_signals import (
    exported,
    fields,
    key,
)
from tests.web.services.test_execution_event_signals import (
    metric_reader as metric_reader_fixture,
)
from tests.web.services.test_persistence_failure_pause import (
    NO_OUTPUTS,
    _finalize,
)
from tests.web.services.test_persistence_failure_pause import (
    _late_bound_sessions as _late_bound_sessions_fixture,
)
from tests.web.services.test_persistence_failure_pause import (
    _resume_raising,
    _settle,
    _start_run,
)
from tests.web.services.test_task_execution_event_writer import (
    canonical as canonical_fixture,
)
from tests.web.services.test_task_execution_event_writer import engine as engine_fixture
from tests.web.services.test_task_execution_event_writer import (
    task_id as task_id_fixture,
)
from xagent.core.agent import PatternRuntime
from xagent.core.agent.checkpoint import (
    CheckpointUnavailableError,
    TraceCheckpointStore,
)
from xagent.web.models import database as database_module
from xagent.web.models.task import Task, TaskStatus
from xagent.web.models.task_execution_event import TaskExecutionEvent
from xagent.web.services import (
    execution_event_signals,
    task_command_execution,
    task_execution,
)
from xagent.web.services import task_execution_event_recovery as event_recovery
from xagent.web.services import (
    task_lease_recovery,
)
from xagent.web.services import task_orchestrator as orchestrator
from xagent.web.services.db_write_retry import DbWriteRetryPolicy
from xagent.web.services.execution_event_signals import report_unknown_outcome
from xagent.web.services.task_auto_recovery import InterruptionSettlementDeferred
from xagent.web.services.task_execution import (
    _ChatMessageWitness,
    _commit_finalization_or_reconcile,
    _finalization_witness_committed,
    _finalize_resumed_task,
)
from xagent.web.services.task_execution_event_writer import FactWitness
from xagent.web.services.task_lease_recovery import (
    recover_expired_task_leases_batch_isolated,
    recover_task_lease_candidate_isolated,
)
from xagent.web.services.task_lease_service import (
    TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR,
    TaskLease,
    acquire_task_lease,
    bind_task_lease_context,
    get_expired_task_lease_candidates,
    utc_now,
)
from xagent.web.services.task_orchestrator import (
    TaskTurnCommitOutcomeUnknown,
    TaskTurnPayload,
    TurnKind,
    _begin_turn_atomic_sync,
    _ClaimedTurn,
    commit_claimed_turn_or_reconcile,
)

canonical = canonical_fixture
engine = engine_fixture
task_id = task_id_fixture
metric_reader = metric_reader_fixture
db_session = db_session_fixture
_clear_bg_manager = _clear_bg_manager_fixture
_late_bound_sessions = _late_bound_sessions_fixture

SENTINEL = "T3A-SENTINEL"
COUNTER = "xagent.execution_event.unknown_outcomes"
PREFIX = "execution_event_outcome_unknown "


@pytest.fixture(autouse=True)
def _capture(caplog):
    caplog.set_level(logging.INFO, logger=execution_event_signals.logger.name)


def records(caplog) -> list[logging.LogRecord]:
    return [r for r in caplog.records if r.getMessage().startswith(PREFIX)]


def lines(caplog) -> list[str]:
    return [r.getMessage() for r in records(caplog)]


def counts(reader) -> dict[tuple[tuple[str, str], ...], int | float]:
    return exported(reader, COUNTER)


def assert_no_leak(caplog) -> None:
    """No line carries exception or payload text (every injected error and
    payload embeds ``SENTINEL``)."""
    for line in lines(caplog):
        assert SENTINEL not in line


def assert_one(
    caplog,
    reader,
    *,
    kind: str,
    task: int,
    run_id: str | None,
    ref: str | None,
    storage: str,
    level: int = logging.WARNING,
) -> dict[str, str]:
    """Exactly one line and one counter increment, with these fields."""
    found = records(caplog)
    assert len(found) == 1, [r.getMessage() for r in found]
    assert found[0].levelno == level
    line = found[0].getMessage()
    assert line.startswith(PREFIX + f"task_id={task} ")
    data = fields(line)
    assert data == {
        "task_id": str(task),
        "run_id": run_id or "-",
        "storage": storage,
        "component": "execution-events",
        "kind": kind,
        "ref": ref or "-",
    }
    assert counts(reader) == {key(operation=kind, storage=storage): 1}
    assert_no_leak(caplog)
    return data


def assert_none(caplog, reader) -> None:
    assert lines(caplog) == []
    assert counts(reader) == {}


# ------------------------------------------------------ report_unknown_outcome


def test_report_unknown_outcome_line_and_counter(caplog, metric_reader):
    report_unknown_outcome("turn_commit", task_id=7, run_id="run-1", ref="turn-1")
    report_unknown_outcome(
        "finalize_ack_reconciled", task_id=8, storage="v2", level=logging.INFO
    )

    first, second = records(caplog)
    assert first.levelno == logging.WARNING
    assert first.getMessage() == (
        "execution_event_outcome_unknown task_id=7 run_id=run-1 storage=unknown "
        "component=execution-events kind=turn_commit ref=turn-1"
    )
    assert second.levelno == logging.INFO
    assert second.getMessage() == (
        "execution_event_outcome_unknown task_id=8 run_id=- storage=v2 "
        "component=execution-events kind=finalize_ack_reconciled ref=-"
    )
    assert counts(metric_reader) == {
        key(operation="turn_commit", storage="unknown"): 1,
        key(operation="finalize_ack_reconciled", storage="v2"): 1,
    }
    assert first.exc_info is None and second.exc_info is None


def test_report_unknown_outcome_never_raises(monkeypatch, caplog):
    def boom(*_args, **_kwargs):
        raise RuntimeError(SENTINEL)

    monkeypatch.setattr(execution_event_signals, "increment_counter", boom)
    # A failing counter must not suppress the log line.
    report_unknown_outcome("tool_effect", task_id=1, storage="v2")
    assert len(lines(caplog)) == 1

    monkeypatch.setattr(execution_event_signals.logger, "log", boom)
    report_unknown_outcome("tool_effect", task_id=1, storage="v2")
    assert len(lines(caplog)) == 1


# --------------------------------------------------------------- tool_effect


@pytest.mark.asyncio
@pytest.mark.parametrize("pause_switch", ["true", "false"], ids=["decided", "legacy"])
@pytest.mark.parametrize("channel", [False, True], ids=["web", "channel"])
async def test_tool_effect_is_reported_once_per_settlement(
    canonical, monkeypatch, caplog, metric_reader, channel, pause_switch
):
    """Both settlement classifiers (the interruption decision and the legacy
    classification) report the same occurrence once, web and channel."""
    monkeypatch.setenv("XAGENT_TASK_INFRA_FAILURE_PAUSE_ENABLED", pause_switch)
    factory, tid = canonical
    with factory() as db:
        lease = acquire_task_lease(db, tid, new_run=True)
    failed = _fail_once(monkeypatch, "tool_execution_end")

    await _run_scheduled_turn(
        tid, lease, _react_execution(tid, FakeTool()), channel=channel
    )

    assert failed == ["tool_execution_end"]
    with factory() as db:
        assert (
            db.get(Task, tid).error_message == TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR
        )
    (attempt,) = _started_attempt_ids(factory, tid)
    assert_one(
        caplog,
        metric_reader,
        kind="tool_effect",
        task=tid,
        run_id=lease.run_id,
        ref=attempt,
        storage="v2",
    )


def _started_attempt_ids(factory, tid: int) -> list[str]:
    with factory() as db:
        return [
            str(attempt)
            for attempt in db.scalars(
                sa.select(TaskExecutionEvent.tool_attempt_id)
                .where(
                    TaskExecutionEvent.task_id == tid,
                    TaskExecutionEvent.kind == "tool_execution_start",
                )
                .order_by(TaskExecutionEvent.sequence)
            )
        ]


@pytest.mark.asyncio
async def test_unstarted_tool_is_not_a_tool_effect(
    canonical, monkeypatch, caplog, metric_reader
):
    monkeypatch.setenv("XAGENT_TASK_INFRA_FAILURE_PAUSE_ENABLED", "false")
    factory, tid = canonical
    with factory() as db:
        lease = acquire_task_lease(db, tid, new_run=True)
    failed = _fail_once(monkeypatch, "tool_execution_start")

    await _run_scheduled_turn(tid, lease, _react_execution(tid, FakeTool()))

    assert failed == ["tool_execution_start"]
    assert_none(caplog, metric_reader)


ATTEMPT = "attempt-unknown"


async def _run_with_unknown_tool(factory, tid: int) -> TaskLease:
    """A RUNNING run whose checkpoint names a started tool attempt with no
    committed outcome."""
    with factory() as db:
        lease = acquire_task_lease(db, tid, new_run=True)
    call = {
        "id": "call",
        "name": "calculator",
        "args": {},
        "assistant_message_id": "batch",
        "tool_attempt_id": ATTEMPT,
    }
    with bind_task_lease_context(lease):
        store = TraceCheckpointStore(tracer_for(tid))
        runtime = PatternRuntime(tracer=store)
        await store.save(
            {
                "execution_id": str(tid),
                "context": {"messages": []},
                "pattern_state": {"pending_tool_calls": [call]},
            }
        )
        await runtime.on_tool_start(tool_call=call)
    return lease


def _expire(factory, tid: int) -> None:
    with factory() as db:
        db.get(Task, tid).lease_expires_at = utc_now() - timedelta(seconds=5)
        db.commit()


async def _expire_run_with_unknown_tool(factory, tid: int) -> TaskLease:
    lease = await _run_with_unknown_tool(factory, tid)
    _expire(factory, tid)
    return lease


@pytest.mark.asyncio
async def test_tool_effect_is_reported_when_lease_recovery_settles_it(
    canonical, caplog, metric_reader
):
    factory, tid = canonical
    lease = await _expire_run_with_unknown_tool(factory, tid)
    with factory() as db:
        candidate = get_expired_task_lease_candidates(db, cutoff=utc_now(), limit=1)[0]

    assert (
        recover_task_lease_candidate_isolated(candidate, recovered_at=utc_now())
        == TaskStatus.FAILED
    )
    # A second attempt on the same candidate settles nothing and reports nothing.
    assert (
        recover_task_lease_candidate_isolated(candidate, recovered_at=utc_now()) is None
    )

    assert_one(
        caplog,
        metric_reader,
        kind="tool_effect",
        task=tid,
        run_id=lease.run_id,
        ref=ATTEMPT,
        storage="v2",
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("path", ["cas", "postgresql"])
async def test_tool_effect_is_reported_by_the_recovery_sweep(
    canonical, engine, monkeypatch, caplog, metric_reader, path
):
    """Both sweep paths, the CAS batch and the PostgreSQL row-lock batch."""
    from tests.web.services.test_task_lease_expiry_interruption import (
        _force_postgresql_batch,
    )

    factory, tid = canonical
    if path == "postgresql":
        _force_postgresql_batch(monkeypatch, engine)
    else:
        monkeypatch.setattr(
            task_lease_recovery, "_use_postgresql_recovery_partitioning", lambda: False
        )
    lease = await _expire_run_with_unknown_tool(factory, tid)

    batch = recover_expired_task_leases_batch_isolated(
        cutoff=utc_now(), batch_size=10, after=None
    )
    again = recover_expired_task_leases_batch_isolated(
        cutoff=utc_now(), batch_size=10, after=None
    )

    assert (batch.recovered, again.recovered) == (1, 0)
    assert_one(
        caplog,
        metric_reader,
        kind="tool_effect",
        task=tid,
        run_id=lease.run_id,
        ref=ATTEMPT,
        storage="v2",
    )


PERSISTENCE_RESULT = {
    "success": False,
    "status": "failed",
    "output": "All patterns failed",
    "error": "All 1 patterns failed: OperationalError: connection reset",
    "interruption_reason": "persistence_failure",
}


def _settle_unknown_tool_effect(path: str, factory, tid: int, lease: TaskLease) -> Any:
    """Settle the run through one settling path that decides it as an
    unknown tool effect."""
    from tests.web.services.test_persistence_failure_pause import (
        _persistence_interruption,
    )
    from xagent.web.services.managed_task_lease import (
        finalize_managed_task_lease_result,
    )

    if path == "settle_decided":
        return _settle(lease, interruption=_persistence_interruption())
    if path == "settle_classified":
        return _settle(lease)
    if path == "managed":
        with factory() as db:
            return finalize_managed_task_lease_result(
                db,
                lease,
                status=TaskStatus.FAILED,
                error_message="channel run abandoned",
            )
    if path == "finalize":
        return _finalize(factory, tid, lease, dict(PERSISTENCE_RESULT))
    assert path == "resume_finalize"
    with factory() as db:
        user_id = int(db.get(Task, tid).user_id)
    return _finalize_resumed_task(
        tid,
        status="failed",
        success=False,
        output="All patterns failed",
        task_owner_user_id=user_id,
        result=dict(PERSISTENCE_RESULT),
        task_lease=lease,
        prepared_outputs=NO_OUTPUTS,
    )


SETTLEMENT_PATHS = [
    "settle_decided",
    "settle_classified",
    "managed",
    "finalize",
    "resume_finalize",
]


@pytest.fixture
def settlement_commit(monkeypatch):
    """Refuse the COMMIT of the settlement transaction (nothing applied).

    That transaction is the one that stages the result fact; the staged row
    is recorded so a test can prove the settlement had decided the run as an
    unknown tool effect before it rolled back. The refusal is a deadlock, not
    a dropped connection, so no other unknown outcome is reported for it.
    """
    from xagent.web.services import task_execution_event_writer

    state: dict[str, Any] = {"refuse": False, "armed": False, "staged": []}
    stage = task_execution_event_writer.stage_result_fact_no_commit
    original = Session.commit

    def staged(db, task, *args, **kwargs):
        state["staged"].append(task.error_message)
        state["armed"] = True
        return stage(db, task, *args, **kwargs)

    def commit(session: Session) -> None:
        if not (state["armed"] and state["refuse"]):
            state["armed"] = False
            return original(session)
        state["armed"] = False
        raise _deadlock()

    monkeypatch.setattr(
        task_execution_event_writer, "stage_result_fact_no_commit", staged
    )
    monkeypatch.setattr(task_execution, "stage_result_fact_no_commit", staged)
    monkeypatch.setattr(Session, "commit", commit)
    return state


@pytest.fixture
def _settles_unknown_tool_effect(monkeypatch):
    """Settlement decides interruptions; nothing schedules a resume."""
    monkeypatch.setenv("XAGENT_TASK_INFRA_FAILURE_PAUSE_ENABLED", "true")
    monkeypatch.setenv("XAGENT_TASK_AUTO_RESUME_ENABLED", "false")


@pytest.mark.asyncio
@pytest.mark.parametrize("path", SETTLEMENT_PATHS)
@pytest.mark.usefixtures("_settles_unknown_tool_effect")
async def test_a_committed_settlement_reports_the_tool_effect_once(
    canonical, settlement_commit, caplog, metric_reader, path
):
    """Every settling path reports its unknown tool effect once, after its
    commit, carrying the attempt the verdict was reached on."""
    factory, tid = canonical
    lease = await _run_with_unknown_tool(factory, tid)

    _settle_unknown_tool_effect(path, factory, tid, lease)

    assert settlement_commit["staged"] == [TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR]
    with factory() as db:
        task = db.get(Task, tid)
        assert task.status == TaskStatus.FAILED
        assert task.error_message == TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR
    _expire(factory, tid)
    with factory() as db:
        assert not get_expired_task_lease_candidates(db, cutoff=utc_now(), limit=1)
    assert_one(
        caplog,
        metric_reader,
        kind="tool_effect",
        task=tid,
        run_id=lease.run_id,
        ref=ATTEMPT,
        storage="v2",
    )


@pytest.mark.asyncio
@pytest.mark.parametrize("path", SETTLEMENT_PATHS)
@pytest.mark.usefixtures("_settles_unknown_tool_effect")
async def test_a_rolled_back_settlement_leaves_the_one_report_to_lease_recovery(
    canonical, settlement_commit, caplog, metric_reader, path
):
    """A settlement that decided the unknown tool effect but did not commit
    reports nothing; TTL recovery settles the run and reports it once."""
    factory, tid = canonical
    lease = await _run_with_unknown_tool(factory, tid)
    settlement_commit["refuse"] = True

    with pytest.raises(OperationalError):
        _settle_unknown_tool_effect(path, factory, tid, lease)

    # The settle paths read the refused COMMIT back, find it absent and rerun
    # it, deciding the run again, until the retry attempts are used up.
    attempts = (
        DbWriteRetryPolicy.from_config().attempts if path.startswith("settle_") else 1
    )
    assert settlement_commit["staged"] == (
        [TASK_UNKNOWN_TOOL_EFFECT_SETTLEMENT_ERROR] * attempts
    )
    with factory() as db:
        assert db.get(Task, tid).status == TaskStatus.RUNNING
    assert_none(caplog, metric_reader)

    settlement_commit["refuse"] = False
    _expire(factory, tid)
    with factory() as db:
        candidate = get_expired_task_lease_candidates(db, cutoff=utc_now(), limit=1)[0]
    assert (
        recover_task_lease_candidate_isolated(candidate, recovered_at=utc_now())
        == TaskStatus.FAILED
    )

    assert_one(
        caplog,
        metric_reader,
        kind="tool_effect",
        task=tid,
        run_id=lease.run_id,
        ref=ATTEMPT,
        storage="v2",
    )


@pytest.mark.asyncio
async def test_recoverable_expired_run_is_not_a_tool_effect(
    canonical, monkeypatch, caplog, metric_reader
):
    from tests.web.services.test_persistence_failure_pause import _checkpoint

    monkeypatch.setattr(
        task_lease_recovery, "_use_postgresql_recovery_partitioning", lambda: False
    )
    factory, tid = canonical
    lease = _start_run(factory, tid)
    _checkpoint(factory, tid, lease.run_id)
    with factory() as db:
        db.get(Task, tid).lease_expires_at = utc_now() - timedelta(seconds=5)
        db.commit()

    batch = recover_expired_task_leases_batch_isolated(
        cutoff=utc_now(), batch_size=10, after=None
    )

    assert batch.recovered == 1
    assert_none(caplog, metric_reader)


# --------------------------------------------------------------- turn_commit


class _FailedCommitSession:
    def __init__(self, error: Exception) -> None:
        self.error = error

    def flush(self) -> None:
        return None

    def commit(self) -> None:
        raise self.error

    def rollback(self) -> None:
        return None

    def close(self) -> None:
        return None


def _claimed() -> _ClaimedTurn:
    return _ClaimedTurn(
        task_lease=MagicMock(runner_id="runner", run_id="run-turn"),
        status=TaskStatus.RUNNING,
        updated_at=None,
        before_message_id=None,
        task_source="sdk",
        run_id="run-turn",
    )


def _unreachable_reconciliation(monkeypatch, first_session=None) -> None:
    """Reconciliation cannot read: every Session after ``first_session`` fails."""
    calls = 0

    def session_factory():
        nonlocal calls
        calls += 1
        if first_session is not None and calls == 1:
            return first_session
        raise RuntimeError(f"reconciliation database unavailable {SENTINEL}")

    monkeypatch.setattr(database_module, "_SessionLocal", session_factory)
    monkeypatch.setattr(orchestrator.time, "sleep", MagicMock())


def test_turn_commit_is_reported_at_begin_turn(monkeypatch, caplog, metric_reader):
    commit_error = ConnectionError(f"commit acknowledgement lost {SENTINEL}")
    _unreachable_reconciliation(monkeypatch, _FailedCommitSession(commit_error))
    monkeypatch.setattr(
        orchestrator, "_claim_turn_no_commit", MagicMock(return_value=_claimed())
    )

    with pytest.raises(TaskTurnCommitOutcomeUnknown):
        _begin_turn_atomic_sync(
            123,
            456,
            payload=TaskTurnPayload(f"accepted {SENTINEL}", turn_id="turn-begin"),
            kind=TurnKind.CREATE,
        )

    assert_one(
        caplog,
        metric_reader,
        kind="turn_commit",
        task=123,
        run_id="run-turn",
        ref="turn-begin",
        storage="unknown",
    )


def test_turn_commit_is_reported_at_a_claimed_commit(
    monkeypatch, caplog, metric_reader
):
    commit_error = ConnectionError("commit acknowledgement lost")
    _unreachable_reconciliation(monkeypatch)

    with pytest.raises(TaskTurnCommitOutcomeUnknown):
        commit_claimed_turn_or_reconcile(
            _FailedCommitSession(commit_error),  # type: ignore[arg-type]
            task_id=124,
            task_owner_user_id=456,
            payload=TaskTurnPayload("accepted", turn_id="turn-claimed"),
            claimed=_claimed(),
        )

    assert_one(
        caplog,
        metric_reader,
        kind="turn_commit",
        task=124,
        run_id="run-turn",
        ref="turn-claimed",
        storage="unknown",
    )


def test_a_reconciled_turn_commit_is_not_reported(monkeypatch, caplog, metric_reader):
    monkeypatch.setattr(
        orchestrator,
        "_reconcile_claimed_turn_after_commit_ack_failure",
        lambda **_kwargs: True,
    )

    commit_claimed_turn_or_reconcile(
        _FailedCommitSession(ConnectionError("lost")),  # type: ignore[arg-type]
        task_id=125,
        task_owner_user_id=456,
        payload=TaskTurnPayload("accepted", turn_id="turn-reconciled"),
        claimed=_claimed(),
    )

    assert_none(caplog, metric_reader)


# ------------------------------------------------------------ command_commit


def _lose_every_commit(monkeypatch, db_session) -> None:
    original = Session.commit

    def lose(session: Session) -> None:
        if session is db_session:
            return original(session)
        raise _commit_error()

    monkeypatch.setattr(Session, "commit", lose)
    monkeypatch.setattr(task_command_execution.time, "sleep", MagicMock())


def test_command_commit_is_reported_for_a_live_delivery(
    db_session, monkeypatch, caplog, metric_reader
):
    owner = _owner_user(db_session, "unknown-live-owner")
    task = _owner_task(db_session, owner.id, status=TaskStatus.RUNNING)
    _lose_every_commit(monkeypatch, db_session)

    with pytest.raises(task_command_execution._TaskCommandCommitOutcomeUnknown):
        task_command_execution._claim_user_message_delivery_isolated(
            task_id=int(task.id),
            task_owner_user_id=int(owner.id),
            content=f"Apply {SENTINEL}",
            attachments=None,
            file_ids=[],
            turn_id="live-unknown-turn",
            expected_run_id="live-run",
            expected_status=TaskStatus.RUNNING,
        )

    assert_one(
        caplog,
        metric_reader,
        kind="command_commit",
        task=int(task.id),
        run_id="live-run",
        ref="live-unknown-turn",
        storage="unknown",
    )


def test_command_commit_is_reported_for_a_created_task_turn(
    db_session, monkeypatch, caplog, metric_reader
):
    owner = _owner_user(db_session, "unknown-created-owner")
    _lose_every_commit(monkeypatch, db_session)

    with pytest.raises(task_command_execution._TaskCommandCommitOutcomeUnknown):
        task_command_execution._prepare_task_message_sync(
            requested_task_id=987654,
            actor_user_id=int(owner.id),
            actor_is_admin=False,
            user_message=f"unknown acceptance {SENTINEL}",
            raw_context={},
            raw_files=[],
            client_message_id="created-unknown-turn",
            turn_id="created-unknown-turn",
            durable_attempt_count=1,
            durable_target_run_id=None,
            pause_accepted=False,
        )

    found = records(caplog)
    assert len(found) == 1
    data = fields(found[0].getMessage())
    assert data["kind"] == "command_commit"
    assert data["ref"] == "created-unknown-turn"
    assert data["storage"] == "unknown"
    assert data["run_id"] != "-"
    assert counts(metric_reader) == {
        key(operation="command_commit", storage="unknown"): 1
    }
    assert_no_leak(caplog)


def _fail_the_acceptance_flush(monkeypatch, db_session, arm_after: str) -> list[str]:
    """Fail the explicit flush right before the acceptance COMMIT.

    ``arm_after`` names the step that runs just before that flush; the next
    flush of a command Session (not the test's) raises. Nothing is written.
    """
    flushed: list[str] = []
    armed = {"on": False}
    original_flush = Session.flush

    def flush(session: Session, *args: Any, **kwargs: Any) -> None:
        if armed["on"] and session is not db_session:
            armed["on"] = False
            flushed.append("failed")
            raise OperationalError("INSERT", {}, Exception(f"flush failed {SENTINEL}"))
        return original_flush(session, *args, **kwargs)

    if arm_after == "claim_snapshot":
        step = task_command_execution._snapshot_user_message_delivery

        def armed_step(*args: Any, **kwargs: Any) -> Any:
            result = step(*args, **kwargs)
            armed["on"] = True
            return result

        monkeypatch.setattr(
            task_command_execution, "_snapshot_user_message_delivery", armed_step
        )
    else:
        assert arm_after == "created_turn_routing"
        orchestrator_cls = orchestrator.TaskTurnOrchestrator
        claim = orchestrator_cls.claim_created_turn_no_commit
        load = task_command_execution._load_task_command_routing_snapshot
        claimed = {"done": False}

        def armed_claim(*args: Any, **kwargs: Any) -> Any:
            result = claim(*args, **kwargs)
            claimed["done"] = True
            return result

        def armed_load(*args: Any, **kwargs: Any) -> Any:
            result = load(*args, **kwargs)
            if claimed["done"]:
                armed["on"] = True
            return result

        monkeypatch.setattr(
            orchestrator_cls, "claim_created_turn_no_commit", staticmethod(armed_claim)
        )
        monkeypatch.setattr(
            task_command_execution, "_load_task_command_routing_snapshot", armed_load
        )
    monkeypatch.setattr(Session, "flush", flush)
    monkeypatch.setattr(task_command_execution.time, "sleep", MagicMock())
    return flushed


def test_a_failed_live_delivery_flush_is_not_an_unknown_commit(
    db_session, monkeypatch, caplog, metric_reader
):
    """A flush that fails wrote nothing: the raised error is unchanged, but
    no unknown commit outcome is reported."""
    owner = _owner_user(db_session, "flush-live-owner")
    task = _owner_task(db_session, owner.id, status=TaskStatus.RUNNING)
    flushed = _fail_the_acceptance_flush(monkeypatch, db_session, "claim_snapshot")

    with pytest.raises(task_command_execution._TaskCommandCommitOutcomeUnknown):
        task_command_execution._claim_user_message_delivery_isolated(
            task_id=int(task.id),
            task_owner_user_id=int(owner.id),
            content=f"Apply {SENTINEL}",
            attachments=None,
            file_ids=[],
            turn_id="live-flush-turn",
            expected_run_id="live-run",
            expected_status=TaskStatus.RUNNING,
        )

    assert flushed == ["failed"]
    assert_none(caplog, metric_reader)


def test_a_failed_created_turn_flush_is_not_an_unknown_commit(
    db_session, monkeypatch, caplog, metric_reader
):
    owner = _owner_user(db_session, "flush-created-owner")
    flushed = _fail_the_acceptance_flush(
        monkeypatch, db_session, "created_turn_routing"
    )

    with pytest.raises(task_command_execution._TaskCommandCommitOutcomeUnknown):
        task_command_execution._prepare_task_message_sync(
            requested_task_id=987655,
            actor_user_id=int(owner.id),
            actor_is_admin=False,
            user_message=f"flush failure {SENTINEL}",
            raw_context={},
            raw_files=[],
            client_message_id="created-flush-turn",
            turn_id="created-flush-turn",
            durable_attempt_count=1,
            durable_target_run_id=None,
            pause_accepted=False,
        )

    assert flushed == ["failed"]
    assert_none(caplog, metric_reader)


# ------------------------------------------------------------------ finalize


def _commit_error() -> OperationalError:
    # psycopg2 reports a lost COMMIT acknowledgement as a dropped connection.
    return OperationalError(
        "COMMIT",
        {},
        Exception(f"commit acknowledgement lost {SENTINEL}"),
        connection_invalidated=True,
    )


def _deadlock() -> OperationalError:
    # PostgreSQL refused the COMMIT (40P01): a definite non-apply.
    return OperationalError("COMMIT", {}, Exception(f"deadlock detected {SENTINEL}"))


def _database_locked() -> OperationalError:
    # SQLite refused the COMMIT after its busy timeout: a definite non-apply.
    return OperationalError("COMMIT", {}, Exception("database is locked"))


def _integrity_error() -> IntegrityError:
    return IntegrityError("COMMIT", {}, Exception(f"deferred constraint {SENTINEL}"))


# Commit errors that cannot hide an applied COMMIT: the server refused it.
REFUSED_COMMITS = {
    "integrity": _integrity_error,
    "deadlock": _deadlock,
    "database_locked": _database_locked,
}


def _failing_db(error: Exception) -> MagicMock:
    db = MagicMock()
    db.commit.side_effect = error
    return db


FACT_WITNESS = FactWitness(
    task_id=31, scope_id="root", idempotency_key="result", event_id="event-1"
)
LEGACY_WITNESS = _ChatMessageWitness(
    task_id=31,
    message_id=5,
    role="assistant",
    message_type="final",
    content=f"answer {SENTINEL}",
)


def test_finalize_ack_reconciled_is_reported_at_info(
    monkeypatch, caplog, metric_reader
):
    monkeypatch.setattr(
        task_execution, "_finalization_witness_committed", lambda w: True
    )

    _commit_finalization_or_reconcile(
        _failing_db(_commit_error()),
        task_id=31,
        witness=FACT_WITNESS,
        run_id="run-f",
        storage="v2",
    )

    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_reconciled",
        task=31,
        run_id="run-f",
        ref=None,
        storage="v2",
        level=logging.INFO,
    )


@pytest.mark.parametrize(
    "witness, storage",
    [(FACT_WITNESS, "v2"), (LEGACY_WITNESS, "v1")],
    ids=["fact", "legacy"],
)
def test_finalize_ack_unresolved_when_the_read_back_fails(
    monkeypatch, caplog, metric_reader, witness, storage
):
    def read_back_fails(_witness):
        raise OSError(f"read-back failed {SENTINEL}")

    monkeypatch.setattr(
        task_execution, "_finalization_witness_committed", read_back_fails
    )
    error = _commit_error()

    with pytest.raises(OperationalError) as raised:
        _commit_finalization_or_reconcile(
            _failing_db(error),
            task_id=31,
            witness=witness,
            run_id="run-f",
            storage=storage,
        )

    assert raised.value is error
    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_unresolved",
        task=31,
        run_id="run-f",
        ref=None,
        storage=storage,
    )


def test_finalize_ack_unresolved_without_a_witness(monkeypatch, caplog, metric_reader):
    error = _commit_error()

    with pytest.raises(OperationalError) as raised:
        _commit_finalization_or_reconcile(_failing_db(error), task_id=31, witness=None)

    assert raised.value is error
    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_unresolved",
        task=31,
        run_id=None,
        ref=None,
        storage="unknown",
    )


def test_finalize_commit_proven_absent_is_a_plain_failure(
    monkeypatch, caplog, metric_reader
):
    """The read-back completed and found nothing: the commit did not apply."""
    monkeypatch.setattr(
        task_execution, "_finalization_witness_committed", lambda w: False
    )
    error = _commit_error()

    with pytest.raises(OperationalError) as raised:
        _commit_finalization_or_reconcile(
            _failing_db(error), task_id=31, witness=FACT_WITNESS, run_id="run-f"
        )

    assert raised.value is error
    assert_none(caplog, metric_reader)


@pytest.mark.parametrize("refused", REFUSED_COMMITS)
@pytest.mark.parametrize("witness", [None, FACT_WITNESS], ids=["no-witness", "fact"])
def test_finalize_commit_error_that_cannot_hide_an_applied_commit_is_not_unknown(
    monkeypatch, caplog, metric_reader, witness, refused
):
    """Only a dropped connection can lose the acknowledgement of an applied
    COMMIT; an OperationalError the server raised (a deadlock, SQLite's
    "database is locked") is a refusal, not a lost acknowledgement."""

    def read_back_fails(_witness):
        raise OSError("read-back failed")

    monkeypatch.setattr(
        task_execution, "_finalization_witness_committed", read_back_fails
    )
    error = REFUSED_COMMITS[refused]()

    with pytest.raises(type(error)) as raised:
        _commit_finalization_or_reconcile(
            _failing_db(error), task_id=31, witness=witness, run_id="run-f"
        )

    assert raised.value is error
    assert_none(caplog, metric_reader)


def test_finalize_ack_unresolved_on_a_closed_connection(caplog, metric_reader):
    """``InterfaceError`` is the DBAPI's connection-level class too."""
    error = InterfaceError("COMMIT", {}, Exception("connection already closed"))

    with pytest.raises(InterfaceError):
        _commit_finalization_or_reconcile(
            _failing_db(error), task_id=31, witness=None, run_id="run-f", storage="v1"
        )

    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_unresolved",
        task=31,
        run_id="run-f",
        ref=None,
        storage="v1",
    )


def test_finalize_ack_unresolved_through_a_wrapped_cause(caplog, metric_reader):
    wrapped = RuntimeError("finalize failed")
    wrapped.__cause__ = _commit_error()

    with pytest.raises(RuntimeError):
        _commit_finalization_or_reconcile(
            _failing_db(wrapped), task_id=31, witness=None, run_id="run-f"
        )

    assert [fields(line)["kind"] for line in lines(caplog)] == [
        "finalize_ack_unresolved"
    ]


def test_finalize_that_commits_normally_reports_nothing(caplog, metric_reader):
    _commit_finalization_or_reconcile(MagicMock(), task_id=31, witness=FACT_WITNESS)
    assert_none(caplog, metric_reader)


SUCCESS = {"success": True, "status": "completed", "output": "done"}


@pytest.fixture
def commit_fate(monkeypatch):
    """Lose the COMMIT of the finalize transaction, applied or not.

    The finalize transaction is the one that just staged the result fact.
    """
    state: dict[str, Any] = {"armed": False, "apply": True, "error": _commit_error}
    stage = task_execution.stage_result_fact_no_commit
    original = Session.commit

    def staged(*args, **kwargs):
        witness = stage(*args, **kwargs)
        state["armed"] = True
        return witness

    def commit(session: Session) -> None:
        if not state["armed"]:
            return original(session)
        state["armed"] = False
        if state["apply"]:
            original(session)
        raise state["error"]()

    monkeypatch.setattr(task_execution, "stage_result_fact_no_commit", staged)
    monkeypatch.setattr(Session, "commit", commit)
    return state


def test_a_lost_finalize_ack_the_read_back_confirms_is_reconciled(
    canonical, commit_fate, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)

    finalized = _finalize(factory, tid, lease, SUCCESS)

    assert finalized.terminal_state_committed
    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_reconciled",
        task=tid,
        run_id=lease.run_id,
        ref=None,
        storage="v2",
        level=logging.INFO,
    )


def test_a_lost_finalize_ack_with_no_read_back_is_unresolved(
    canonical, commit_fate, monkeypatch, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)
    commit_fate["apply"] = False

    def read_back_fails(_witness):
        raise CheckpointUnavailableError(f"read-back unavailable {SENTINEL}")

    monkeypatch.setattr(
        task_execution, "_finalization_witness_committed", read_back_fails
    )

    with pytest.raises(OperationalError):
        _finalize(factory, tid, lease, SUCCESS)

    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_unresolved",
        task=tid,
        run_id=lease.run_id,
        ref=None,
        storage="v2",
    )


def test_a_finalize_commit_the_read_back_proves_absent_is_not_unknown(
    canonical, commit_fate, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)
    commit_fate["apply"] = False
    assert (
        _finalization_witness_committed
        is task_execution._finalization_witness_committed
    )

    with pytest.raises(OperationalError):
        _finalize(factory, tid, lease, SUCCESS)

    assert_none(caplog, metric_reader)


# ------------------------------------------------- resumed finalize ack


def _finalize_resumed_success(factory, tid, lease):
    with factory() as db:
        user_id = int(db.get(Task, tid).user_id)
    return _finalize_resumed_task(
        tid,
        status="completed",
        success=True,
        output="done",
        task_owner_user_id=user_id,
        result=dict(SUCCESS),
        task_lease=lease,
        prepared_outputs=NO_OUTPUTS,
    )


def test_a_lost_resumed_finalize_ack_the_read_back_confirms_is_reconciled(
    canonical, commit_fate, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)

    finalized = _finalize_resumed_success(factory, tid, lease)

    assert finalized["lease_released"] is True
    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_reconciled",
        task=tid,
        run_id=lease.run_id,
        ref=None,
        storage="v2",
        level=logging.INFO,
    )


def test_a_lost_resumed_finalize_ack_with_no_read_back_is_unresolved(
    canonical, commit_fate, monkeypatch, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)
    commit_fate["apply"] = False

    def read_back_fails(_witness):
        raise CheckpointUnavailableError(f"read-back unavailable {SENTINEL}")

    monkeypatch.setattr(
        task_execution, "_finalization_witness_committed", read_back_fails
    )

    with pytest.raises(OperationalError):
        _finalize_resumed_success(factory, tid, lease)

    assert_one(
        caplog,
        metric_reader,
        kind="finalize_ack_unresolved",
        task=tid,
        run_id=lease.run_id,
        ref=None,
        storage="v2",
    )


def test_a_resumed_finalize_commit_the_read_back_proves_absent_is_not_unknown(
    canonical, commit_fate, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)
    commit_fate["apply"] = False

    with pytest.raises(OperationalError):
        _finalize_resumed_success(factory, tid, lease)

    assert_none(caplog, metric_reader)


@pytest.mark.parametrize("refused", REFUSED_COMMITS)
def test_resumed_finalize_commit_refusal_is_not_unknown(
    canonical, commit_fate, caplog, metric_reader, refused
):
    """A COMMIT the server refused did not apply: a plain failure."""
    factory, tid = canonical
    lease = _start_run(factory, tid)
    commit_fate["apply"] = False
    commit_fate["error"] = REFUSED_COMMITS[refused]
    with factory() as db:
        user_id = int(db.get(Task, tid).user_id)

    with pytest.raises((IntegrityError, OperationalError)):
        _finalize_resumed_task(
            tid,
            status="completed",
            success=True,
            output="done",
            task_owner_user_id=user_id,
            result=dict(SUCCESS),
            task_lease=lease,
            prepared_outputs=NO_OUTPUTS,
        )

    assert_none(caplog, metric_reader)


def test_resumed_finalize_that_commits_reports_nothing(
    canonical, caplog, metric_reader
):
    factory, tid = canonical
    lease = _start_run(factory, tid)
    with factory() as db:
        user_id = int(db.get(Task, tid).user_id)

    finalized = _finalize_resumed_task(
        tid,
        status="completed",
        success=True,
        output="done",
        task_owner_user_id=user_id,
        result=dict(SUCCESS),
        task_lease=lease,
        prepared_outputs=NO_OUTPUTS,
    )

    assert finalized["lease_released"]
    assert_none(caplog, metric_reader)


# --------------------------------------------------------- settlement_deferred


@pytest.mark.asyncio
async def test_a_deferred_run_result_is_reported_once(canonical, caplog, metric_reader):
    factory, tid = canonical
    lease = _start_run(factory, tid)

    async def execute(**_kwargs):
        raise InterruptionSettlementDeferred(f"checkpoint not resolvable {SENTINEL}")

    await _run_scheduled_turn(tid, lease, execute)

    assert_one(
        caplog,
        metric_reader,
        kind="settlement_deferred",
        task=tid,
        run_id=lease.run_id,
        ref=None,
        storage="unknown",
    )
    with factory() as db:
        assert db.get(Task, tid).status == TaskStatus.RUNNING


@pytest.mark.asyncio
async def test_a_deferral_raised_by_the_settlement_is_reported_once(
    canonical, monkeypatch, caplog, metric_reader
):
    """The run failed and its own settlement could not decide: the one
    occurrence is reported where that settlement's error is handled."""
    factory, tid = canonical
    lease = _start_run(factory, tid)

    async def execute(**_kwargs):
        raise RuntimeError("tool crashed")

    def deferred(*_args, **_kwargs):
        raise InterruptionSettlementDeferred(f"not decidable now {SENTINEL}")

    monkeypatch.setattr(orchestrator, "settle_task_lease_isolated", deferred)

    await _run_scheduled_turn(tid, lease, execute)

    assert_one(
        caplog,
        metric_reader,
        kind="settlement_deferred",
        task=tid,
        run_id=lease.run_id,
        ref=None,
        storage="unknown",
    )


def test_the_raising_settlement_itself_reports_nothing(
    canonical, monkeypatch, caplog, metric_reader
):
    """Only the handlers report, so a deferral never counts twice."""
    factory, tid = canonical
    lease = _start_run(factory, tid)

    def unavailable(*_args, **_kwargs):
        raise CheckpointUnavailableError("recovery read failed")

    from tests.web.services.test_persistence_failure_pause import (
        _checkpoint,
        _persistence_interruption,
    )

    _checkpoint(factory, tid, lease.run_id)
    monkeypatch.setattr(event_recovery, "read_event_checkpoint", unavailable)
    with pytest.raises(InterruptionSettlementDeferred):
        _settle(lease, interruption=_persistence_interruption())

    assert_none(caplog, metric_reader)


@pytest.mark.asyncio
async def test_a_deferred_resume_is_reported_once(caplog, metric_reader):
    settle = MagicMock(return_value=True)

    await _resume_raising(
        InterruptionSettlementDeferred(f"checkpoint not resolvable {SENTINEL}"),
        settle,
    )

    settle.assert_not_called()
    assert_one(
        caplog,
        metric_reader,
        kind="settlement_deferred",
        task=42,
        run_id="run-a",
        ref=None,
        storage="unknown",
    )


@pytest.mark.asyncio
async def test_a_resume_settlement_deferral_is_reported_once(caplog, metric_reader):
    settle = MagicMock(
        side_effect=InterruptionSettlementDeferred(f"not decidable now {SENTINEL}")
    )

    await _resume_raising(RuntimeError("tool crashed"), settle)

    settle.assert_called_once()
    assert_one(
        caplog,
        metric_reader,
        kind="settlement_deferred",
        task=42,
        run_id="run-a",
        ref=None,
        storage="unknown",
    )


@pytest.mark.asyncio
async def test_other_settlement_errors_are_not_deferrals(caplog, metric_reader):
    settle = MagicMock(side_effect=RuntimeError("settlement failed"))

    await _resume_raising(RuntimeError("tool crashed"), settle)

    assert_none(caplog, metric_reader)